Merge 'Setup tracing to allow output during test runs' from Preston Thorpe

## The problem:
Sometimes you want to run the tests _and_ look at logs, and previously
that wasn't possible because all the tests relied on stdout/stderr.
## The solution:
Setup `tracing-appender` to automatically log to a designated
`testing/test.log` when `RUST_LOG` is set.
This PR also starts a `testing.md` file describing testing in limbo. My
hope is that people who know stuff about DST and the fuzzing setup will
fill in the TODOs :)

Reviewed-by: Jussi Saurio <jussi.saurio@gmail.com>

Closes #1277

.gitignore

@@ -34,3 +34,4 @@ dist/
 # testing
 testing/limbo_output.txt
 **/limbo_output.txt
+testing/test.log

Cargo.lock

@@ -583,6 +583,15 @@ dependencies = [
  "itertools",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-deque"
 version = "0.8.6"
@@ -1678,6 +1687,7 @@ dependencies = [
  "shlex",
  "syntect",
  "tracing",
+ "tracing-appender",
  "tracing-subscriber",
 ]
 
@@ -3472,6 +3482,18 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-appender"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
+dependencies = [
+ "crossbeam-channel",
+ "thiserror 1.0.69",
+ "time",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-attributes"
 version = "0.1.28"

cli/Cargo.toml

@@ -39,6 +39,7 @@ rustyline = { version = "15.0.0", default-features = true, features = [
 shlex = "1.3.0"
 syntect = "5.2.0"
 tracing = "0.1.41"
+tracing-appender = "0.2.3"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
 
 

cli/app.rs

@@ -6,6 +6,8 @@ use crate::{
 };
 use comfy_table::{Attribute, Cell, CellAlignment, Color, ContentArrangement, Row, Table};
 use limbo_core::{Database, LimboError, OwnedValue, Statement, StepResult};
+use tracing_appender::non_blocking::WorkerGuard;
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
 use clap::Parser;
 use rustyline::{history::DefaultHistory, Editor};
@@ -49,6 +51,8 @@ pub struct Opts {
     pub vfs: Option<String>,
     #[clap(long, help = "Enable experimental MVCC feature")]
     pub experimental_mvcc: bool,
+    #[clap(short = 't', long, help = "specify output file for log traces")]
+    pub tracing_output: Option<String>,
 }
 
 const PROMPT: &str = "limbo> ";
@@ -130,6 +134,8 @@ impl<'a> Limbo<'a> {
             })
             .expect("Error setting Ctrl-C handler");
         }
+        let sql = opts.sql.clone();
+        let quiet = opts.quiet;
         let mut app = Self {
             prompt: PROMPT.to_string(),
             io,
@@ -137,21 +143,25 @@ impl<'a> Limbo<'a> {
             conn,
             interrupt_count,
             input_buff: String::new(),
-            opts: Settings::from(&opts),
+            opts: Settings::from(opts),
             rl,
         };
-
-        if opts.sql.is_some() {
-            app.handle_first_input(opts.sql.as_ref().unwrap());
-        }
-        if !opts.quiet {
-            app.write_fmt(format_args!("Limbo v{}", env!("CARGO_PKG_VERSION")))?;
-            app.writeln("Enter \".help\" for usage hints.")?;
-            app.display_in_memory()?;
-        }
+        app.first_run(sql, quiet)?;
         Ok(app)
     }
 
+    fn first_run(&mut self, sql: Option<String>, quiet: bool) -> io::Result<()> {
+        if let Some(sql) = sql {
+            self.handle_first_input(&sql);
+        }
+        if !quiet {
+            self.write_fmt(format_args!("Limbo v{}", env!("CARGO_PKG_VERSION")))?;
+            self.writeln("Enter \".help\" for usage hints.")?;
+            self.display_in_memory()?;
+        }
+        Ok(())
+    }
+
     fn handle_first_input(&mut self, cmd: &str) {
         if cmd.trim().starts_with('.') {
             self.handle_dot_command(&cmd[1..]);
@@ -695,6 +705,32 @@ impl<'a> Limbo<'a> {
         Ok(())
     }
 
+    pub fn init_tracing(&mut self) -> Result<WorkerGuard, std::io::Error> {
+        let (non_blocking, guard) = if let Some(file) = &self.opts.tracing_output {
+            tracing_appender::non_blocking(
+                std::fs::File::options()
+                    .append(true)
+                    .create(true)
+                    .open(file)?,
+            )
+        } else {
+            tracing_appender::non_blocking(std::io::stderr())
+        };
+        if let Err(e) = tracing_subscriber::registry()
+            .with(
+                tracing_subscriber::fmt::layer()
+                    .with_writer(non_blocking)
+                    .with_line_number(true)
+                    .with_thread_ids(true),
+            )
+            .with(EnvFilter::from_default_env())
+            .try_init()
+        {
+            println!("Unable to setup tracing appender: {:?}", e);
+        }
+        Ok(guard)
+    }
+
     fn display_schema(&mut self, table: Option<&str>) -> anyhow::Result<()> {
         let sql = match table {
         Some(table_name) => format!(

cli/input.rs

@@ -81,28 +81,30 @@ pub struct Settings {
     pub echo: bool,
     pub is_stdout: bool,
     pub io: Io,
+    pub tracing_output: Option<String>,
 }
 
-impl From<&Opts> for Settings {
-    fn from(opts: &Opts) -> Self {
+impl From<Opts> for Settings {
+    fn from(opts: Opts) -> Self {
         Self {
             null_value: String::new(),
             output_mode: opts.output_mode,
             echo: false,
             is_stdout: opts.output.is_empty(),
-            output_filename: opts.output.clone(),
+            output_filename: opts.output,
             db_file: opts
                 .database
                 .as_ref()
                 .map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string()),
             io: match opts.vfs.as_ref().unwrap_or(&String::new()).as_str() {
-                "memory" => Io::Memory,
+                "memory" | ":memory:" => Io::Memory,
                 "syscall" => Io::Syscall,
                 #[cfg(all(target_os = "linux", feature = "io_uring"))]
                 "io_uring" => Io::IoUring,
                 "" => Io::default(),
                 vfs => Io::External(vfs.to_string()),
             },
+            tracing_output: opts.tracing_output,
         }
     }
 }

cli/main.rs

@@ -7,7 +7,6 @@ mod opcodes_dictionary;
 
 use rustyline::{error::ReadlineError, Config, Editor};
 use std::sync::atomic::Ordering;
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
 fn rustyline_config() -> Config {
     Config::builder()
@@ -17,15 +16,8 @@ fn rustyline_config() -> Config {
 
 fn main() -> anyhow::Result<()> {
     let mut rl = Editor::with_config(rustyline_config())?;
-    tracing_subscriber::registry()
-        .with(
-            tracing_subscriber::fmt::layer()
-                .with_line_number(true)
-                .with_thread_ids(true),
-        )
-        .with(EnvFilter::from_default_env())
-        .init();
     let mut app = app::Limbo::new(&mut rl)?;
+    let _guard = app.init_tracing()?;
     let home = dirs::home_dir().expect("Could not determine home directory");
     let history_file = home.join(".limbo_history");
     if history_file.exists() {

docs/testing.md

@@ -0,0 +1,85 @@
+# Testing in Limbo
+
+Limbo supports a comprehensive testing system to ensure correctness, performance, and compatibility with SQLite.
+
+## 1. Compatibility Tests
+
+The `make test` target is the main entry point.
+
+Most compatibility tests live in the testing/ directory and are written in SQLite’s TCL test format. These tests ensure that Limbo matches SQLite’s behavior exactly. The database used during these tests is located at testing/testing.db, which includes the following schema:
+
+```sql
+CREATE TABLE users (
+    id INTEGER PRIMARY KEY,
+    first_name TEXT,
+    last_name TEXT,
+    email TEXT,
+    phone_number TEXT,
+    address TEXT,
+    city TEXT,
+    state TEXT,
+    zipcode TEXT,
+    age INTEGER
+);
+CREATE TABLE products (
+    id INTEGER PRIMARY KEY,
+    name TEXT,
+    price REAL
+);
+CREATE INDEX age_idx ON users (age);
+```
+
+You can freely write queries against these tables during compatibility testing.
+
+### Shell and Python-based Tests
+
+For cases where output or behavior differs intentionally from SQLite (e.g. due to new features or limitations), tests should be placed in the testing/cli_tests/ directory and written in Python.
+
+These tests use the TestLimboShell class:
+
+```python
+from cli_tests.common import TestLimboShell
+
+def test_uuid():
+    limbo = TestLimboShell()
+    limbo.run_test_fn("SELECT uuid4_str();", lambda res: len(res) == 36)
+    limbo.quit()
+```
+
+You can use run_test, run_test_fn, or debug_print to interact with the shell and validate results. 
+The constructor takes an optional argument with the `sql` you want to initiate the tests with.  You can also enable blob testing or override the executable and flags.
+
+Use these Python-based tests for validating:
+
+  - Output formatting
+
+  - Shell commands and .dot interactions
+
+  - Limbo-specific extensions in `testing/cli_tests/extensions.py`
+
+  - Any known divergence from SQLite behavior
+
+
+> Logging and Tracing
+If you wish to trace internal events during test execution, you can set the RUST_LOG environment variable before running the test. For example:
+
+```bash
+RUST_LOG=none,limbo_core=trace make test
+```
+
+This will enable trace-level logs for the limbo_core crate and disable logs elsewhere. Logging all internal traces to the `testing/test.log` file. 
+
+**Note:** trace logs can be very verbose—it's not uncommon for a single test run to generate megabytes of logs.
+
+
+## Deterministic Simulation Testing (DST):
+
+TODO!
+
+
+## Fuzzing
+
+TODO!
+
+
+

scripts/limbo-sqlite3

@@ -1,3 +1,8 @@
 #!/bin/bash
 
-target/debug/limbo -m list "$@"
+# if RUST_LOG is non-empty, enable tracing output
+if [ -n "$RUST_LOG" ]; then
+   target/debug/limbo -m list -t testing/test.log "$@"
+else
+   target/debug/limbo -m list "$@"
+fi