diff --git a/.gitignore b/.gitignore index 8a7437707..369a9b7ef 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ dist/ # testing testing/limbo_output.txt **/limbo_output.txt +testing/test.log diff --git a/Cargo.lock b/Cargo.lock index 2e7a615c6..96aec95db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -583,6 +583,15 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1678,6 +1687,7 @@ dependencies = [ "shlex", "syntect", "tracing", + "tracing-appender", "tracing-subscriber", ] @@ -3472,6 +3482,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror 1.0.69", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.28" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ddd44519f..2f1625420 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -39,6 +39,7 @@ rustyline = { version = "15.0.0", default-features = true, features = [ shlex = "1.3.0" syntect = "5.2.0" tracing = "0.1.41" +tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } diff --git a/cli/app.rs b/cli/app.rs index 40e187d43..e5aa851a6 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -6,6 +6,8 @@ use crate::{ }; use comfy_table::{Attribute, Cell, CellAlignment, Color, ContentArrangement, Row, Table}; use limbo_core::{Database, LimboError, OwnedValue, Statement, StepResult}; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; use clap::Parser; use rustyline::{history::DefaultHistory, Editor}; @@ -49,6 +51,8 @@ pub struct Opts { pub vfs: Option, #[clap(long, help = "Enable experimental MVCC feature")] pub experimental_mvcc: bool, + #[clap(short = 't', long, help = "specify output file for log traces")] + pub tracing_output: Option, } const PROMPT: &str = "limbo> "; @@ -130,6 +134,8 @@ impl<'a> Limbo<'a> { }) .expect("Error setting Ctrl-C handler"); } + let sql = opts.sql.clone(); + let quiet = opts.quiet; let mut app = Self { prompt: PROMPT.to_string(), io, @@ -137,21 +143,25 @@ impl<'a> Limbo<'a> { conn, interrupt_count, input_buff: String::new(), - opts: Settings::from(&opts), + opts: Settings::from(opts), rl, }; - - if opts.sql.is_some() { - app.handle_first_input(opts.sql.as_ref().unwrap()); - } - if !opts.quiet { - app.write_fmt(format_args!("Limbo v{}", env!("CARGO_PKG_VERSION")))?; - app.writeln("Enter \".help\" for usage hints.")?; - app.display_in_memory()?; - } + app.first_run(sql, quiet)?; Ok(app) } + fn first_run(&mut self, sql: Option, quiet: bool) -> io::Result<()> { + if let Some(sql) = sql { + self.handle_first_input(&sql); + } + if !quiet { + self.write_fmt(format_args!("Limbo v{}", env!("CARGO_PKG_VERSION")))?; + self.writeln("Enter \".help\" for usage hints.")?; + self.display_in_memory()?; + } + Ok(()) + } + fn handle_first_input(&mut self, cmd: &str) { if cmd.trim().starts_with('.') { self.handle_dot_command(&cmd[1..]); @@ -695,6 +705,32 @@ impl<'a> Limbo<'a> { Ok(()) } + pub fn init_tracing(&mut self) -> Result { + let (non_blocking, guard) = if let Some(file) = &self.opts.tracing_output { + tracing_appender::non_blocking( + std::fs::File::options() + .append(true) + .create(true) + .open(file)?, + ) + } else { + tracing_appender::non_blocking(std::io::stderr()) + }; + if let Err(e) = tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .with_writer(non_blocking) + .with_line_number(true) + .with_thread_ids(true), + ) + .with(EnvFilter::from_default_env()) + .try_init() + { + println!("Unable to setup tracing appender: {:?}", e); + } + Ok(guard) + } + fn display_schema(&mut self, table: Option<&str>) -> anyhow::Result<()> { let sql = match table { Some(table_name) => format!( diff --git a/cli/input.rs b/cli/input.rs index 4361394c0..7b505a99f 100644 --- a/cli/input.rs +++ b/cli/input.rs @@ -81,28 +81,30 @@ pub struct Settings { pub echo: bool, pub is_stdout: bool, pub io: Io, + pub tracing_output: Option, } -impl From<&Opts> for Settings { - fn from(opts: &Opts) -> Self { +impl From for Settings { + fn from(opts: Opts) -> Self { Self { null_value: String::new(), output_mode: opts.output_mode, echo: false, is_stdout: opts.output.is_empty(), - output_filename: opts.output.clone(), + output_filename: opts.output, db_file: opts .database .as_ref() .map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string()), io: match opts.vfs.as_ref().unwrap_or(&String::new()).as_str() { - "memory" => Io::Memory, + "memory" | ":memory:" => Io::Memory, "syscall" => Io::Syscall, #[cfg(all(target_os = "linux", feature = "io_uring"))] "io_uring" => Io::IoUring, "" => Io::default(), vfs => Io::External(vfs.to_string()), }, + tracing_output: opts.tracing_output, } } } diff --git a/cli/main.rs b/cli/main.rs index 4e8eca02a..ec81b64af 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -7,7 +7,6 @@ mod opcodes_dictionary; use rustyline::{error::ReadlineError, Config, Editor}; use std::sync::atomic::Ordering; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; fn rustyline_config() -> Config { Config::builder() @@ -17,15 +16,8 @@ fn rustyline_config() -> Config { fn main() -> anyhow::Result<()> { let mut rl = Editor::with_config(rustyline_config())?; - tracing_subscriber::registry() - .with( - tracing_subscriber::fmt::layer() - .with_line_number(true) - .with_thread_ids(true), - ) - .with(EnvFilter::from_default_env()) - .init(); let mut app = app::Limbo::new(&mut rl)?; + let _guard = app.init_tracing()?; let home = dirs::home_dir().expect("Could not determine home directory"); let history_file = home.join(".limbo_history"); if history_file.exists() { diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 000000000..21823957f --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,85 @@ +# Testing in Limbo + +Limbo supports a comprehensive testing system to ensure correctness, performance, and compatibility with SQLite. + +## 1. Compatibility Tests + +The `make test` target is the main entry point. + +Most compatibility tests live in the testing/ directory and are written in SQLite’s TCL test format. These tests ensure that Limbo matches SQLite’s behavior exactly. The database used during these tests is located at testing/testing.db, which includes the following schema: + +```sql +CREATE TABLE users ( + id INTEGER PRIMARY KEY, + first_name TEXT, + last_name TEXT, + email TEXT, + phone_number TEXT, + address TEXT, + city TEXT, + state TEXT, + zipcode TEXT, + age INTEGER +); +CREATE TABLE products ( + id INTEGER PRIMARY KEY, + name TEXT, + price REAL +); +CREATE INDEX age_idx ON users (age); +``` + +You can freely write queries against these tables during compatibility testing. + +### Shell and Python-based Tests + +For cases where output or behavior differs intentionally from SQLite (e.g. due to new features or limitations), tests should be placed in the testing/cli_tests/ directory and written in Python. + +These tests use the TestLimboShell class: + +```python +from cli_tests.common import TestLimboShell + +def test_uuid(): + limbo = TestLimboShell() + limbo.run_test_fn("SELECT uuid4_str();", lambda res: len(res) == 36) + limbo.quit() +``` + +You can use run_test, run_test_fn, or debug_print to interact with the shell and validate results. +The constructor takes an optional argument with the `sql` you want to initiate the tests with. You can also enable blob testing or override the executable and flags. + +Use these Python-based tests for validating: + + - Output formatting + + - Shell commands and .dot interactions + + - Limbo-specific extensions in `testing/cli_tests/extensions.py` + + - Any known divergence from SQLite behavior + + +> Logging and Tracing +If you wish to trace internal events during test execution, you can set the RUST_LOG environment variable before running the test. For example: + +```bash +RUST_LOG=none,limbo_core=trace make test +``` + +This will enable trace-level logs for the limbo_core crate and disable logs elsewhere. Logging all internal traces to the `testing/test.log` file. + +**Note:** trace logs can be very verbose—it's not uncommon for a single test run to generate megabytes of logs. + + +## Deterministic Simulation Testing (DST): + +TODO! + + +## Fuzzing + +TODO! + + + diff --git a/scripts/limbo-sqlite3 b/scripts/limbo-sqlite3 index 8e9f0389a..d448a2d6a 100755 --- a/scripts/limbo-sqlite3 +++ b/scripts/limbo-sqlite3 @@ -1,3 +1,8 @@ #!/bin/bash -target/debug/limbo -m list "$@" +# if RUST_LOG is non-empty, enable tracing output +if [ -n "$RUST_LOG" ]; then + target/debug/limbo -m list -t testing/test.log "$@" +else + target/debug/limbo -m list "$@" +fi