diff --git a/Cargo.lock b/Cargo.lock index 132bfdee9..2ec4c3d13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1733,6 +1733,7 @@ dependencies = [ "rand_chacha 0.3.1", "regex", "regex-syntax", + "rusqlite", "serde", "serde_json", "tempfile", diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index 0c2a355a1..8db80b810 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -28,3 +28,4 @@ clap = { version = "4.5", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } notify = "8.0.0" +rusqlite = { version = "0.29", features = ["bundled"] } diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index c5027e572..0b2952f21 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -11,7 +11,7 @@ use crate::{ }, table::Value, }, - runner::env::SimConnection, + runner::env::{SimConnection, SimulatorEnvTrait}, SimulatorEnv, }; @@ -239,7 +239,7 @@ impl Display for Interaction { } } -type AssertionFunc = dyn Fn(&Vec, &SimulatorEnv) -> Result; +type AssertionFunc = dyn Fn(&Vec, &dyn SimulatorEnvTrait) -> Result; enum AssertionAST { Pick(), @@ -523,7 +523,7 @@ impl Interaction { pub(crate) fn execute_assertion( &self, stack: &Vec, - env: &SimulatorEnv, + env: &impl SimulatorEnvTrait, ) -> Result<()> { match self { Self::Query(_) => { @@ -554,7 +554,7 @@ impl Interaction { pub(crate) fn execute_assumption( &self, stack: &Vec, - env: &SimulatorEnv, + env: &dyn SimulatorEnvTrait, ) -> Result<()> { match self { Self::Query(_) => { diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 61fbf5344..cbcd2c479 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -9,7 +9,7 @@ use crate::{ }, table::Value, }, - runner::env::SimulatorEnv, + runner::env::{SimulatorEnv, SimulatorEnvTrait}, }; use super::{ @@ -170,8 +170,8 @@ impl Property { message: format!("table {} exists", insert.table()), func: Box::new({ let table_name = table.clone(); - move |_: &Vec, env: &SimulatorEnv| { - Ok(env.tables.iter().any(|t| t.name == table_name)) + move |_: &Vec, env: &dyn SimulatorEnvTrait| { + Ok(env.tables().iter().any(|t| t.name == table_name)) } }), }); @@ -182,7 +182,7 @@ impl Property { row.iter().map(|v| v.to_string()).collect::>(), insert.table(), ), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + func: Box::new(move |stack: &Vec, _: &dyn SimulatorEnvTrait| { let rows = stack.last().unwrap(); match rows { Ok(rows) => Ok(rows.iter().any(|r| r == &row)), @@ -206,8 +206,8 @@ impl Property { let assumption = Interaction::Assumption(Assertion { message: "Double-Create-Failure should not be called on an existing table" .to_string(), - func: Box::new(move |_: &Vec, env: &SimulatorEnv| { - Ok(!env.tables.iter().any(|t| t.name == table_name)) + func: Box::new(move |_: &Vec, env: &dyn SimulatorEnvTrait| { + Ok(!env.tables().iter().any(|t| t.name == table_name)) }), }); @@ -220,7 +220,7 @@ impl Property { message: "creating two tables with the name should result in a failure for the second query" .to_string(), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + func: Box::new(move |stack: &Vec, _: &dyn SimulatorEnvTrait| { let last = stack.last().unwrap(); match last { Ok(_) => Ok(false), @@ -245,8 +245,8 @@ impl Property { message: format!("table {} exists", table_name), func: Box::new({ let table_name = table_name.clone(); - move |_: &Vec, env: &SimulatorEnv| { - Ok(env.tables.iter().any(|t| t.name == table_name)) + move |_: &Vec, env: &dyn SimulatorEnvTrait| { + Ok(env.tables().iter().any(|t| t.name == table_name)) } }), }); @@ -257,7 +257,7 @@ impl Property { let assertion = Interaction::Assertion(Assertion { message: "select query should respect the limit clause".to_string(), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + func: Box::new(move |stack: &Vec, _: &dyn SimulatorEnvTrait| { let last = stack.last().unwrap(); match last { Ok(rows) => Ok(limit >= rows.len()), @@ -281,8 +281,8 @@ impl Property { message: format!("table {} exists", table), func: Box::new({ let table = table.clone(); - move |_: &Vec, env: &SimulatorEnv| { - Ok(env.tables.iter().any(|t| t.name == table)) + move |_: &Vec, env: &dyn SimulatorEnvTrait| { + Ok(env.tables().iter().any(|t| t.name == table)) } }), }); @@ -292,7 +292,7 @@ impl Property { "select '{}' should return no values for table '{}'", predicate, table, ), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + func: Box::new(move |stack: &Vec, _: &dyn SimulatorEnvTrait| { let rows = stack.last().unwrap(); match rows { Ok(rows) => Ok(rows.is_empty()), @@ -332,8 +332,8 @@ impl Property { message: format!("table {} exists", table), func: Box::new({ let table = table.clone(); - move |_: &Vec, env: &SimulatorEnv| { - Ok(env.tables.iter().any(|t| t.name == table)) + move |_: &Vec, env: &dyn SimulatorEnvTrait| { + Ok(env.tables().iter().any(|t| t.name == table)) } }), }); @@ -345,7 +345,7 @@ impl Property { "select query should result in an error for table '{}'", table ), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + func: Box::new(move |stack: &Vec, _: &dyn SimulatorEnvTrait| { let last = stack.last().unwrap(); match last { Ok(_) => Ok(false), @@ -377,8 +377,8 @@ impl Property { message: format!("table {} exists", table), func: Box::new({ let table = table.clone(); - move |_: &Vec, env: &SimulatorEnv| { - Ok(env.tables.iter().any(|t| t.name == table)) + move |_: &Vec, env: &dyn SimulatorEnvTrait| { + Ok(env.tables().iter().any(|t| t.name == table)) } }), }); @@ -401,7 +401,7 @@ impl Property { let assertion = Interaction::Assertion(Assertion { message: "select queries should return the same amount of results".to_string(), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + func: Box::new(move |stack: &Vec, _: &dyn SimulatorEnvTrait| { let select_star = stack.last().unwrap(); let select_predicate = stack.get(stack.len() - 2).unwrap(); match (select_predicate, select_star) { diff --git a/simulator/main.rs b/simulator/main.rs index e8a5b34cf..a2f07d95e 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -9,7 +9,7 @@ use rand::prelude::*; use runner::cli::SimulatorCLI; use runner::env::SimulatorEnv; use runner::execution::{execute_plans, Execution, ExecutionHistory, ExecutionResult}; -use runner::watch; +use runner::{differential, watch}; use std::any::Any; use std::backtrace::Backtrace; use std::io::Write; @@ -85,10 +85,30 @@ fn main() -> Result<(), String> { if cli_opts.watch { watch_mode(seed, &cli_opts, &paths, last_execution.clone()).unwrap(); + } else if cli_opts.differential { + differential_testing(env, plans, last_execution.clone()) } else { - run_simulator(seed, &cli_opts, &paths, env, plans, last_execution.clone()); + run_simulator(&cli_opts, &paths, env, plans, last_execution.clone()); } + // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. + println!("database path: {:?}", paths.db); + if cli_opts.doublecheck { + println!("doublecheck database path: {:?}", paths.doublecheck_db); + } else if cli_opts.shrink { + println!("shrunk database path: {:?}", paths.shrunk_db); + } + println!("simulator plan path: {:?}", paths.plan); + println!( + "simulator plan serialized path: {:?}", + paths.plan.with_extension("plan.json") + ); + if cli_opts.shrink { + println!("shrunk plan path: {:?}", paths.shrunk_plan); + } + println!("simulator history path: {:?}", paths.history); + println!("seed: {}", seed); + Ok(()) } @@ -153,7 +173,6 @@ fn watch_mode( } fn run_simulator( - seed: u64, cli_opts: &SimulatorCLI, paths: &Paths, env: SimulatorEnv, @@ -278,24 +297,6 @@ fn run_simulator( } } } - - // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. - println!("database path: {:?}", paths.db); - if cli_opts.doublecheck { - println!("doublecheck database path: {:?}", paths.doublecheck_db); - } else if cli_opts.shrink { - println!("shrunk database path: {:?}", paths.shrunk_db); - } - println!("simulator plan path: {:?}", paths.plan); - println!( - "simulator plan serialized path: {:?}", - paths.plan.with_extension("plan.json") - ); - if cli_opts.shrink { - println!("shrunk plan path: {:?}", paths.shrunk_plan); - } - println!("simulator history path: {:?}", paths.history); - println!("seed: {}", seed); } fn doublecheck( @@ -361,6 +362,29 @@ fn doublecheck( } } +fn differential_testing( + env: SimulatorEnv, + plans: Vec, + last_execution: Arc>, +) { + let env = Arc::new(Mutex::new(env)); + let result = SandboxedResult::from( + std::panic::catch_unwind(|| { + let plan = plans[0].clone(); + differential::run_simulation(env, &mut [plan], last_execution.clone()) + }), + last_execution.clone(), + ); + + if let SandboxedResult::Correct = result { + log::info!("simulation succeeded"); + println!("simulation succeeded"); + } else { + log::error!("simulation failed"); + println!("simulation failed"); + } +} + #[derive(Debug)] enum SandboxedResult { Panicked { diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index aa3697b27..a18c47212 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -49,6 +49,8 @@ pub struct SimulatorCLI { help = "enable watch mode that reruns the simulation on file changes" )] pub watch: bool, + #[clap(long, help = "run differential testing between sqlite and Limbo")] + pub differential: bool, } impl SimulatorCLI { diff --git a/simulator/runner/differential.rs b/simulator/runner/differential.rs new file mode 100644 index 000000000..bfadc5687 --- /dev/null +++ b/simulator/runner/differential.rs @@ -0,0 +1,326 @@ +use std::sync::{Arc, Mutex}; + +use crate::{ + generation::{ + pick_index, + plan::{Interaction, InteractionPlanState, ResultSet}, + }, + model::{ + query::Query, + table::{Table, Value}, + }, + runner::execution::ExecutionContinuation, + InteractionPlan, +}; + +use super::{ + env::{ConnectionTrait, SimConnection, SimulatorEnv, SimulatorEnvTrait}, + execution::{execute_interaction, Execution, ExecutionHistory, ExecutionResult}, +}; + +pub(crate) struct SimulatorEnvRusqlite { + pub(crate) tables: Vec, + pub(crate) connections: Vec, +} + +pub(crate) enum RusqliteConnection { + Connected(rusqlite::Connection), + Disconnected, +} + +impl ConnectionTrait for RusqliteConnection { + fn is_connected(&self) -> bool { + match self { + RusqliteConnection::Connected(_) => true, + RusqliteConnection::Disconnected => false, + } + } + + fn disconnect(&mut self) { + *self = RusqliteConnection::Disconnected; + } +} + +impl SimulatorEnvTrait for SimulatorEnvRusqlite { + fn tables(&self) -> &Vec
{ + &self.tables + } + + fn tables_mut(&mut self) -> &mut Vec
{ + &mut self.tables + } +} + +pub(crate) fn run_simulation( + env: Arc>, + plans: &mut [InteractionPlan], + last_execution: Arc>, +) -> ExecutionResult { + log::info!("Executing database interaction plan..."); + + let mut states = plans + .iter() + .map(|_| InteractionPlanState { + stack: vec![], + interaction_pointer: 0, + secondary_pointer: 0, + }) + .collect::>(); + let env = env.lock().unwrap(); + + let rusqlite_env = SimulatorEnvRusqlite { + tables: env.tables.clone(), + connections: (0..env.connections.len()) + .map(|_| RusqliteConnection::Connected(rusqlite::Connection::open_in_memory().unwrap())) + .collect::>(), + }; + let mut rusqlite_states = plans + .iter() + .map(|_| InteractionPlanState { + stack: vec![], + interaction_pointer: 0, + secondary_pointer: 0, + }) + .collect::>(); + + let result = execute_plans( + Arc::new(Mutex::new(env.clone())), + rusqlite_env, + plans, + &mut states, + &mut rusqlite_states, + last_execution, + ); + + env.io.print_stats(); + + log::info!("Simulation completed"); + + result +} + +fn execute_query_rusqlite( + connection: &rusqlite::Connection, + query: &Query, +) -> rusqlite::Result>> { + match query { + Query::Create(create) => { + connection.execute(create.to_string().as_str(), ())?; + Ok(vec![]) + } + Query::Select(select) => { + let mut stmt = connection.prepare(select.to_string().as_str())?; + let columns = stmt.column_count(); + let rows = stmt.query_map([], |row| { + let mut values = vec![]; + for i in 0..columns { + let value = row.get_unwrap(i); + match value { + rusqlite::types::Value::Null => values.push(Value::Null), + rusqlite::types::Value::Integer(i) => values.push(Value::Integer(i)), + rusqlite::types::Value::Real(f) => values.push(Value::Float(f)), + rusqlite::types::Value::Text(s) => values.push(Value::Text(s)), + rusqlite::types::Value::Blob(b) => values.push(Value::Blob(b)), + } + } + Ok(values) + })?; + let mut result = vec![]; + for row in rows { + result.push(row?); + } + Ok(result) + } + Query::Insert(insert) => { + connection.execute(insert.to_string().as_str(), ())?; + Ok(vec![]) + } + Query::Delete(delete) => { + connection.execute(delete.to_string().as_str(), ())?; + Ok(vec![]) + } + Query::Drop(drop) => { + connection.execute(drop.to_string().as_str(), ())?; + Ok(vec![]) + } + } +} + +pub(crate) fn execute_plans( + env: Arc>, + mut rusqlite_env: SimulatorEnvRusqlite, + plans: &mut [InteractionPlan], + states: &mut [InteractionPlanState], + rusqlite_states: &mut [InteractionPlanState], + last_execution: Arc>, +) -> ExecutionResult { + let mut history = ExecutionHistory::new(); + let now = std::time::Instant::now(); + + let mut env = env.lock().unwrap(); + for _tick in 0..env.opts.ticks { + // Pick the connection to interact with + let connection_index = pick_index(env.connections.len(), &mut env.rng); + let state = &mut states[connection_index]; + + history.history.push(Execution::new( + connection_index, + state.interaction_pointer, + state.secondary_pointer, + )); + let mut last_execution = last_execution.lock().unwrap(); + last_execution.connection_index = connection_index; + last_execution.interaction_index = state.interaction_pointer; + last_execution.secondary_index = state.secondary_pointer; + // Execute the interaction for the selected connection + match execute_plan( + &mut env, + &mut rusqlite_env, + connection_index, + plans, + states, + rusqlite_states, + ) { + Ok(_) => {} + Err(err) => { + return ExecutionResult::new(history, Some(err)); + } + } + // Check if the maximum time for the simulation has been reached + if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { + return ExecutionResult::new( + history, + Some(limbo_core::LimboError::InternalError( + "maximum time for simulation reached".into(), + )), + ); + } + } + + ExecutionResult::new(history, None) +} + +fn execute_plan( + env: &mut SimulatorEnv, + rusqlite_env: &mut SimulatorEnvRusqlite, + connection_index: usize, + plans: &mut [InteractionPlan], + states: &mut [InteractionPlanState], + rusqlite_states: &mut [InteractionPlanState], +) -> limbo_core::Result<()> { + let connection = &env.connections[connection_index]; + let plan = &mut plans[connection_index]; + let state = &mut states[connection_index]; + let rusqlite_state = &mut rusqlite_states[connection_index]; + if state.interaction_pointer >= plan.plan.len() { + return Ok(()); + } + + let interaction = &plan.plan[state.interaction_pointer].interactions()[state.secondary_pointer]; + + if let SimConnection::Disconnected = connection { + log::info!("connecting {}", connection_index); + env.connections[connection_index] = SimConnection::Connected(env.db.connect()); + } else { + let limbo_result = + execute_interaction(env, connection_index, interaction, &mut state.stack); + let ruqlite_result = execute_interaction_rusqlite( + rusqlite_env, + connection_index, + interaction, + &mut rusqlite_state.stack, + ); + + match (limbo_result, ruqlite_result) { + (Ok(next_execution), Ok(next_execution_rusqlite)) => { + if next_execution != next_execution_rusqlite { + log::error!("limbo and rusqlite results do not match"); + return Err(limbo_core::LimboError::InternalError( + "limbo and rusqlite results do not match".into(), + )); + } + log::debug!("connection {} processed", connection_index); + // Move to the next interaction or property + match next_execution { + ExecutionContinuation::NextInteraction => { + if state.secondary_pointer + 1 + >= plan.plan[state.interaction_pointer].interactions().len() + { + // If we have reached the end of the interactions for this property, move to the next property + state.interaction_pointer += 1; + state.secondary_pointer = 0; + } else { + // Otherwise, move to the next interaction + state.secondary_pointer += 1; + } + } + ExecutionContinuation::NextProperty => { + // Skip to the next property + state.interaction_pointer += 1; + state.secondary_pointer = 0; + } + } + } + (Err(err), Ok(_)) => { + log::error!("limbo and rusqlite results do not match"); + log::error!("limbo error {}", err); + return Err(err); + } + (Ok(_), Err(err)) => { + log::error!("limbo and rusqlite results do not match"); + log::error!("rusqlite error {}", err); + return Err(err); + } + (Err(err), Err(err_rusqlite)) => { + log::error!("limbo and rusqlite both fail, requires manual check"); + log::error!("limbo error {}", err); + log::error!("rusqlite error {}", err_rusqlite); + return Err(err); + } + } + } + + Ok(()) +} + +fn execute_interaction_rusqlite( + env: &mut SimulatorEnvRusqlite, + connection_index: usize, + interaction: &Interaction, + stack: &mut Vec, +) -> limbo_core::Result { + log::info!("executing in rusqlite: {}", interaction); + match interaction { + Interaction::Query(query) => { + let conn = match &mut env.connections[connection_index] { + RusqliteConnection::Connected(conn) => conn, + RusqliteConnection::Disconnected => unreachable!(), + }; + + log::debug!("{}", interaction); + let results = execute_query_rusqlite(conn, query).map_err(|e| { + limbo_core::LimboError::InternalError(format!("error executing query: {}", e)) + }); + log::debug!("{:?}", results); + stack.push(results); + } + Interaction::Assertion(_) => { + interaction.execute_assertion(stack, env)?; + stack.clear(); + } + Interaction::Assumption(_) => { + let assumption_result = interaction.execute_assumption(stack, env); + stack.clear(); + + if assumption_result.is_err() { + log::warn!("assumption failed: {:?}", assumption_result); + return Ok(ExecutionContinuation::NextProperty); + } + } + Interaction::Fault(_) => { + log::debug!("faults are not supported in differential testing mode"); + } + } + + Ok(ExecutionContinuation::NextInteraction) +} diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs index 93c3b4b2a..1b7240b9b 100644 --- a/simulator/runner/env.rs +++ b/simulator/runner/env.rs @@ -12,6 +12,11 @@ use crate::runner::io::SimulatorIO; use super::cli::SimulatorCLI; +pub trait SimulatorEnvTrait { + fn tables(&self) -> &Vec
; + fn tables_mut(&mut self) -> &mut Vec
; +} + #[derive(Clone)] pub(crate) struct SimulatorEnv { pub(crate) opts: SimulatorOpts, @@ -22,6 +27,16 @@ pub(crate) struct SimulatorEnv { pub(crate) rng: ChaCha8Rng, } +impl SimulatorEnvTrait for SimulatorEnv { + fn tables(&self) -> &Vec
{ + &self.tables + } + + fn tables_mut(&mut self) -> &mut Vec
{ + &mut self.tables + } +} + impl SimulatorEnv { pub(crate) fn new(seed: u64, cli_opts: &SimulatorCLI, db_path: &Path) -> Self { let mut rng = ChaCha8Rng::seed_from_u64(seed); @@ -92,12 +107,30 @@ impl SimulatorEnv { } } +pub trait ConnectionTrait { + fn is_connected(&self) -> bool; + fn disconnect(&mut self); +} + #[derive(Clone)] pub(crate) enum SimConnection { Connected(Rc), Disconnected, } +impl ConnectionTrait for SimConnection { + fn is_connected(&self) -> bool { + match self { + SimConnection::Connected(_) => true, + SimConnection::Disconnected => false, + } + } + + fn disconnect(&mut self) { + *self = SimConnection::Disconnected; + } +} + #[derive(Debug, Clone)] pub(crate) struct SimulatorOpts { pub(crate) ticks: usize, diff --git a/simulator/runner/execution.rs b/simulator/runner/execution.rs index 6342dff3a..822660260 100644 --- a/simulator/runner/execution.rs +++ b/simulator/runner/execution.rs @@ -157,6 +157,7 @@ fn execute_plan( /// `execute_interaction` uses this type in conjunction with a result, where /// the `Err` case indicates a full-stop due to a bug, and the `Ok` case /// indicates the next step in the plan. +#[derive(PartialEq)] pub(crate) enum ExecutionContinuation { /// Default continuation, execute the next interaction. NextInteraction, diff --git a/simulator/runner/mod.rs b/simulator/runner/mod.rs index 2eabaef8b..36a6fbb0a 100644 --- a/simulator/runner/mod.rs +++ b/simulator/runner/mod.rs @@ -1,4 +1,5 @@ pub mod cli; +pub mod differential; pub mod env; pub mod execution; #[allow(dead_code)]