Merge 'simulator: --differential mode against SQLite' from Alperen Keleş

Closes #987
This commit is contained in:
Pekka Enberg
2025-02-12 09:15:57 +02:00
10 changed files with 433 additions and 44 deletions

1
Cargo.lock generated
View File

@@ -1733,6 +1733,7 @@ dependencies = [
"rand_chacha 0.3.1",
"regex",
"regex-syntax",
"rusqlite",
"serde",
"serde_json",
"tempfile",

View File

@@ -28,3 +28,4 @@ clap = { version = "4.5", features = ["derive"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0" }
notify = "8.0.0"
rusqlite = { version = "0.29", features = ["bundled"] }

View File

@@ -11,7 +11,7 @@ use crate::{
},
table::Value,
},
runner::env::SimConnection,
runner::env::{SimConnection, SimulatorEnvTrait},
SimulatorEnv,
};
@@ -239,7 +239,7 @@ impl Display for Interaction {
}
}
type AssertionFunc = dyn Fn(&Vec<ResultSet>, &SimulatorEnv) -> Result<bool>;
type AssertionFunc = dyn Fn(&Vec<ResultSet>, &dyn SimulatorEnvTrait) -> Result<bool>;
enum AssertionAST {
Pick(),
@@ -523,7 +523,7 @@ impl Interaction {
pub(crate) fn execute_assertion(
&self,
stack: &Vec<ResultSet>,
env: &SimulatorEnv,
env: &impl SimulatorEnvTrait,
) -> Result<()> {
match self {
Self::Query(_) => {
@@ -554,7 +554,7 @@ impl Interaction {
pub(crate) fn execute_assumption(
&self,
stack: &Vec<ResultSet>,
env: &SimulatorEnv,
env: &dyn SimulatorEnvTrait,
) -> Result<()> {
match self {
Self::Query(_) => {

View File

@@ -9,7 +9,7 @@ use crate::{
},
table::Value,
},
runner::env::SimulatorEnv,
runner::env::{SimulatorEnv, SimulatorEnvTrait},
};
use super::{
@@ -170,8 +170,8 @@ impl Property {
message: format!("table {} exists", insert.table()),
func: Box::new({
let table_name = table.clone();
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table_name))
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table_name))
}
}),
});
@@ -182,7 +182,7 @@ impl Property {
row.iter().map(|v| v.to_string()).collect::<Vec<String>>(),
insert.table(),
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
let rows = stack.last().unwrap();
match rows {
Ok(rows) => Ok(rows.iter().any(|r| r == &row)),
@@ -206,8 +206,8 @@ impl Property {
let assumption = Interaction::Assumption(Assertion {
message: "Double-Create-Failure should not be called on an existing table"
.to_string(),
func: Box::new(move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(!env.tables.iter().any(|t| t.name == table_name))
func: Box::new(move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(!env.tables().iter().any(|t| t.name == table_name))
}),
});
@@ -220,7 +220,7 @@ impl Property {
message:
"creating two tables with the name should result in a failure for the second query"
.to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
let last = stack.last().unwrap();
match last {
Ok(_) => Ok(false),
@@ -245,8 +245,8 @@ impl Property {
message: format!("table {} exists", table_name),
func: Box::new({
let table_name = table_name.clone();
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table_name))
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table_name))
}
}),
});
@@ -257,7 +257,7 @@ impl Property {
let assertion = Interaction::Assertion(Assertion {
message: "select query should respect the limit clause".to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
let last = stack.last().unwrap();
match last {
Ok(rows) => Ok(limit >= rows.len()),
@@ -281,8 +281,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
}
}),
});
@@ -292,7 +292,7 @@ impl Property {
"select '{}' should return no values for table '{}'",
predicate, table,
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
let rows = stack.last().unwrap();
match rows {
Ok(rows) => Ok(rows.is_empty()),
@@ -332,8 +332,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
}
}),
});
@@ -345,7 +345,7 @@ impl Property {
"select query should result in an error for table '{}'",
table
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
let last = stack.last().unwrap();
match last {
Ok(_) => Ok(false),
@@ -377,8 +377,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
}
}),
});
@@ -401,7 +401,7 @@ impl Property {
let assertion = Interaction::Assertion(Assertion {
message: "select queries should return the same amount of results".to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
let select_star = stack.last().unwrap();
let select_predicate = stack.get(stack.len() - 2).unwrap();
match (select_predicate, select_star) {

View File

@@ -9,7 +9,7 @@ use rand::prelude::*;
use runner::cli::SimulatorCLI;
use runner::env::SimulatorEnv;
use runner::execution::{execute_plans, Execution, ExecutionHistory, ExecutionResult};
use runner::watch;
use runner::{differential, watch};
use std::any::Any;
use std::backtrace::Backtrace;
use std::io::Write;
@@ -85,10 +85,30 @@ fn main() -> Result<(), String> {
if cli_opts.watch {
watch_mode(seed, &cli_opts, &paths, last_execution.clone()).unwrap();
} else if cli_opts.differential {
differential_testing(env, plans, last_execution.clone())
} else {
run_simulator(seed, &cli_opts, &paths, env, plans, last_execution.clone());
run_simulator(&cli_opts, &paths, env, plans, last_execution.clone());
}
// Print the seed, the locations of the database and the plan file at the end again for easily accessing them.
println!("database path: {:?}", paths.db);
if cli_opts.doublecheck {
println!("doublecheck database path: {:?}", paths.doublecheck_db);
} else if cli_opts.shrink {
println!("shrunk database path: {:?}", paths.shrunk_db);
}
println!("simulator plan path: {:?}", paths.plan);
println!(
"simulator plan serialized path: {:?}",
paths.plan.with_extension("plan.json")
);
if cli_opts.shrink {
println!("shrunk plan path: {:?}", paths.shrunk_plan);
}
println!("simulator history path: {:?}", paths.history);
println!("seed: {}", seed);
Ok(())
}
@@ -153,7 +173,6 @@ fn watch_mode(
}
fn run_simulator(
seed: u64,
cli_opts: &SimulatorCLI,
paths: &Paths,
env: SimulatorEnv,
@@ -278,24 +297,6 @@ fn run_simulator(
}
}
}
// Print the seed, the locations of the database and the plan file at the end again for easily accessing them.
println!("database path: {:?}", paths.db);
if cli_opts.doublecheck {
println!("doublecheck database path: {:?}", paths.doublecheck_db);
} else if cli_opts.shrink {
println!("shrunk database path: {:?}", paths.shrunk_db);
}
println!("simulator plan path: {:?}", paths.plan);
println!(
"simulator plan serialized path: {:?}",
paths.plan.with_extension("plan.json")
);
if cli_opts.shrink {
println!("shrunk plan path: {:?}", paths.shrunk_plan);
}
println!("simulator history path: {:?}", paths.history);
println!("seed: {}", seed);
}
fn doublecheck(
@@ -361,6 +362,29 @@ fn doublecheck(
}
}
fn differential_testing(
env: SimulatorEnv,
plans: Vec<InteractionPlan>,
last_execution: Arc<Mutex<Execution>>,
) {
let env = Arc::new(Mutex::new(env));
let result = SandboxedResult::from(
std::panic::catch_unwind(|| {
let plan = plans[0].clone();
differential::run_simulation(env, &mut [plan], last_execution.clone())
}),
last_execution.clone(),
);
if let SandboxedResult::Correct = result {
log::info!("simulation succeeded");
println!("simulation succeeded");
} else {
log::error!("simulation failed");
println!("simulation failed");
}
}
#[derive(Debug)]
enum SandboxedResult {
Panicked {

View File

@@ -49,6 +49,8 @@ pub struct SimulatorCLI {
help = "enable watch mode that reruns the simulation on file changes"
)]
pub watch: bool,
#[clap(long, help = "run differential testing between sqlite and Limbo")]
pub differential: bool,
}
impl SimulatorCLI {

View File

@@ -0,0 +1,326 @@
use std::sync::{Arc, Mutex};
use crate::{
generation::{
pick_index,
plan::{Interaction, InteractionPlanState, ResultSet},
},
model::{
query::Query,
table::{Table, Value},
},
runner::execution::ExecutionContinuation,
InteractionPlan,
};
use super::{
env::{ConnectionTrait, SimConnection, SimulatorEnv, SimulatorEnvTrait},
execution::{execute_interaction, Execution, ExecutionHistory, ExecutionResult},
};
pub(crate) struct SimulatorEnvRusqlite {
pub(crate) tables: Vec<Table>,
pub(crate) connections: Vec<RusqliteConnection>,
}
pub(crate) enum RusqliteConnection {
Connected(rusqlite::Connection),
Disconnected,
}
impl ConnectionTrait for RusqliteConnection {
fn is_connected(&self) -> bool {
match self {
RusqliteConnection::Connected(_) => true,
RusqliteConnection::Disconnected => false,
}
}
fn disconnect(&mut self) {
*self = RusqliteConnection::Disconnected;
}
}
impl SimulatorEnvTrait for SimulatorEnvRusqlite {
fn tables(&self) -> &Vec<Table> {
&self.tables
}
fn tables_mut(&mut self) -> &mut Vec<Table> {
&mut self.tables
}
}
pub(crate) fn run_simulation(
env: Arc<Mutex<SimulatorEnv>>,
plans: &mut [InteractionPlan],
last_execution: Arc<Mutex<Execution>>,
) -> ExecutionResult {
log::info!("Executing database interaction plan...");
let mut states = plans
.iter()
.map(|_| InteractionPlanState {
stack: vec![],
interaction_pointer: 0,
secondary_pointer: 0,
})
.collect::<Vec<_>>();
let env = env.lock().unwrap();
let rusqlite_env = SimulatorEnvRusqlite {
tables: env.tables.clone(),
connections: (0..env.connections.len())
.map(|_| RusqliteConnection::Connected(rusqlite::Connection::open_in_memory().unwrap()))
.collect::<Vec<_>>(),
};
let mut rusqlite_states = plans
.iter()
.map(|_| InteractionPlanState {
stack: vec![],
interaction_pointer: 0,
secondary_pointer: 0,
})
.collect::<Vec<_>>();
let result = execute_plans(
Arc::new(Mutex::new(env.clone())),
rusqlite_env,
plans,
&mut states,
&mut rusqlite_states,
last_execution,
);
env.io.print_stats();
log::info!("Simulation completed");
result
}
fn execute_query_rusqlite(
connection: &rusqlite::Connection,
query: &Query,
) -> rusqlite::Result<Vec<Vec<Value>>> {
match query {
Query::Create(create) => {
connection.execute(create.to_string().as_str(), ())?;
Ok(vec![])
}
Query::Select(select) => {
let mut stmt = connection.prepare(select.to_string().as_str())?;
let columns = stmt.column_count();
let rows = stmt.query_map([], |row| {
let mut values = vec![];
for i in 0..columns {
let value = row.get_unwrap(i);
match value {
rusqlite::types::Value::Null => values.push(Value::Null),
rusqlite::types::Value::Integer(i) => values.push(Value::Integer(i)),
rusqlite::types::Value::Real(f) => values.push(Value::Float(f)),
rusqlite::types::Value::Text(s) => values.push(Value::Text(s)),
rusqlite::types::Value::Blob(b) => values.push(Value::Blob(b)),
}
}
Ok(values)
})?;
let mut result = vec![];
for row in rows {
result.push(row?);
}
Ok(result)
}
Query::Insert(insert) => {
connection.execute(insert.to_string().as_str(), ())?;
Ok(vec![])
}
Query::Delete(delete) => {
connection.execute(delete.to_string().as_str(), ())?;
Ok(vec![])
}
Query::Drop(drop) => {
connection.execute(drop.to_string().as_str(), ())?;
Ok(vec![])
}
}
}
pub(crate) fn execute_plans(
env: Arc<Mutex<SimulatorEnv>>,
mut rusqlite_env: SimulatorEnvRusqlite,
plans: &mut [InteractionPlan],
states: &mut [InteractionPlanState],
rusqlite_states: &mut [InteractionPlanState],
last_execution: Arc<Mutex<Execution>>,
) -> ExecutionResult {
let mut history = ExecutionHistory::new();
let now = std::time::Instant::now();
let mut env = env.lock().unwrap();
for _tick in 0..env.opts.ticks {
// Pick the connection to interact with
let connection_index = pick_index(env.connections.len(), &mut env.rng);
let state = &mut states[connection_index];
history.history.push(Execution::new(
connection_index,
state.interaction_pointer,
state.secondary_pointer,
));
let mut last_execution = last_execution.lock().unwrap();
last_execution.connection_index = connection_index;
last_execution.interaction_index = state.interaction_pointer;
last_execution.secondary_index = state.secondary_pointer;
// Execute the interaction for the selected connection
match execute_plan(
&mut env,
&mut rusqlite_env,
connection_index,
plans,
states,
rusqlite_states,
) {
Ok(_) => {}
Err(err) => {
return ExecutionResult::new(history, Some(err));
}
}
// Check if the maximum time for the simulation has been reached
if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 {
return ExecutionResult::new(
history,
Some(limbo_core::LimboError::InternalError(
"maximum time for simulation reached".into(),
)),
);
}
}
ExecutionResult::new(history, None)
}
fn execute_plan(
env: &mut SimulatorEnv,
rusqlite_env: &mut SimulatorEnvRusqlite,
connection_index: usize,
plans: &mut [InteractionPlan],
states: &mut [InteractionPlanState],
rusqlite_states: &mut [InteractionPlanState],
) -> limbo_core::Result<()> {
let connection = &env.connections[connection_index];
let plan = &mut plans[connection_index];
let state = &mut states[connection_index];
let rusqlite_state = &mut rusqlite_states[connection_index];
if state.interaction_pointer >= plan.plan.len() {
return Ok(());
}
let interaction = &plan.plan[state.interaction_pointer].interactions()[state.secondary_pointer];
if let SimConnection::Disconnected = connection {
log::info!("connecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect());
} else {
let limbo_result =
execute_interaction(env, connection_index, interaction, &mut state.stack);
let ruqlite_result = execute_interaction_rusqlite(
rusqlite_env,
connection_index,
interaction,
&mut rusqlite_state.stack,
);
match (limbo_result, ruqlite_result) {
(Ok(next_execution), Ok(next_execution_rusqlite)) => {
if next_execution != next_execution_rusqlite {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
log::debug!("connection {} processed", connection_index);
// Move to the next interaction or property
match next_execution {
ExecutionContinuation::NextInteraction => {
if state.secondary_pointer + 1
>= plan.plan[state.interaction_pointer].interactions().len()
{
// If we have reached the end of the interactions for this property, move to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
} else {
// Otherwise, move to the next interaction
state.secondary_pointer += 1;
}
}
ExecutionContinuation::NextProperty => {
// Skip to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
}
}
}
(Err(err), Ok(_)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo error {}", err);
return Err(err);
}
(Ok(_), Err(err)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("rusqlite error {}", err);
return Err(err);
}
(Err(err), Err(err_rusqlite)) => {
log::error!("limbo and rusqlite both fail, requires manual check");
log::error!("limbo error {}", err);
log::error!("rusqlite error {}", err_rusqlite);
return Err(err);
}
}
}
Ok(())
}
fn execute_interaction_rusqlite(
env: &mut SimulatorEnvRusqlite,
connection_index: usize,
interaction: &Interaction,
stack: &mut Vec<ResultSet>,
) -> limbo_core::Result<ExecutionContinuation> {
log::info!("executing in rusqlite: {}", interaction);
match interaction {
Interaction::Query(query) => {
let conn = match &mut env.connections[connection_index] {
RusqliteConnection::Connected(conn) => conn,
RusqliteConnection::Disconnected => unreachable!(),
};
log::debug!("{}", interaction);
let results = execute_query_rusqlite(conn, query).map_err(|e| {
limbo_core::LimboError::InternalError(format!("error executing query: {}", e))
});
log::debug!("{:?}", results);
stack.push(results);
}
Interaction::Assertion(_) => {
interaction.execute_assertion(stack, env)?;
stack.clear();
}
Interaction::Assumption(_) => {
let assumption_result = interaction.execute_assumption(stack, env);
stack.clear();
if assumption_result.is_err() {
log::warn!("assumption failed: {:?}", assumption_result);
return Ok(ExecutionContinuation::NextProperty);
}
}
Interaction::Fault(_) => {
log::debug!("faults are not supported in differential testing mode");
}
}
Ok(ExecutionContinuation::NextInteraction)
}

View File

@@ -12,6 +12,11 @@ use crate::runner::io::SimulatorIO;
use super::cli::SimulatorCLI;
pub trait SimulatorEnvTrait {
fn tables(&self) -> &Vec<Table>;
fn tables_mut(&mut self) -> &mut Vec<Table>;
}
#[derive(Clone)]
pub(crate) struct SimulatorEnv {
pub(crate) opts: SimulatorOpts,
@@ -22,6 +27,16 @@ pub(crate) struct SimulatorEnv {
pub(crate) rng: ChaCha8Rng,
}
impl SimulatorEnvTrait for SimulatorEnv {
fn tables(&self) -> &Vec<Table> {
&self.tables
}
fn tables_mut(&mut self) -> &mut Vec<Table> {
&mut self.tables
}
}
impl SimulatorEnv {
pub(crate) fn new(seed: u64, cli_opts: &SimulatorCLI, db_path: &Path) -> Self {
let mut rng = ChaCha8Rng::seed_from_u64(seed);
@@ -92,12 +107,30 @@ impl SimulatorEnv {
}
}
pub trait ConnectionTrait {
fn is_connected(&self) -> bool;
fn disconnect(&mut self);
}
#[derive(Clone)]
pub(crate) enum SimConnection {
Connected(Rc<Connection>),
Disconnected,
}
impl ConnectionTrait for SimConnection {
fn is_connected(&self) -> bool {
match self {
SimConnection::Connected(_) => true,
SimConnection::Disconnected => false,
}
}
fn disconnect(&mut self) {
*self = SimConnection::Disconnected;
}
}
#[derive(Debug, Clone)]
pub(crate) struct SimulatorOpts {
pub(crate) ticks: usize,

View File

@@ -157,6 +157,7 @@ fn execute_plan(
/// `execute_interaction` uses this type in conjunction with a result, where
/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case
/// indicates the next step in the plan.
#[derive(PartialEq)]
pub(crate) enum ExecutionContinuation {
/// Default continuation, execute the next interaction.
NextInteraction,

View File

@@ -1,4 +1,5 @@
pub mod cli;
pub mod differential;
pub mod env;
pub mod execution;
#[allow(dead_code)]