From 8f50154db26ffa641752a20888bbcb27a08224c1 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 11:16:44 -0300 Subject: [PATCH 01/16] separate struct defining code from struct generation code. Also move `Remaining` to a metrics file --- simulator/generation/plan.rs | 1002 +----------------------------- simulator/generation/property.rs | 340 +--------- simulator/generation/query.rs | 4 +- simulator/main.rs | 5 +- simulator/model/interactions.rs | 981 +++++++++++++++++++++++++++++ simulator/model/metrics.rs | 99 +++ simulator/model/mod.rs | 6 + simulator/model/property.rs | 239 +++++++ simulator/runner/differential.rs | 2 +- simulator/runner/doublecheck.rs | 2 +- simulator/runner/execution.rs | 10 +- simulator/shrink/plan.rs | 6 +- 12 files changed, 1362 insertions(+), 1334 deletions(-) create mode 100644 simulator/model/interactions.rs create mode 100644 simulator/model/metrics.rs create mode 100644 simulator/model/property.rs diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 484db67c1..ff7f570d6 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -1,269 +1,32 @@ -use std::{ - fmt::{Debug, Display}, - ops::{Deref, DerefMut}, - path::Path, - rc::Rc, - sync::Arc, - vec, -}; - -use indexmap::IndexSet; -use serde::{Deserialize, Serialize}; +use std::vec; use sql_generation::{ generation::{Arbitrary, ArbitraryFrom, GenerationContext, frequency}, - model::{ - query::{ - Create, - transaction::{Begin, Commit}, - }, - table::SimValue, + model::query::{ + Create, + transaction::{Begin, Commit}, }, }; -use tracing::error; -use turso_core::{Connection, Result, StepResult}; use crate::{ SimulatorEnv, generation::{ - Shadow, WeightedDistribution, + WeightedDistribution, property::PropertyDistribution, query::{QueryDistribution, possible_queries}, }, - model::Query, - runner::env::{ShadowTablesMut, SimConnection, SimulationType}, + model::{ + Query, + interactions::{ + Fault, Interaction, InteractionPlan, InteractionPlanIterator, InteractionStats, + InteractionType, Interactions, InteractionsType, + }, + metrics::Remaining, + property::Property, + }, }; -use super::property::{Property, remaining}; - -pub(crate) type ResultSet = Result>>; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub(crate) struct InteractionPlan { - plan: Vec, - pub mvcc: bool, - // Len should not count transactions statements, just so we can generate more meaningful interactions per run - len: usize, -} - impl InteractionPlan { - pub(crate) fn new(mvcc: bool) -> Self { - Self { - plan: Vec::new(), - mvcc, - len: 0, - } - } - - pub fn new_with(plan: Vec, mvcc: bool) -> Self { - let len = plan - .iter() - .filter(|interaction| !interaction.ignore()) - .count(); - Self { plan, mvcc, len } - } - - #[inline] - fn new_len(&self) -> usize { - self.plan - .iter() - .filter(|interaction| !interaction.ignore()) - .count() - } - - /// Length of interactions that are not transaction statements - #[inline] - pub fn len(&self) -> usize { - self.len - } - - pub fn push(&mut self, interactions: Interactions) { - if !interactions.ignore() { - self.len += 1; - } - self.plan.push(interactions); - } - - pub fn remove(&mut self, index: usize) -> Interactions { - let interactions = self.plan.remove(index); - if !interactions.ignore() { - self.len -= 1; - } - interactions - } - - pub fn truncate(&mut self, len: usize) { - self.plan.truncate(len); - self.len = self.new_len(); - } - - pub fn retain_mut(&mut self, mut f: F) - where - F: FnMut(&mut Interactions) -> bool, - { - let f = |t: &mut Interactions| { - let ignore = t.ignore(); - let retain = f(t); - // removed an interaction that was not previously ignored - if !retain && !ignore { - self.len -= 1; - } - retain - }; - self.plan.retain_mut(f); - } - - #[expect(dead_code)] - pub fn retain(&mut self, mut f: F) - where - F: FnMut(&Interactions) -> bool, - { - let f = |t: &Interactions| { - let ignore = t.ignore(); - let retain = f(t); - // removed an interaction that was not previously ignored - if !retain && !ignore { - self.len -= 1; - } - retain - }; - self.plan.retain(f); - self.len = self.new_len(); - } - - /// Compute via diff computes a a plan from a given `.plan` file without the need to parse - /// sql. This is possible because there are two versions of the plan file, one that is human - /// readable and one that is serialized as JSON. Under watch mode, the users will be able to - /// delete interactions from the human readable file, and this function uses the JSON file as - /// a baseline to detect with interactions were deleted and constructs the plan from the - /// remaining interactions. - pub(crate) fn compute_via_diff(plan_path: &Path) -> impl InteractionPlanIterator { - let interactions = std::fs::read_to_string(plan_path).unwrap(); - let interactions = interactions.lines().collect::>(); - - let plan: InteractionPlan = serde_json::from_str( - std::fs::read_to_string(plan_path.with_extension("json")) - .unwrap() - .as_str(), - ) - .unwrap(); - - let mut plan = plan - .plan - .into_iter() - .map(|i| i.interactions()) - .collect::>(); - - let (mut i, mut j) = (0, 0); - - while i < interactions.len() && j < plan.len() { - if interactions[i].starts_with("-- begin") - || interactions[i].starts_with("-- end") - || interactions[i].is_empty() - { - i += 1; - continue; - } - - // interactions[i] is the i'th line in the human readable plan - // plan[j][k] is the k'th interaction in the j'th property - let mut k = 0; - - while k < plan[j].len() { - if i >= interactions.len() { - let _ = plan.split_off(j + 1); - let _ = plan[j].split_off(k); - break; - } - error!("Comparing '{}' with '{}'", interactions[i], plan[j][k]); - if interactions[i].contains(plan[j][k].to_string().as_str()) { - i += 1; - k += 1; - } else { - plan[j].remove(k); - panic!("Comparing '{}' with '{}'", interactions[i], plan[j][k]); - } - } - - if plan[j].is_empty() { - plan.remove(j); - } else { - j += 1; - } - } - let _ = plan.split_off(j); - PlanIterator { - iter: plan.into_iter().flatten(), - } - } - - pub fn interactions_list(&self) -> Vec { - self.plan - .clone() - .into_iter() - .flat_map(|interactions| interactions.interactions().into_iter()) - .collect() - } - - pub fn interactions_list_with_secondary_index(&self) -> Vec<(usize, Interaction)> { - self.plan - .clone() - .into_iter() - .enumerate() - .flat_map(|(idx, interactions)| { - interactions - .interactions() - .into_iter() - .map(move |interaction| (idx, interaction)) - }) - .collect() - } - - pub(crate) fn stats(&self) -> InteractionStats { - let mut stats = InteractionStats::default(); - - fn query_stat(q: &Query, stats: &mut InteractionStats) { - match q { - Query::Select(_) => stats.select_count += 1, - Query::Insert(_) => stats.insert_count += 1, - Query::Delete(_) => stats.delete_count += 1, - Query::Create(_) => stats.create_count += 1, - Query::Drop(_) => stats.drop_count += 1, - Query::Update(_) => stats.update_count += 1, - Query::CreateIndex(_) => stats.create_index_count += 1, - Query::Begin(_) => stats.begin_count += 1, - Query::Commit(_) => stats.commit_count += 1, - Query::Rollback(_) => stats.rollback_count += 1, - Query::AlterTable(_) => stats.alter_table_count += 1, - Query::DropIndex(_) => stats.drop_index_count += 1, - Query::Placeholder => {} - Query::Pragma(_) => stats.pragma_count += 1, - } - } - for interactions in &self.plan { - match &interactions.interactions { - InteractionsType::Property(property) => { - if matches!(property, Property::AllTableHaveExpectedContent { .. }) { - // Skip Property::AllTableHaveExpectedContent when counting stats - // this allows us to generate more relevant interactions as we count less Select's to the Stats - continue; - } - for interaction in &property.interactions(interactions.connection_index) { - if let InteractionType::Query(query) = &interaction.interaction { - query_stat(query, &mut stats); - } - } - } - InteractionsType::Query(query) => { - query_stat(query, &mut stats); - } - InteractionsType::Fault(_) => {} - } - } - - stats - } - pub fn init_plan(env: &mut SimulatorEnv) -> Self { let mut plan = InteractionPlan::new(env.profile.experimental_mvcc); @@ -287,7 +50,7 @@ impl InteractionPlan { ) -> Option> { let num_interactions = env.opts.max_interactions as usize; // If last interaction needs to check all db tables, generate the Property to do so - if let Some(i) = self.plan.last() + if let Some(i) = self.plan().last() && i.check_tables() { let check_all_tables = Interactions::new( @@ -379,67 +142,6 @@ impl InteractionPlan { rng, } } - - pub fn static_iterator(&self) -> impl InteractionPlanIterator { - PlanIterator { - iter: self.interactions_list().into_iter(), - } - } -} - -impl Deref for InteractionPlan { - type Target = Vec; - - fn deref(&self) -> &Self::Target { - &self.plan - } -} - -impl DerefMut for InteractionPlan { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.plan - } -} - -impl IntoIterator for InteractionPlan { - type Item = Interactions; - - type IntoIter = as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.plan.into_iter() - } -} - -impl<'a> IntoIterator for &'a InteractionPlan { - type Item = &'a Interactions; - - type IntoIter = <&'a Vec as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.plan.iter() - } -} - -impl<'a> IntoIterator for &'a mut InteractionPlan { - type Item = &'a mut Interactions; - - type IntoIter = <&'a mut Vec as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.plan.iter_mut() - } -} - -pub trait InteractionPlanIterator { - fn next(&mut self, env: &mut SimulatorEnv) -> Option; -} - -impl InteractionPlanIterator for &mut T { - #[inline] - fn next(&mut self, env: &mut SimulatorEnv) -> Option { - T::next(self, env) - } } pub struct PlanGenerator<'a, R: rand::Rng> { @@ -476,7 +178,7 @@ impl<'a, R: rand::Rng> PlanGenerator<'a, R> { let conn_ctx = env.connection_context(interaction.connection_index); - let remaining_ = remaining( + let remaining_ = Remaining::new( env.opts.max_interactions, &env.profile.query, &stats, @@ -587,676 +289,6 @@ impl<'a, R: rand::Rng> InteractionPlanIterator for PlanGenerator<'a, R> { } } -pub struct PlanIterator> { - iter: I, -} - -impl InteractionPlanIterator for PlanIterator -where - I: Iterator, -{ - #[inline] - fn next(&mut self, _env: &mut SimulatorEnv) -> Option { - self.iter.next() - } -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -pub struct InteractionPlanState { - pub interaction_pointer: usize, -} - -#[derive(Debug, Default, Clone)] -pub struct ConnectionState { - pub stack: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Interactions { - pub connection_index: usize, - pub interactions: InteractionsType, -} - -impl Interactions { - pub fn new(connection_index: usize, interactions: InteractionsType) -> Self { - Self { - connection_index, - interactions, - } - } - - pub fn get_extensional_queries(&mut self) -> Option<&mut Vec> { - match &mut self.interactions { - InteractionsType::Property(property) => property.get_extensional_queries(), - InteractionsType::Query(..) | InteractionsType::Fault(..) => None, - } - } - - /// Whether the interaction needs to check the database tables - pub fn check_tables(&self) -> bool { - match &self.interactions { - InteractionsType::Property(property) => property.check_tables(), - InteractionsType::Query(..) | InteractionsType::Fault(..) => false, - } - } - - /// Interactions that are not counted/ignored in the InteractionPlan. - /// Used in InteractionPlan to not count certain interactions to its length, as they are just auxiliary. This allows more - /// meaningful interactions to be generation - fn ignore(&self) -> bool { - self.is_transaction() - || matches!( - self.interactions, - InteractionsType::Property(Property::AllTableHaveExpectedContent { .. }) - ) - } -} - -impl Deref for Interactions { - type Target = InteractionsType; - - fn deref(&self) -> &Self::Target { - &self.interactions - } -} - -impl DerefMut for Interactions { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.interactions - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum InteractionsType { - Property(Property), - Query(Query), - Fault(Fault), -} - -impl InteractionsType { - pub fn is_transaction(&self) -> bool { - match self { - InteractionsType::Query(query) => query.is_transaction(), - _ => false, - } - } -} - -impl Interactions { - pub(crate) fn interactions(&self) -> Vec { - match &self.interactions { - InteractionsType::Property(property) => property.interactions(self.connection_index), - InteractionsType::Query(query) => vec![Interaction::new( - self.connection_index, - InteractionType::Query(query.clone()), - )], - InteractionsType::Fault(fault) => vec![Interaction::new( - self.connection_index, - InteractionType::Fault(*fault), - )], - } - } - - pub(crate) fn dependencies(&self) -> IndexSet { - match &self.interactions { - InteractionsType::Property(property) => property - .interactions(self.connection_index) - .iter() - .fold(IndexSet::new(), |mut acc, i| match &i.interaction { - InteractionType::Query(q) => { - acc.extend(q.dependencies()); - acc - } - _ => acc, - }), - InteractionsType::Query(query) => query.dependencies(), - InteractionsType::Fault(_) => IndexSet::new(), - } - } - - pub(crate) fn uses(&self) -> Vec { - match &self.interactions { - InteractionsType::Property(property) => property - .interactions(self.connection_index) - .iter() - .fold(vec![], |mut acc, i| match &i.interaction { - InteractionType::Query(q) => { - acc.extend(q.uses()); - acc - } - _ => acc, - }), - InteractionsType::Query(query) => query.uses(), - InteractionsType::Fault(_) => vec![], - } - } -} - -// FIXME: for the sql display come back and add connection index as a comment -impl Display for InteractionPlan { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for interactions in &self.plan { - match &interactions.interactions { - InteractionsType::Property(property) => { - let name = property.name(); - writeln!(f, "-- begin testing '{name}'")?; - for interaction in property.interactions(interactions.connection_index) { - writeln!(f, "\t{interaction}")?; - } - writeln!(f, "-- end testing '{name}'")?; - } - InteractionsType::Fault(fault) => { - writeln!(f, "-- FAULT '{fault}'")?; - } - InteractionsType::Query(query) => { - writeln!(f, "{query}; -- {}", interactions.connection_index)?; - } - } - } - - Ok(()) - } -} - -#[derive(Debug, Clone, Copy, Default)] -pub(crate) struct InteractionStats { - pub select_count: u32, - pub insert_count: u32, - pub delete_count: u32, - pub update_count: u32, - pub create_count: u32, - pub create_index_count: u32, - pub drop_count: u32, - pub begin_count: u32, - pub commit_count: u32, - pub rollback_count: u32, - pub alter_table_count: u32, - pub drop_index_count: u32, - pub pragma_count: u32, -} - -impl Display for InteractionStats { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Read: {}, Insert: {}, Delete: {}, Update: {}, Create: {}, CreateIndex: {}, Drop: {}, Begin: {}, Commit: {}, Rollback: {}, Alter Table: {}, Drop Index: {}", - self.select_count, - self.insert_count, - self.delete_count, - self.update_count, - self.create_count, - self.create_index_count, - self.drop_count, - self.begin_count, - self.commit_count, - self.rollback_count, - self.alter_table_count, - self.drop_index_count, - ) - } -} - -type AssertionFunc = dyn Fn(&Vec, &mut SimulatorEnv) -> Result>; - -#[derive(Clone)] -pub struct Assertion { - pub func: Rc, - pub name: String, // For display purposes in the plan -} - -impl Debug for Assertion { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Assertion") - .field("name", &self.name) - .finish() - } -} - -impl Assertion { - pub fn new(name: String, func: F) -> Self - where - F: Fn(&Vec, &mut SimulatorEnv) -> Result> + 'static, - { - Self { - func: Rc::new(func), - name, - } - } -} - -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub enum Fault { - Disconnect, - ReopenDatabase, -} - -impl Display for Fault { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Fault::Disconnect => write!(f, "DISCONNECT"), - Fault::ReopenDatabase => write!(f, "REOPEN_DATABASE"), - } - } -} - -#[derive(Debug, Clone)] -pub struct Interaction { - pub connection_index: usize, - pub interaction: InteractionType, - pub ignore_error: bool, -} - -impl Deref for Interaction { - type Target = InteractionType; - - fn deref(&self) -> &Self::Target { - &self.interaction - } -} - -impl DerefMut for Interaction { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.interaction - } -} - -impl Interaction { - pub fn new(connection_index: usize, interaction: InteractionType) -> Self { - Self { - connection_index, - interaction, - ignore_error: false, - } - } - - pub fn new_ignore_error(connection_index: usize, interaction: InteractionType) -> Self { - Self { - connection_index, - interaction, - ignore_error: true, - } - } -} - -#[derive(Debug, Clone)] -pub enum InteractionType { - Query(Query), - Assumption(Assertion), - Assertion(Assertion), - Fault(Fault), - /// Will attempt to run any random query. However, when the connection tries to sync it will - /// close all connections and reopen the database and assert that no data was lost - FsyncQuery(Query), - FaultyQuery(Query), -} - -// FIXME: add the connection index here later -impl Display for Interaction { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}; -- {}", self.interaction, self.connection_index) - } -} - -impl Display for InteractionType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Query(query) => write!(f, "{query}"), - Self::Assumption(assumption) => write!(f, "-- ASSUME {}", assumption.name), - Self::Assertion(assertion) => { - write!(f, "-- ASSERT {};", assertion.name) - } - Self::Fault(fault) => write!(f, "-- FAULT '{fault}'"), - Self::FsyncQuery(query) => { - writeln!(f, "-- FSYNC QUERY")?; - writeln!(f, "{query};")?; - write!(f, "{query};") - } - Self::FaultyQuery(query) => write!(f, "{query}; -- FAULTY QUERY"), - } - } -} - -impl Shadow for InteractionType { - type Result = anyhow::Result>>; - fn shadow(&self, env: &mut ShadowTablesMut) -> Self::Result { - match self { - Self::Query(query) => { - if !query.is_transaction() { - env.add_query(query); - } - query.shadow(env) - } - Self::Assumption(_) - | Self::Assertion(_) - | Self::Fault(_) - | Self::FaultyQuery(_) - | Self::FsyncQuery(_) => Ok(vec![]), - } - } -} - -impl InteractionType { - pub fn is_ddl(&self) -> bool { - match self { - InteractionType::Query(query) - | InteractionType::FsyncQuery(query) - | InteractionType::FaultyQuery(query) => query.is_ddl(), - _ => false, - } - } - - pub(crate) fn execute_query(&self, conn: &mut Arc) -> ResultSet { - if let Self::Query(query) = self { - assert!( - !matches!(query, Query::Placeholder), - "simulation cannot have a placeholder Query for execution" - ); - - let query_str = query.to_string(); - let rows = conn.query(&query_str); - if rows.is_err() { - let err = rows.err(); - tracing::debug!( - "Error running query '{}': {:?}", - &query_str[0..query_str.len().min(4096)], - err - ); - // Do not panic on parse error, because DoubleCreateFailure relies on it - return Err(err.unwrap()); - } - let rows = rows?; - assert!(rows.is_some()); - let mut rows = rows.unwrap(); - let mut out = Vec::new(); - while let Ok(row) = rows.step() { - match row { - StepResult::Row => { - let row = rows.row().unwrap(); - let mut r = Vec::new(); - for v in row.get_values() { - let v = v.into(); - r.push(v); - } - out.push(r); - } - StepResult::IO => { - rows.run_once().unwrap(); - } - StepResult::Interrupt => {} - StepResult::Done => { - break; - } - StepResult::Busy => { - return Err(turso_core::LimboError::Busy); - } - } - } - - Ok(out) - } else { - unreachable!("unexpected: this function should only be called on queries") - } - } - - pub(crate) fn execute_assertion( - &self, - stack: &Vec, - env: &mut SimulatorEnv, - ) -> Result<()> { - match self { - Self::Assertion(assertion) => { - let result = assertion.func.as_ref()(stack, env); - match result { - Ok(Ok(())) => Ok(()), - Ok(Err(message)) => Err(turso_core::LimboError::InternalError(format!( - "Assertion '{}' failed: {}", - assertion.name, message - ))), - Err(err) => Err(turso_core::LimboError::InternalError(format!( - "Assertion '{}' execution error: {}", - assertion.name, err - ))), - } - } - _ => { - unreachable!("unexpected: this function should only be called on assertions") - } - } - } - - pub(crate) fn execute_assumption( - &self, - stack: &Vec, - env: &mut SimulatorEnv, - ) -> Result<()> { - match self { - Self::Assumption(assumption) => { - let result = assumption.func.as_ref()(stack, env); - match result { - Ok(Ok(())) => Ok(()), - Ok(Err(message)) => Err(turso_core::LimboError::InternalError(format!( - "Assumption '{}' failed: {}", - assumption.name, message - ))), - Err(err) => Err(turso_core::LimboError::InternalError(format!( - "Assumption '{}' execution error: {}", - assumption.name, err - ))), - } - } - _ => { - unreachable!("unexpected: this function should only be called on assumptions") - } - } - } - - pub(crate) fn execute_fault(&self, env: &mut SimulatorEnv, conn_index: usize) -> Result<()> { - match self { - Self::Fault(fault) => { - match fault { - Fault::Disconnect => { - if env.connections[conn_index].is_connected() { - if env.conn_in_transaction(conn_index) { - env.rollback_conn(conn_index); - } - env.connections[conn_index].disconnect(); - } else { - return Err(turso_core::LimboError::InternalError( - "connection already disconnected".into(), - )); - } - } - Fault::ReopenDatabase => { - reopen_database(env); - } - } - Ok(()) - } - _ => { - unreachable!("unexpected: this function should only be called on faults") - } - } - } - - pub(crate) fn execute_fsync_query( - &self, - conn: Arc, - env: &mut SimulatorEnv, - ) -> ResultSet { - if let Self::FsyncQuery(query) = self { - let query_str = query.to_string(); - let rows = conn.query(&query_str); - if rows.is_err() { - let err = rows.err(); - tracing::debug!( - "Error running query '{}': {:?}", - &query_str[0..query_str.len().min(4096)], - err - ); - return Err(err.unwrap()); - } - let mut rows = rows.unwrap().unwrap(); - let mut out = Vec::new(); - while let Ok(row) = rows.step() { - match row { - StepResult::Row => { - let row = rows.row().unwrap(); - let mut r = Vec::new(); - for v in row.get_values() { - let v = v.into(); - r.push(v); - } - out.push(r); - } - StepResult::IO => { - let syncing = env.io.syncing(); - if syncing { - reopen_database(env); - } else { - rows.run_once().unwrap(); - } - } - StepResult::Done => { - break; - } - StepResult::Busy => { - return Err(turso_core::LimboError::Busy); - } - StepResult::Interrupt => {} - } - } - - Ok(out) - } else { - unreachable!("unexpected: this function should only be called on queries") - } - } - - pub(crate) fn execute_faulty_query( - &self, - conn: &Arc, - env: &mut SimulatorEnv, - ) -> ResultSet { - use rand::Rng; - if let Self::FaultyQuery(query) = self { - let query_str = query.to_string(); - let rows = conn.query(&query_str); - if rows.is_err() { - let err = rows.err(); - tracing::debug!( - "Error running query '{}': {:?}", - &query_str[0..query_str.len().min(4096)], - err - ); - if let Some(turso_core::LimboError::ParseError(e)) = err { - panic!("Unexpected parse error: {e}"); - } - return Err(err.unwrap()); - } - let mut rows = rows.unwrap().unwrap(); - let mut out = Vec::new(); - let mut current_prob = 0.05; - let mut incr = 0.001; - loop { - let syncing = env.io.syncing(); - let inject_fault = env.rng.random_bool(current_prob); - // TODO: avoid for now injecting faults when syncing - if inject_fault && !syncing { - env.io.inject_fault(true); - } - - match rows.step()? { - StepResult::Row => { - let row = rows.row().unwrap(); - let mut r = Vec::new(); - for v in row.get_values() { - let v = v.into(); - r.push(v); - } - out.push(r); - } - StepResult::IO => { - rows.run_once()?; - current_prob += incr; - if current_prob > 1.0 { - current_prob = 1.0; - } else { - incr *= 1.01; - } - } - StepResult::Done => { - break; - } - StepResult::Busy => { - return Err(turso_core::LimboError::Busy); - } - StepResult::Interrupt => {} - } - } - - Ok(out) - } else { - unreachable!("unexpected: this function should only be called on queries") - } - } -} - -fn reopen_database(env: &mut SimulatorEnv) { - // 1. Close all connections without default checkpoint-on-close behavior - // to expose bugs related to how we handle WAL - let mvcc = env.profile.experimental_mvcc; - let indexes = env.profile.query.gen_opts.indexes; - let num_conns = env.connections.len(); - env.connections.clear(); - - // Clear all open files - // TODO: for correct reporting of faults we should get all the recorded numbers and transfer to the new file - env.io.close_files(); - - // 2. Re-open database - match env.type_ { - SimulationType::Differential => { - for _ in 0..num_conns { - env.connections.push(SimConnection::SQLiteConnection( - rusqlite::Connection::open(env.get_db_path()) - .expect("Failed to open SQLite connection"), - )); - } - } - SimulationType::Default | SimulationType::Doublecheck => { - env.db = None; - let db = match turso_core::Database::open_file_with_flags( - env.io.clone(), - env.get_db_path().to_str().expect("path should be 'to_str'"), - turso_core::OpenFlags::default(), - turso_core::DatabaseOpts::new() - .with_mvcc(mvcc) - .with_indexes(indexes) - .with_autovacuum(true), - None, - ) { - Ok(db) => db, - Err(e) => { - tracing::error!( - "Failed to open database at {}: {}", - env.get_db_path().display(), - e - ); - panic!("Failed to open database: {e}"); - } - }; - - env.db = Some(db); - - for _ in 0..num_conns { - env.connections.push(SimConnection::LimboConnection( - env.db.as_ref().expect("db to be Some").connect().unwrap(), - )); - } - } - }; -} - fn random_fault( rng: &mut R, env: &SimulatorEnv, @@ -1277,7 +309,7 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats, usize)> for Interactions { conn_ctx: &C, (env, stats, conn_index): (&SimulatorEnv, InteractionStats, usize), ) -> Self { - let remaining_ = remaining( + let remaining_ = Remaining::new( env.opts.max_interactions, &env.profile.query, &stats, diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 8f10b96a2..75343b13b 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -6,7 +6,6 @@ //! an optimization issue that is good to point out for the future use rand::distr::{Distribution, weighted::WeightedIndex}; -use serde::{Deserialize, Serialize}; use sql_generation::{ generation::{Arbitrary, ArbitraryFrom, GenerationContext, pick, pick_index}, model::{ @@ -27,253 +26,20 @@ use turso_parser::ast::{self, Distinctness}; use crate::{ common::print_diff, - generation::{ - Shadow as _, WeightedDistribution, plan::InteractionType, query::QueryDistribution, + generation::{Shadow as _, WeightedDistribution, query::QueryDistribution}, + model::{ + Query, QueryCapabilities, QueryDiscriminants, ResultSet, + interactions::{Assertion, Interaction, InteractionType}, + metrics::Remaining, + property::{InteractiveQueryInfo, Property, PropertyDiscriminants}, }, - model::{Query, QueryCapabilities, QueryDiscriminants}, - profiles::query::QueryProfile, runner::env::SimulatorEnv, }; -use super::plan::{Assertion, Interaction, InteractionStats, ResultSet}; - -/// Properties are representations of executable specifications -/// about the database behavior. -#[derive(Debug, Clone, Serialize, Deserialize, strum::EnumDiscriminants)] -#[strum_discriminants(derive(strum::EnumIter))] -pub enum Property { - /// Insert-Select is a property in which the inserted row - /// must be in the resulting rows of a select query that has a - /// where clause that matches the inserted row. - /// The execution of the property is as follows - /// INSERT INTO VALUES (...) - /// I_0 - /// I_1 - /// ... - /// I_n - /// SELECT * FROM WHERE - /// The interactions in the middle has the following constraints; - /// - There will be no errors in the middle interactions. - /// - The inserted row will not be deleted. - /// - The inserted row will not be updated. - /// - The table `t` will not be renamed, dropped, or altered. - InsertValuesSelect { - /// The insert query - insert: Insert, - /// Selected row index - row_index: usize, - /// Additional interactions in the middle of the property - queries: Vec, - /// The select query - select: Select, - /// Interactive query information if any - interactive: Option, - }, - /// ReadYourUpdatesBack is a property in which the updated rows - /// must be in the resulting rows of a select query that has a - /// where clause that matches the updated row. - /// The execution of the property is as follows - /// UPDATE SET WHERE - /// SELECT FROM WHERE - /// These interactions are executed in immediate succession - /// just to verify the property that our updates did what they - /// were supposed to do. - ReadYourUpdatesBack { - update: Update, - select: Select, - }, - /// TableHasExpectedContent is a property in which the table - /// must have the expected content, i.e. all the insertions and - /// updates and deletions should have been persisted in the way - /// we think they were. - /// The execution of the property is as follows - /// SELECT * FROM - /// ASSERT - TableHasExpectedContent { - table: String, - }, - /// AllTablesHaveExpectedContent is a property in which the table - /// must have the expected content, i.e. all the insertions and - /// updates and deletions should have been persisted in the way - /// we think they were. - /// The execution of the property is as follows - /// SELECT * FROM - /// ASSERT - /// for each table in the simulator model - AllTableHaveExpectedContent { - tables: Vec, - }, - /// Double Create Failure is a property in which creating - /// the same table twice leads to an error. - /// The execution of the property is as follows - /// CREATE TABLE (...) - /// I_0 - /// I_1 - /// ... - /// I_n - /// CREATE TABLE (...) -> Error - /// The interactions in the middle has the following constraints; - /// - There will be no errors in the middle interactions. - /// - Table `t` will not be renamed or dropped. - DoubleCreateFailure { - /// The create query - create: Create, - /// Additional interactions in the middle of the property - queries: Vec, - }, - /// Select Limit is a property in which the select query - /// has a limit clause that is respected by the query. - /// The execution of the property is as follows - /// SELECT * FROM WHERE LIMIT - /// This property is a single-interaction property. - /// The interaction has the following constraints; - /// - The select query will respect the limit clause. - SelectLimit { - /// The select query - select: Select, - }, - /// Delete-Select is a property in which the deleted row - /// must not be in the resulting rows of a select query that has a - /// where clause that matches the deleted row. In practice, `p1` of - /// the delete query will be used as the predicate for the select query, - /// hence the select should return NO ROWS. - /// The execution of the property is as follows - /// DELETE FROM WHERE - /// I_0 - /// I_1 - /// ... - /// I_n - /// SELECT * FROM WHERE - /// The interactions in the middle has the following constraints; - /// - There will be no errors in the middle interactions. - /// - A row that holds for the predicate will not be inserted. - /// - The table `t` will not be renamed, dropped, or altered. - DeleteSelect { - table: String, - predicate: Predicate, - queries: Vec, - }, - /// Drop-Select is a property in which selecting from a dropped table - /// should result in an error. - /// The execution of the property is as follows - /// DROP TABLE - /// I_0 - /// I_1 - /// ... - /// I_n - /// SELECT * FROM WHERE -> Error - /// The interactions in the middle has the following constraints; - /// - There will be no errors in the middle interactions. - /// - The table `t` will not be created, no table will be renamed to `t`. - DropSelect { - table: String, - queries: Vec, - select: Select, - }, - /// Select-Select-Optimizer is a property in which we test the optimizer by - /// running two equivalent select queries, one with `SELECT from ` - /// and the other with `SELECT * from WHERE `. As highlighted by - /// Rigger et al. in Non-Optimizing Reference Engine Construction(NoREC), SQLite - /// tends to optimize `where` statements while keeping the result column expressions - /// unoptimized. This property is used to test the optimizer. The property is successful - /// if the two queries return the same number of rows. - SelectSelectOptimizer { - table: String, - predicate: Predicate, - }, - /// Where-True-False-Null is a property that tests the boolean logic implementation - /// in the database. It relies on the fact that `P == true || P == false || P == null` should return true, - /// as SQLite uses a ternary logic system. This property is invented in "Finding Bugs in Database Systems via Query Partitioning" - /// by Rigger et al. and it is canonically called Ternary Logic Partitioning (TLP). - WhereTrueFalseNull { - select: Select, - predicate: Predicate, - }, - /// UNION-ALL-Preserves-Cardinality is a property that tests the UNION ALL operator - /// implementation in the database. It relies on the fact that `SELECT * FROM WHERE UNION ALL SELECT * FROM WHERE ` - /// should return the same number of rows as `SELECT FROM WHERE `. - /// > The property is succesfull when the UNION ALL of 2 select queries returns the same number of rows - /// > as the sum of the two select queries. - UNIONAllPreservesCardinality { - select: Select, - where_clause: Predicate, - }, - /// FsyncNoWait is a property which tests if we do not loose any data after not waiting for fsync. - /// - /// # Interactions - /// - Executes the `query` without waiting for fsync - /// - Drop all connections and Reopen the database - /// - Execute the `query` again - /// - Query tables to assert that the values were inserted - /// - FsyncNoWait { - query: Query, - }, - FaultyQuery { - query: Query, - }, - /// Property used to subsititute a property with its queries only - Queries { - queries: Vec, - }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InteractiveQueryInfo { - start_with_immediate: bool, - end_with_commit: bool, -} - type PropertyQueryGenFunc<'a, R, G> = fn(&mut R, &G, &QueryDistribution, &Property) -> Option; impl Property { - pub(crate) fn name(&self) -> &str { - match self { - Property::InsertValuesSelect { .. } => "Insert-Values-Select", - Property::ReadYourUpdatesBack { .. } => "Read-Your-Updates-Back", - Property::TableHasExpectedContent { .. } => "Table-Has-Expected-Content", - Property::AllTableHaveExpectedContent { .. } => "All-Tables-Have-Expected-Content", - Property::DoubleCreateFailure { .. } => "Double-Create-Failure", - Property::SelectLimit { .. } => "Select-Limit", - Property::DeleteSelect { .. } => "Delete-Select", - Property::DropSelect { .. } => "Drop-Select", - Property::SelectSelectOptimizer { .. } => "Select-Select-Optimizer", - Property::WhereTrueFalseNull { .. } => "Where-True-False-Null", - Property::FsyncNoWait { .. } => "FsyncNoWait", - Property::FaultyQuery { .. } => "FaultyQuery", - Property::UNIONAllPreservesCardinality { .. } => "UNION-All-Preserves-Cardinality", - Property::Queries { .. } => "Queries", - } - } - - /// Property Does some sort of fault injection - pub fn check_tables(&self) -> bool { - matches!( - self, - Property::FsyncNoWait { .. } | Property::FaultyQuery { .. } - ) - } - - pub fn get_extensional_queries(&mut self) -> Option<&mut Vec> { - match self { - Property::InsertValuesSelect { queries, .. } - | Property::DoubleCreateFailure { queries, .. } - | Property::DeleteSelect { queries, .. } - | Property::DropSelect { queries, .. } - | Property::Queries { queries } => Some(queries), - Property::FsyncNoWait { .. } | Property::FaultyQuery { .. } => None, - Property::SelectLimit { .. } - | Property::SelectSelectOptimizer { .. } - | Property::WhereTrueFalseNull { .. } - | Property::UNIONAllPreservesCardinality { .. } - | Property::ReadYourUpdatesBack { .. } - | Property::TableHasExpectedContent { .. } - | Property::AllTableHaveExpectedContent { .. } => None, - } - } - pub(super) fn get_extensional_query_gen_function(&self) -> PropertyQueryGenFunc where R: rand::Rng + ?Sized, @@ -1369,100 +1135,6 @@ fn assert_all_table_values( }) } -#[derive(Debug)] -pub(super) struct Remaining { - pub select: u32, - pub insert: u32, - pub create: u32, - pub create_index: u32, - pub delete: u32, - pub update: u32, - pub drop: u32, - pub alter_table: u32, - pub drop_index: u32, - pub pragma_count: u32, -} - -pub(super) fn remaining( - max_interactions: u32, - opts: &QueryProfile, - stats: &InteractionStats, - mvcc: bool, - context: &impl GenerationContext, -) -> Remaining { - let total_weight = opts.total_weight(); - - let total_select = (max_interactions * opts.select_weight) / total_weight; - let total_insert = (max_interactions * opts.insert_weight) / total_weight; - let total_create = (max_interactions * opts.create_table_weight) / total_weight; - let total_create_index = (max_interactions * opts.create_index_weight) / total_weight; - let total_delete = (max_interactions * opts.delete_weight) / total_weight; - let total_update = (max_interactions * opts.update_weight) / total_weight; - let total_drop = (max_interactions * opts.drop_table_weight) / total_weight; - let total_alter_table = (max_interactions * opts.alter_table_weight) / total_weight; - let total_drop_index = (max_interactions * opts.drop_index) / total_weight; - let total_pragma = (max_interactions * opts.pragma_weight) / total_weight; - - let remaining_select = total_select - .checked_sub(stats.select_count) - .unwrap_or_default(); - let remaining_insert = total_insert - .checked_sub(stats.insert_count) - .unwrap_or_default(); - let remaining_create = total_create - .checked_sub(stats.create_count) - .unwrap_or_default(); - let mut remaining_create_index = total_create_index - .checked_sub(stats.create_index_count) - .unwrap_or_default(); - let remaining_delete = total_delete - .checked_sub(stats.delete_count) - .unwrap_or_default(); - let remaining_update = total_update - .checked_sub(stats.update_count) - .unwrap_or_default(); - let remaining_drop = total_drop.checked_sub(stats.drop_count).unwrap_or_default(); - let remaining_pragma = total_pragma - .checked_sub(stats.pragma_count) - .unwrap_or_default(); - - let remaining_alter_table = total_alter_table - .checked_sub(stats.alter_table_count) - .unwrap_or_default(); - - let mut remaining_drop_index = total_drop_index - .checked_sub(stats.alter_table_count) - .unwrap_or_default(); - - if mvcc { - // TODO: index not supported yet for mvcc - remaining_create_index = 0; - remaining_drop_index = 0; - } - - // if there are no indexes do not allow creation of drop_index - if !context - .tables() - .iter() - .any(|table| !table.indexes.is_empty()) - { - remaining_drop_index = 0; - } - - Remaining { - select: remaining_select, - insert: remaining_insert, - create: remaining_create, - create_index: remaining_create_index, - delete: remaining_delete, - drop: remaining_drop, - update: remaining_update, - alter_table: remaining_alter_table, - drop_index: remaining_drop_index, - pragma_count: remaining_pragma, - } -} - fn property_insert_values_select( rng: &mut R, _query_distr: &QueryDistribution, diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index 02d82c17f..5a94cabe0 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -1,6 +1,6 @@ use crate::{ generation::WeightedDistribution, - model::{Query, QueryDiscriminants}, + model::{Query, QueryDiscriminants, metrics::Remaining}, }; use rand::{ Rng, @@ -20,8 +20,6 @@ use sql_generation::{ }, }; -use super::property::Remaining; - fn random_create(rng: &mut R, conn_ctx: &impl GenerationContext) -> Query { let mut create = Create::arbitrary(rng, conn_ctx); while conn_ctx diff --git a/simulator/main.rs b/simulator/main.rs index a23157dda..9523dbdce 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -1,7 +1,6 @@ #![allow(clippy::arc_with_non_send_sync)] use anyhow::anyhow; use clap::Parser; -use generation::plan::{InteractionPlan, InteractionPlanState}; use notify::event::{DataChange, ModifyKind}; use notify::{EventKind, RecursiveMode, Watcher}; use rand::prelude::*; @@ -22,7 +21,9 @@ use tracing_subscriber::field::MakeExt; use tracing_subscriber::fmt::format; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; -use crate::generation::plan::{ConnectionState, InteractionPlanIterator}; +use crate::model::interactions::{ + ConnectionState, InteractionPlan, InteractionPlanIterator, InteractionPlanState, +}; use crate::profiles::Profile; use crate::runner::doublecheck; use crate::runner::env::{Paths, SimulationPhase, SimulationType}; diff --git a/simulator/model/interactions.rs b/simulator/model/interactions.rs new file mode 100644 index 000000000..204167e61 --- /dev/null +++ b/simulator/model/interactions.rs @@ -0,0 +1,981 @@ +use std::{ + fmt::{Debug, Display}, + ops::{Deref, DerefMut}, + path::Path, + rc::Rc, + sync::Arc, +}; + +use indexmap::IndexSet; +use serde::{Deserialize, Serialize}; +use sql_generation::model::table::SimValue; +use turso_core::{Connection, Result, StepResult}; + +use crate::{ + generation::Shadow, + model::{Query, ResultSet, property::Property}, + runner::env::{ShadowTablesMut, SimConnection, SimulationType, SimulatorEnv}, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub(crate) struct InteractionPlan { + plan: Vec, + pub mvcc: bool, + // Len should not count transactions statements, just so we can generate more meaningful interactions per run + len: usize, +} + +impl InteractionPlan { + pub(crate) fn new(mvcc: bool) -> Self { + Self { + plan: Vec::new(), + mvcc, + len: 0, + } + } + + pub fn new_with(plan: Vec, mvcc: bool) -> Self { + let len = plan + .iter() + .filter(|interaction| !interaction.ignore()) + .count(); + Self { plan, mvcc, len } + } + + #[inline] + fn new_len(&self) -> usize { + self.plan + .iter() + .filter(|interaction| !interaction.ignore()) + .count() + } + + /// Length of interactions that are not transaction statements + #[inline] + pub fn len(&self) -> usize { + self.len + } + + #[inline] + pub fn plan(&self) -> &[Interactions] { + &self.plan + } + + pub fn push(&mut self, interactions: Interactions) { + if !interactions.ignore() { + self.len += 1; + } + self.plan.push(interactions); + } + + pub fn remove(&mut self, index: usize) -> Interactions { + let interactions = self.plan.remove(index); + if !interactions.ignore() { + self.len -= 1; + } + interactions + } + + pub fn truncate(&mut self, len: usize) { + self.plan.truncate(len); + self.len = self.new_len(); + } + + pub fn retain_mut(&mut self, mut f: F) + where + F: FnMut(&mut Interactions) -> bool, + { + let f = |t: &mut Interactions| { + let ignore = t.ignore(); + let retain = f(t); + // removed an interaction that was not previously ignored + if !retain && !ignore { + self.len -= 1; + } + retain + }; + self.plan.retain_mut(f); + } + + #[expect(dead_code)] + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&Interactions) -> bool, + { + let f = |t: &Interactions| { + let ignore = t.ignore(); + let retain = f(t); + // removed an interaction that was not previously ignored + if !retain && !ignore { + self.len -= 1; + } + retain + }; + self.plan.retain(f); + self.len = self.new_len(); + } + + /// Compute via diff computes a a plan from a given `.plan` file without the need to parse + /// sql. This is possible because there are two versions of the plan file, one that is human + /// readable and one that is serialized as JSON. Under watch mode, the users will be able to + /// delete interactions from the human readable file, and this function uses the JSON file as + /// a baseline to detect with interactions were deleted and constructs the plan from the + /// remaining interactions. + pub(crate) fn compute_via_diff(plan_path: &Path) -> impl InteractionPlanIterator { + let interactions = std::fs::read_to_string(plan_path).unwrap(); + let interactions = interactions.lines().collect::>(); + + let plan: InteractionPlan = serde_json::from_str( + std::fs::read_to_string(plan_path.with_extension("json")) + .unwrap() + .as_str(), + ) + .unwrap(); + + let mut plan = plan + .plan + .into_iter() + .map(|i| i.interactions()) + .collect::>(); + + let (mut i, mut j) = (0, 0); + + while i < interactions.len() && j < plan.len() { + if interactions[i].starts_with("-- begin") + || interactions[i].starts_with("-- end") + || interactions[i].is_empty() + { + i += 1; + continue; + } + + // interactions[i] is the i'th line in the human readable plan + // plan[j][k] is the k'th interaction in the j'th property + let mut k = 0; + + while k < plan[j].len() { + if i >= interactions.len() { + let _ = plan.split_off(j + 1); + let _ = plan[j].split_off(k); + break; + } + tracing::error!("Comparing '{}' with '{}'", interactions[i], plan[j][k]); + if interactions[i].contains(plan[j][k].to_string().as_str()) { + i += 1; + k += 1; + } else { + plan[j].remove(k); + panic!("Comparing '{}' with '{}'", interactions[i], plan[j][k]); + } + } + + if plan[j].is_empty() { + plan.remove(j); + } else { + j += 1; + } + } + let _ = plan.split_off(j); + PlanIterator { + iter: plan.into_iter().flatten(), + } + } + + pub fn interactions_list(&self) -> Vec { + self.plan + .clone() + .into_iter() + .flat_map(|interactions| interactions.interactions().into_iter()) + .collect() + } + + pub fn interactions_list_with_secondary_index(&self) -> Vec<(usize, Interaction)> { + self.plan + .clone() + .into_iter() + .enumerate() + .flat_map(|(idx, interactions)| { + interactions + .interactions() + .into_iter() + .map(move |interaction| (idx, interaction)) + }) + .collect() + } + + pub(crate) fn stats(&self) -> InteractionStats { + let mut stats = InteractionStats::default(); + + fn query_stat(q: &Query, stats: &mut InteractionStats) { + match q { + Query::Select(_) => stats.select_count += 1, + Query::Insert(_) => stats.insert_count += 1, + Query::Delete(_) => stats.delete_count += 1, + Query::Create(_) => stats.create_count += 1, + Query::Drop(_) => stats.drop_count += 1, + Query::Update(_) => stats.update_count += 1, + Query::CreateIndex(_) => stats.create_index_count += 1, + Query::Begin(_) => stats.begin_count += 1, + Query::Commit(_) => stats.commit_count += 1, + Query::Rollback(_) => stats.rollback_count += 1, + Query::AlterTable(_) => stats.alter_table_count += 1, + Query::DropIndex(_) => stats.drop_index_count += 1, + Query::Placeholder => {} + Query::Pragma(_) => stats.pragma_count += 1, + } + } + for interactions in &self.plan { + match &interactions.interactions { + InteractionsType::Property(property) => { + if matches!(property, Property::AllTableHaveExpectedContent { .. }) { + // Skip Property::AllTableHaveExpectedContent when counting stats + // this allows us to generate more relevant interactions as we count less Select's to the Stats + continue; + } + for interaction in &property.interactions(interactions.connection_index) { + if let InteractionType::Query(query) = &interaction.interaction { + query_stat(query, &mut stats); + } + } + } + InteractionsType::Query(query) => { + query_stat(query, &mut stats); + } + InteractionsType::Fault(_) => {} + } + } + + stats + } + + pub fn static_iterator(&self) -> impl InteractionPlanIterator { + PlanIterator { + iter: self.interactions_list().into_iter(), + } + } +} + +impl Deref for InteractionPlan { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.plan + } +} + +impl DerefMut for InteractionPlan { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.plan + } +} + +impl IntoIterator for InteractionPlan { + type Item = Interactions; + + type IntoIter = as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.plan.into_iter() + } +} + +impl<'a> IntoIterator for &'a InteractionPlan { + type Item = &'a Interactions; + + type IntoIter = <&'a Vec as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.plan.iter() + } +} + +impl<'a> IntoIterator for &'a mut InteractionPlan { + type Item = &'a mut Interactions; + + type IntoIter = <&'a mut Vec as IntoIterator>::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.plan.iter_mut() + } +} + +pub trait InteractionPlanIterator { + fn next(&mut self, env: &mut SimulatorEnv) -> Option; +} + +impl InteractionPlanIterator for &mut T { + #[inline] + fn next(&mut self, env: &mut SimulatorEnv) -> Option { + T::next(self, env) + } +} + +pub struct PlanIterator> { + iter: I, +} + +impl InteractionPlanIterator for PlanIterator +where + I: Iterator, +{ + #[inline] + fn next(&mut self, _env: &mut SimulatorEnv) -> Option { + self.iter.next() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct InteractionPlanState { + pub interaction_pointer: usize, +} + +#[derive(Debug, Default, Clone)] +pub struct ConnectionState { + pub stack: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Interactions { + pub connection_index: usize, + pub interactions: InteractionsType, +} + +impl Interactions { + pub fn new(connection_index: usize, interactions: InteractionsType) -> Self { + Self { + connection_index, + interactions, + } + } + + pub fn get_extensional_queries(&mut self) -> Option<&mut Vec> { + match &mut self.interactions { + InteractionsType::Property(property) => property.get_extensional_queries(), + InteractionsType::Query(..) | InteractionsType::Fault(..) => None, + } + } + + /// Whether the interaction needs to check the database tables + pub fn check_tables(&self) -> bool { + match &self.interactions { + InteractionsType::Property(property) => property.check_tables(), + InteractionsType::Query(..) | InteractionsType::Fault(..) => false, + } + } + + /// Interactions that are not counted/ignored in the InteractionPlan. + /// Used in InteractionPlan to not count certain interactions to its length, as they are just auxiliary. This allows more + /// meaningful interactions to be generation + fn ignore(&self) -> bool { + self.is_transaction() + || matches!( + self.interactions, + InteractionsType::Property(Property::AllTableHaveExpectedContent { .. }) + ) + } +} + +impl Deref for Interactions { + type Target = InteractionsType; + + fn deref(&self) -> &Self::Target { + &self.interactions + } +} + +impl DerefMut for Interactions { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.interactions + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum InteractionsType { + Property(Property), + Query(Query), + Fault(Fault), +} + +impl InteractionsType { + pub fn is_transaction(&self) -> bool { + match self { + InteractionsType::Query(query) => query.is_transaction(), + _ => false, + } + } +} + +impl Interactions { + pub(crate) fn interactions(&self) -> Vec { + match &self.interactions { + InteractionsType::Property(property) => property.interactions(self.connection_index), + InteractionsType::Query(query) => vec![Interaction::new( + self.connection_index, + InteractionType::Query(query.clone()), + )], + InteractionsType::Fault(fault) => vec![Interaction::new( + self.connection_index, + InteractionType::Fault(*fault), + )], + } + } + + pub(crate) fn dependencies(&self) -> IndexSet { + match &self.interactions { + InteractionsType::Property(property) => property + .interactions(self.connection_index) + .iter() + .fold(IndexSet::new(), |mut acc, i| match &i.interaction { + InteractionType::Query(q) => { + acc.extend(q.dependencies()); + acc + } + _ => acc, + }), + InteractionsType::Query(query) => query.dependencies(), + InteractionsType::Fault(_) => IndexSet::new(), + } + } + + pub(crate) fn uses(&self) -> Vec { + match &self.interactions { + InteractionsType::Property(property) => property + .interactions(self.connection_index) + .iter() + .fold(vec![], |mut acc, i| match &i.interaction { + InteractionType::Query(q) => { + acc.extend(q.uses()); + acc + } + _ => acc, + }), + InteractionsType::Query(query) => query.uses(), + InteractionsType::Fault(_) => vec![], + } + } +} + +// FIXME: for the sql display come back and add connection index as a comment +impl Display for InteractionPlan { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for interactions in &self.plan { + match &interactions.interactions { + InteractionsType::Property(property) => { + let name = property.name(); + writeln!(f, "-- begin testing '{name}'")?; + for interaction in property.interactions(interactions.connection_index) { + writeln!(f, "\t{interaction}")?; + } + writeln!(f, "-- end testing '{name}'")?; + } + InteractionsType::Fault(fault) => { + writeln!(f, "-- FAULT '{fault}'")?; + } + InteractionsType::Query(query) => { + writeln!(f, "{query}; -- {}", interactions.connection_index)?; + } + } + } + + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct InteractionStats { + pub select_count: u32, + pub insert_count: u32, + pub delete_count: u32, + pub update_count: u32, + pub create_count: u32, + pub create_index_count: u32, + pub drop_count: u32, + pub begin_count: u32, + pub commit_count: u32, + pub rollback_count: u32, + pub alter_table_count: u32, + pub drop_index_count: u32, + pub pragma_count: u32, +} + +impl Display for InteractionStats { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Read: {}, Insert: {}, Delete: {}, Update: {}, Create: {}, CreateIndex: {}, Drop: {}, Begin: {}, Commit: {}, Rollback: {}, Alter Table: {}, Drop Index: {}", + self.select_count, + self.insert_count, + self.delete_count, + self.update_count, + self.create_count, + self.create_index_count, + self.drop_count, + self.begin_count, + self.commit_count, + self.rollback_count, + self.alter_table_count, + self.drop_index_count, + ) + } +} + +type AssertionFunc = dyn Fn(&Vec, &mut SimulatorEnv) -> Result>; + +#[derive(Clone)] +pub struct Assertion { + pub func: Rc, + pub name: String, // For display purposes in the plan +} + +impl Debug for Assertion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Assertion") + .field("name", &self.name) + .finish() + } +} + +impl Assertion { + pub fn new(name: String, func: F) -> Self + where + F: Fn(&Vec, &mut SimulatorEnv) -> Result> + 'static, + { + Self { + func: Rc::new(func), + name, + } + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum Fault { + Disconnect, + ReopenDatabase, +} + +impl Display for Fault { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Fault::Disconnect => write!(f, "DISCONNECT"), + Fault::ReopenDatabase => write!(f, "REOPEN_DATABASE"), + } + } +} + +#[derive(Debug, Clone)] +pub struct Interaction { + pub connection_index: usize, + pub interaction: InteractionType, + pub ignore_error: bool, +} + +impl Deref for Interaction { + type Target = InteractionType; + + fn deref(&self) -> &Self::Target { + &self.interaction + } +} + +impl DerefMut for Interaction { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.interaction + } +} + +impl Interaction { + pub fn new(connection_index: usize, interaction: InteractionType) -> Self { + Self { + connection_index, + interaction, + ignore_error: false, + } + } + + pub fn new_ignore_error(connection_index: usize, interaction: InteractionType) -> Self { + Self { + connection_index, + interaction, + ignore_error: true, + } + } +} + +#[derive(Debug, Clone)] +pub enum InteractionType { + Query(Query), + Assumption(Assertion), + Assertion(Assertion), + Fault(Fault), + /// Will attempt to run any random query. However, when the connection tries to sync it will + /// close all connections and reopen the database and assert that no data was lost + FsyncQuery(Query), + FaultyQuery(Query), +} + +// FIXME: add the connection index here later +impl Display for Interaction { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}; -- {}", self.interaction, self.connection_index) + } +} + +impl Display for InteractionType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Query(query) => write!(f, "{query}"), + Self::Assumption(assumption) => write!(f, "-- ASSUME {}", assumption.name), + Self::Assertion(assertion) => { + write!(f, "-- ASSERT {};", assertion.name) + } + Self::Fault(fault) => write!(f, "-- FAULT '{fault}'"), + Self::FsyncQuery(query) => { + writeln!(f, "-- FSYNC QUERY")?; + writeln!(f, "{query};")?; + write!(f, "{query};") + } + Self::FaultyQuery(query) => write!(f, "{query}; -- FAULTY QUERY"), + } + } +} + +impl Shadow for InteractionType { + type Result = anyhow::Result>>; + fn shadow(&self, env: &mut ShadowTablesMut) -> Self::Result { + match self { + Self::Query(query) => { + if !query.is_transaction() { + env.add_query(query); + } + query.shadow(env) + } + Self::Assumption(_) + | Self::Assertion(_) + | Self::Fault(_) + | Self::FaultyQuery(_) + | Self::FsyncQuery(_) => Ok(vec![]), + } + } +} + +impl InteractionType { + pub fn is_ddl(&self) -> bool { + match self { + InteractionType::Query(query) + | InteractionType::FsyncQuery(query) + | InteractionType::FaultyQuery(query) => query.is_ddl(), + _ => false, + } + } + + pub(crate) fn execute_query(&self, conn: &mut Arc) -> ResultSet { + if let Self::Query(query) = self { + assert!( + !matches!(query, Query::Placeholder), + "simulation cannot have a placeholder Query for execution" + ); + + let query_str = query.to_string(); + let rows = conn.query(&query_str); + if rows.is_err() { + let err = rows.err(); + tracing::debug!( + "Error running query '{}': {:?}", + &query_str[0..query_str.len().min(4096)], + err + ); + // Do not panic on parse error, because DoubleCreateFailure relies on it + return Err(err.unwrap()); + } + let rows = rows?; + assert!(rows.is_some()); + let mut rows = rows.unwrap(); + let mut out = Vec::new(); + while let Ok(row) = rows.step() { + match row { + StepResult::Row => { + let row = rows.row().unwrap(); + let mut r = Vec::new(); + for v in row.get_values() { + let v = v.into(); + r.push(v); + } + out.push(r); + } + StepResult::IO => { + rows.run_once().unwrap(); + } + StepResult::Interrupt => {} + StepResult::Done => { + break; + } + StepResult::Busy => { + return Err(turso_core::LimboError::Busy); + } + } + } + + Ok(out) + } else { + unreachable!("unexpected: this function should only be called on queries") + } + } + + pub(crate) fn execute_assertion( + &self, + stack: &Vec, + env: &mut SimulatorEnv, + ) -> Result<()> { + match self { + Self::Assertion(assertion) => { + let result = assertion.func.as_ref()(stack, env); + match result { + Ok(Ok(())) => Ok(()), + Ok(Err(message)) => Err(turso_core::LimboError::InternalError(format!( + "Assertion '{}' failed: {}", + assertion.name, message + ))), + Err(err) => Err(turso_core::LimboError::InternalError(format!( + "Assertion '{}' execution error: {}", + assertion.name, err + ))), + } + } + _ => { + unreachable!("unexpected: this function should only be called on assertions") + } + } + } + + pub(crate) fn execute_assumption( + &self, + stack: &Vec, + env: &mut SimulatorEnv, + ) -> Result<()> { + match self { + Self::Assumption(assumption) => { + let result = assumption.func.as_ref()(stack, env); + match result { + Ok(Ok(())) => Ok(()), + Ok(Err(message)) => Err(turso_core::LimboError::InternalError(format!( + "Assumption '{}' failed: {}", + assumption.name, message + ))), + Err(err) => Err(turso_core::LimboError::InternalError(format!( + "Assumption '{}' execution error: {}", + assumption.name, err + ))), + } + } + _ => { + unreachable!("unexpected: this function should only be called on assumptions") + } + } + } + + pub(crate) fn execute_fault(&self, env: &mut SimulatorEnv, conn_index: usize) -> Result<()> { + match self { + Self::Fault(fault) => { + match fault { + Fault::Disconnect => { + if env.connections[conn_index].is_connected() { + if env.conn_in_transaction(conn_index) { + env.rollback_conn(conn_index); + } + env.connections[conn_index].disconnect(); + } else { + return Err(turso_core::LimboError::InternalError( + "connection already disconnected".into(), + )); + } + } + Fault::ReopenDatabase => { + reopen_database(env); + } + } + Ok(()) + } + _ => { + unreachable!("unexpected: this function should only be called on faults") + } + } + } + + pub(crate) fn execute_fsync_query( + &self, + conn: Arc, + env: &mut SimulatorEnv, + ) -> ResultSet { + if let Self::FsyncQuery(query) = self { + let query_str = query.to_string(); + let rows = conn.query(&query_str); + if rows.is_err() { + let err = rows.err(); + tracing::debug!( + "Error running query '{}': {:?}", + &query_str[0..query_str.len().min(4096)], + err + ); + return Err(err.unwrap()); + } + let mut rows = rows.unwrap().unwrap(); + let mut out = Vec::new(); + while let Ok(row) = rows.step() { + match row { + StepResult::Row => { + let row = rows.row().unwrap(); + let mut r = Vec::new(); + for v in row.get_values() { + let v = v.into(); + r.push(v); + } + out.push(r); + } + StepResult::IO => { + let syncing = env.io.syncing(); + if syncing { + reopen_database(env); + } else { + rows.run_once().unwrap(); + } + } + StepResult::Done => { + break; + } + StepResult::Busy => { + return Err(turso_core::LimboError::Busy); + } + StepResult::Interrupt => {} + } + } + + Ok(out) + } else { + unreachable!("unexpected: this function should only be called on queries") + } + } + + pub(crate) fn execute_faulty_query( + &self, + conn: &Arc, + env: &mut SimulatorEnv, + ) -> ResultSet { + use rand::Rng; + if let Self::FaultyQuery(query) = self { + let query_str = query.to_string(); + let rows = conn.query(&query_str); + if rows.is_err() { + let err = rows.err(); + tracing::debug!( + "Error running query '{}': {:?}", + &query_str[0..query_str.len().min(4096)], + err + ); + if let Some(turso_core::LimboError::ParseError(e)) = err { + panic!("Unexpected parse error: {e}"); + } + return Err(err.unwrap()); + } + let mut rows = rows.unwrap().unwrap(); + let mut out = Vec::new(); + let mut current_prob = 0.05; + let mut incr = 0.001; + loop { + let syncing = env.io.syncing(); + let inject_fault = env.rng.random_bool(current_prob); + // TODO: avoid for now injecting faults when syncing + if inject_fault && !syncing { + env.io.inject_fault(true); + } + + match rows.step()? { + StepResult::Row => { + let row = rows.row().unwrap(); + let mut r = Vec::new(); + for v in row.get_values() { + let v = v.into(); + r.push(v); + } + out.push(r); + } + StepResult::IO => { + rows.run_once()?; + current_prob += incr; + if current_prob > 1.0 { + current_prob = 1.0; + } else { + incr *= 1.01; + } + } + StepResult::Done => { + break; + } + StepResult::Busy => { + return Err(turso_core::LimboError::Busy); + } + StepResult::Interrupt => {} + } + } + + Ok(out) + } else { + unreachable!("unexpected: this function should only be called on queries") + } + } +} + +fn reopen_database(env: &mut SimulatorEnv) { + // 1. Close all connections without default checkpoint-on-close behavior + // to expose bugs related to how we handle WAL + let mvcc = env.profile.experimental_mvcc; + let indexes = env.profile.query.gen_opts.indexes; + let num_conns = env.connections.len(); + env.connections.clear(); + + // Clear all open files + // TODO: for correct reporting of faults we should get all the recorded numbers and transfer to the new file + env.io.close_files(); + + // 2. Re-open database + match env.type_ { + SimulationType::Differential => { + for _ in 0..num_conns { + env.connections.push(SimConnection::SQLiteConnection( + rusqlite::Connection::open(env.get_db_path()) + .expect("Failed to open SQLite connection"), + )); + } + } + SimulationType::Default | SimulationType::Doublecheck => { + env.db = None; + let db = match turso_core::Database::open_file_with_flags( + env.io.clone(), + env.get_db_path().to_str().expect("path should be 'to_str'"), + turso_core::OpenFlags::default(), + turso_core::DatabaseOpts::new() + .with_mvcc(mvcc) + .with_indexes(indexes) + .with_autovacuum(true), + None, + ) { + Ok(db) => db, + Err(e) => { + tracing::error!( + "Failed to open database at {}: {}", + env.get_db_path().display(), + e + ); + panic!("Failed to open database: {e}"); + } + }; + + env.db = Some(db); + + for _ in 0..num_conns { + env.connections.push(SimConnection::LimboConnection( + env.db.as_ref().expect("db to be Some").connect().unwrap(), + )); + } + } + }; +} diff --git a/simulator/model/metrics.rs b/simulator/model/metrics.rs new file mode 100644 index 000000000..217e47f6f --- /dev/null +++ b/simulator/model/metrics.rs @@ -0,0 +1,99 @@ +use sql_generation::generation::GenerationContext; + +use crate::{model::interactions::InteractionStats, profiles::query::QueryProfile}; + +#[derive(Debug)] +pub struct Remaining { + pub select: u32, + pub insert: u32, + pub create: u32, + pub create_index: u32, + pub delete: u32, + pub update: u32, + pub drop: u32, + pub alter_table: u32, + pub drop_index: u32, + pub pragma_count: u32, +} + +impl Remaining { + pub fn new( + max_interactions: u32, + opts: &QueryProfile, + stats: &InteractionStats, + mvcc: bool, + context: &impl GenerationContext, + ) -> Remaining { + let total_weight = opts.total_weight(); + + let total_select = (max_interactions * opts.select_weight) / total_weight; + let total_insert = (max_interactions * opts.insert_weight) / total_weight; + let total_create = (max_interactions * opts.create_table_weight) / total_weight; + let total_create_index = (max_interactions * opts.create_index_weight) / total_weight; + let total_delete = (max_interactions * opts.delete_weight) / total_weight; + let total_update = (max_interactions * opts.update_weight) / total_weight; + let total_drop = (max_interactions * opts.drop_table_weight) / total_weight; + let total_alter_table = (max_interactions * opts.alter_table_weight) / total_weight; + let total_drop_index = (max_interactions * opts.drop_index) / total_weight; + let total_pragma = (max_interactions * opts.pragma_weight) / total_weight; + + let remaining_select = total_select + .checked_sub(stats.select_count) + .unwrap_or_default(); + let remaining_insert = total_insert + .checked_sub(stats.insert_count) + .unwrap_or_default(); + let remaining_create = total_create + .checked_sub(stats.create_count) + .unwrap_or_default(); + let mut remaining_create_index = total_create_index + .checked_sub(stats.create_index_count) + .unwrap_or_default(); + let remaining_delete = total_delete + .checked_sub(stats.delete_count) + .unwrap_or_default(); + let remaining_update = total_update + .checked_sub(stats.update_count) + .unwrap_or_default(); + let remaining_drop = total_drop.checked_sub(stats.drop_count).unwrap_or_default(); + let remaining_pragma = total_pragma + .checked_sub(stats.pragma_count) + .unwrap_or_default(); + + let remaining_alter_table = total_alter_table + .checked_sub(stats.alter_table_count) + .unwrap_or_default(); + + let mut remaining_drop_index = total_drop_index + .checked_sub(stats.alter_table_count) + .unwrap_or_default(); + + if mvcc { + // TODO: index not supported yet for mvcc + remaining_create_index = 0; + remaining_drop_index = 0; + } + + // if there are no indexes do not allow creation of drop_index + if !context + .tables() + .iter() + .any(|table| !table.indexes.is_empty()) + { + remaining_drop_index = 0; + } + + Remaining { + select: remaining_select, + insert: remaining_insert, + create: remaining_create, + create_index: remaining_create_index, + delete: remaining_delete, + drop: remaining_drop, + update: remaining_update, + alter_table: remaining_alter_table, + drop_index: remaining_drop_index, + pragma_count: remaining_pragma, + } + } +} diff --git a/simulator/model/mod.rs b/simulator/model/mod.rs index ee45fbcba..f38966e40 100644 --- a/simulator/model/mod.rs +++ b/simulator/model/mod.rs @@ -20,6 +20,12 @@ use turso_parser::ast::Distinctness; use crate::{generation::Shadow, runner::env::ShadowTablesMut}; +pub mod interactions; +pub mod metrics; +pub mod property; + +pub(crate) type ResultSet = turso_core::Result>>; + // This type represents the potential queries on the database. #[derive(Debug, Clone, Serialize, Deserialize, strum::EnumDiscriminants)] pub enum Query { diff --git a/simulator/model/property.rs b/simulator/model/property.rs new file mode 100644 index 000000000..7dff89577 --- /dev/null +++ b/simulator/model/property.rs @@ -0,0 +1,239 @@ +use serde::{Deserialize, Serialize}; +use sql_generation::model::query::{Create, Insert, Select, predicate::Predicate, update::Update}; + +use crate::model::Query; + +/// Properties are representations of executable specifications +/// about the database behavior. +#[derive(Debug, Clone, Serialize, Deserialize, strum::EnumDiscriminants)] +#[strum_discriminants(derive(strum::EnumIter))] +pub enum Property { + /// Insert-Select is a property in which the inserted row + /// must be in the resulting rows of a select query that has a + /// where clause that matches the inserted row. + /// The execution of the property is as follows + /// INSERT INTO VALUES (...) + /// I_0 + /// I_1 + /// ... + /// I_n + /// SELECT * FROM WHERE + /// The interactions in the middle has the following constraints; + /// - There will be no errors in the middle interactions. + /// - The inserted row will not be deleted. + /// - The inserted row will not be updated. + /// - The table `t` will not be renamed, dropped, or altered. + InsertValuesSelect { + /// The insert query + insert: Insert, + /// Selected row index + row_index: usize, + /// Additional interactions in the middle of the property + queries: Vec, + /// The select query + select: Select, + /// Interactive query information if any + interactive: Option, + }, + /// ReadYourUpdatesBack is a property in which the updated rows + /// must be in the resulting rows of a select query that has a + /// where clause that matches the updated row. + /// The execution of the property is as follows + /// UPDATE SET WHERE + /// SELECT FROM WHERE + /// These interactions are executed in immediate succession + /// just to verify the property that our updates did what they + /// were supposed to do. + ReadYourUpdatesBack { + update: Update, + select: Select, + }, + /// TableHasExpectedContent is a property in which the table + /// must have the expected content, i.e. all the insertions and + /// updates and deletions should have been persisted in the way + /// we think they were. + /// The execution of the property is as follows + /// SELECT * FROM + /// ASSERT + TableHasExpectedContent { + table: String, + }, + /// AllTablesHaveExpectedContent is a property in which the table + /// must have the expected content, i.e. all the insertions and + /// updates and deletions should have been persisted in the way + /// we think they were. + /// The execution of the property is as follows + /// SELECT * FROM + /// ASSERT + /// for each table in the simulator model + AllTableHaveExpectedContent { + tables: Vec, + }, + /// Double Create Failure is a property in which creating + /// the same table twice leads to an error. + /// The execution of the property is as follows + /// CREATE TABLE (...) + /// I_0 + /// I_1 + /// ... + /// I_n + /// CREATE TABLE (...) -> Error + /// The interactions in the middle has the following constraints; + /// - There will be no errors in the middle interactions. + /// - Table `t` will not be renamed or dropped. + DoubleCreateFailure { + /// The create query + create: Create, + /// Additional interactions in the middle of the property + queries: Vec, + }, + /// Select Limit is a property in which the select query + /// has a limit clause that is respected by the query. + /// The execution of the property is as follows + /// SELECT * FROM WHERE LIMIT + /// This property is a single-interaction property. + /// The interaction has the following constraints; + /// - The select query will respect the limit clause. + SelectLimit { + /// The select query + select: Select, + }, + /// Delete-Select is a property in which the deleted row + /// must not be in the resulting rows of a select query that has a + /// where clause that matches the deleted row. In practice, `p1` of + /// the delete query will be used as the predicate for the select query, + /// hence the select should return NO ROWS. + /// The execution of the property is as follows + /// DELETE FROM WHERE + /// I_0 + /// I_1 + /// ... + /// I_n + /// SELECT * FROM WHERE + /// The interactions in the middle has the following constraints; + /// - There will be no errors in the middle interactions. + /// - A row that holds for the predicate will not be inserted. + /// - The table `t` will not be renamed, dropped, or altered. + DeleteSelect { + table: String, + predicate: Predicate, + queries: Vec, + }, + /// Drop-Select is a property in which selecting from a dropped table + /// should result in an error. + /// The execution of the property is as follows + /// DROP TABLE + /// I_0 + /// I_1 + /// ... + /// I_n + /// SELECT * FROM WHERE -> Error + /// The interactions in the middle has the following constraints; + /// - There will be no errors in the middle interactions. + /// - The table `t` will not be created, no table will be renamed to `t`. + DropSelect { + table: String, + queries: Vec, + select: Select, + }, + /// Select-Select-Optimizer is a property in which we test the optimizer by + /// running two equivalent select queries, one with `SELECT from ` + /// and the other with `SELECT * from WHERE `. As highlighted by + /// Rigger et al. in Non-Optimizing Reference Engine Construction(NoREC), SQLite + /// tends to optimize `where` statements while keeping the result column expressions + /// unoptimized. This property is used to test the optimizer. The property is successful + /// if the two queries return the same number of rows. + SelectSelectOptimizer { + table: String, + predicate: Predicate, + }, + /// Where-True-False-Null is a property that tests the boolean logic implementation + /// in the database. It relies on the fact that `P == true || P == false || P == null` should return true, + /// as SQLite uses a ternary logic system. This property is invented in "Finding Bugs in Database Systems via Query Partitioning" + /// by Rigger et al. and it is canonically called Ternary Logic Partitioning (TLP). + WhereTrueFalseNull { + select: Select, + predicate: Predicate, + }, + /// UNION-ALL-Preserves-Cardinality is a property that tests the UNION ALL operator + /// implementation in the database. It relies on the fact that `SELECT * FROM WHERE UNION ALL SELECT * FROM WHERE ` + /// should return the same number of rows as `SELECT FROM WHERE `. + /// > The property is succesfull when the UNION ALL of 2 select queries returns the same number of rows + /// > as the sum of the two select queries. + UNIONAllPreservesCardinality { + select: Select, + where_clause: Predicate, + }, + /// FsyncNoWait is a property which tests if we do not loose any data after not waiting for fsync. + /// + /// # Interactions + /// - Executes the `query` without waiting for fsync + /// - Drop all connections and Reopen the database + /// - Execute the `query` again + /// - Query tables to assert that the values were inserted + /// + FsyncNoWait { + query: Query, + }, + FaultyQuery { + query: Query, + }, + /// Property used to subsititute a property with its queries only + Queries { + queries: Vec, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InteractiveQueryInfo { + pub start_with_immediate: bool, + pub end_with_commit: bool, +} + +impl Property { + pub(crate) fn name(&self) -> &str { + match self { + Property::InsertValuesSelect { .. } => "Insert-Values-Select", + Property::ReadYourUpdatesBack { .. } => "Read-Your-Updates-Back", + Property::TableHasExpectedContent { .. } => "Table-Has-Expected-Content", + Property::AllTableHaveExpectedContent { .. } => "All-Tables-Have-Expected-Content", + Property::DoubleCreateFailure { .. } => "Double-Create-Failure", + Property::SelectLimit { .. } => "Select-Limit", + Property::DeleteSelect { .. } => "Delete-Select", + Property::DropSelect { .. } => "Drop-Select", + Property::SelectSelectOptimizer { .. } => "Select-Select-Optimizer", + Property::WhereTrueFalseNull { .. } => "Where-True-False-Null", + Property::FsyncNoWait { .. } => "FsyncNoWait", + Property::FaultyQuery { .. } => "FaultyQuery", + Property::UNIONAllPreservesCardinality { .. } => "UNION-All-Preserves-Cardinality", + Property::Queries { .. } => "Queries", + } + } + + /// Property Does some sort of fault injection + pub fn check_tables(&self) -> bool { + matches!( + self, + Property::FsyncNoWait { .. } | Property::FaultyQuery { .. } + ) + } + + pub fn get_extensional_queries(&mut self) -> Option<&mut Vec> { + match self { + Property::InsertValuesSelect { queries, .. } + | Property::DoubleCreateFailure { queries, .. } + | Property::DeleteSelect { queries, .. } + | Property::DropSelect { queries, .. } + | Property::Queries { queries } => Some(queries), + Property::FsyncNoWait { .. } | Property::FaultyQuery { .. } => None, + Property::SelectLimit { .. } + | Property::SelectSelectOptimizer { .. } + | Property::WhereTrueFalseNull { .. } + | Property::UNIONAllPreservesCardinality { .. } + | Property::ReadYourUpdatesBack { .. } + | Property::TableHasExpectedContent { .. } + | Property::AllTableHaveExpectedContent { .. } => None, + } + } +} diff --git a/simulator/runner/differential.rs b/simulator/runner/differential.rs index bf6797eb2..6b458e4fb 100644 --- a/simulator/runner/differential.rs +++ b/simulator/runner/differential.rs @@ -8,7 +8,7 @@ use similar_asserts::SimpleDiff; use sql_generation::model::table::SimValue; use crate::{ - generation::plan::{ConnectionState, InteractionPlanIterator, InteractionPlanState}, + model::interactions::{ConnectionState, InteractionPlanIterator, InteractionPlanState}, runner::execution::ExecutionContinuation, }; diff --git a/simulator/runner/doublecheck.rs b/simulator/runner/doublecheck.rs index a76965048..0b8e41eb3 100644 --- a/simulator/runner/doublecheck.rs +++ b/simulator/runner/doublecheck.rs @@ -4,7 +4,7 @@ use std::{ }; use crate::{ - generation::plan::{ConnectionState, InteractionPlanIterator, InteractionPlanState}, + model::interactions::{ConnectionState, InteractionPlanIterator, InteractionPlanState}, runner::execution::ExecutionContinuation, }; diff --git a/simulator/runner/execution.rs b/simulator/runner/execution.rs index 1e30de520..148a996fb 100644 --- a/simulator/runner/execution.rs +++ b/simulator/runner/execution.rs @@ -5,14 +5,14 @@ use tracing::instrument; use turso_core::{Connection, LimboError, Result, StepResult, Value}; use crate::{ - generation::{ - Shadow as _, - plan::{ + generation::Shadow as _, + model::{ + Query, ResultSet, + interactions::{ ConnectionState, Interaction, InteractionPlanIterator, InteractionPlanState, - InteractionType, ResultSet, + InteractionType, }, }, - model::Query, }; use super::env::{SimConnection, SimulatorEnv}; diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 9f80f78cc..0ea62d45a 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -2,11 +2,11 @@ use indexmap::IndexSet; use crate::{ SandboxedResult, SimulatorEnv, - generation::{ - plan::{InteractionPlan, InteractionType, Interactions, InteractionsType}, + model::{ + Query, + interactions::{InteractionPlan, InteractionType, Interactions, InteractionsType}, property::Property, }, - model::Query, run_simulation, runner::execution::Execution, }; From 4f143f385a7b4bbd5345c83973efe2cfe65fa499 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 11:55:19 -0300 Subject: [PATCH 02/16] - modify bugbase to not save interaction plan in a `plan.json`. As we will track `Interaction` instead of `Interactions` in the Plan, this change will impossibilitate the serialization of the InteractionPlan with Serde Json. - make --load just load the previous cli args --- simulator/main.rs | 134 ++++++------- simulator/runner/bugbase.rs | 365 ++++++++++++------------------------ simulator/runner/cli.rs | 2 +- 3 files changed, 169 insertions(+), 332 deletions(-) diff --git a/simulator/main.rs b/simulator/main.rs index 9523dbdce..1012a34d3 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -4,7 +4,7 @@ use clap::Parser; use notify::event::{DataChange, ModifyKind}; use notify::{EventKind, RecursiveMode, Watcher}; use rand::prelude::*; -use runner::bugbase::{Bug, BugBase, LoadedBug}; +use runner::bugbase::BugBase; use runner::cli::{SimulatorCLI, SimulatorCommand}; use runner::differential; use runner::env::SimulatorEnv; @@ -43,7 +43,7 @@ fn main() -> anyhow::Result<()> { let profile = Profile::parse_from_type(cli_opts.profile.clone())?; tracing::debug!(sim_profile = ?profile); - if let Some(ref command) = cli_opts.subcommand { + if let Some(command) = cli_opts.subcommand.take() { match command { SimulatorCommand::List => { let mut bugbase = BugBase::load()?; @@ -51,10 +51,10 @@ fn main() -> anyhow::Result<()> { } SimulatorCommand::Loop { n, short_circuit } => { banner(); - for i in 0..*n { + for i in 0..n { println!("iteration {i}"); - let result = testing_main(&cli_opts, &profile); - if result.is_err() && *short_circuit { + let result = testing_main(&mut cli_opts, &profile); + if result.is_err() && short_circuit { println!("short circuiting after {i} iterations"); return result; } else if result.is_err() { @@ -66,7 +66,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } SimulatorCommand::Test { filter } => { - let mut bugbase = BugBase::load()?; + let bugbase = BugBase::load()?; let bugs = bugbase.load_bugs()?; let mut bugs = bugs .into_iter() @@ -75,7 +75,7 @@ fn main() -> anyhow::Result<()> { .runs .into_iter() .filter_map(|run| run.error.clone().map(|_| run)) - .filter(|run| run.error.as_ref().unwrap().contains(filter)) + .filter(|run| run.error.as_ref().unwrap().contains(&filter)) .map(|run| run.cli_options) .collect::>(); @@ -100,7 +100,7 @@ fn main() -> anyhow::Result<()> { let results = bugs .into_iter() - .map(|cli_opts| testing_main(&cli_opts, &profile)) + .map(|mut cli_opts| testing_main(&mut cli_opts, &profile)) .collect::>(); let (successes, failures): (Vec<_>, Vec<_>) = @@ -118,11 +118,11 @@ fn main() -> anyhow::Result<()> { } } else { banner(); - testing_main(&cli_opts, &profile) + testing_main(&mut cli_opts, &profile) } } -fn testing_main(cli_opts: &SimulatorCLI, profile: &Profile) -> anyhow::Result<()> { +fn testing_main(cli_opts: &mut SimulatorCLI, profile: &Profile) -> anyhow::Result<()> { let mut bugbase = if cli_opts.disable_bugbase { None } else { @@ -260,11 +260,6 @@ fn run_simulator( tracing::info!("{}", plan.stats()); std::fs::write(env.get_plan_path(), plan.to_string()).unwrap(); - std::fs::write( - env.get_plan_path().with_extension("json"), - serde_json::to_string_pretty(&*plan).unwrap(), - ) - .unwrap(); // No doublecheck, run shrinking if panicking or found a bug. match &result { @@ -385,7 +380,7 @@ fn run_simulator( ); // Save the shrunk database if let Some(bugbase) = bugbase.as_deref_mut() { - bugbase.make_shrunk( + bugbase.save_shrunk( seed, cli_opts, final_plan.clone(), @@ -471,81 +466,58 @@ impl SandboxedResult { } fn setup_simulation( - bugbase: Option<&mut BugBase>, - cli_opts: &SimulatorCLI, + mut bugbase: Option<&mut BugBase>, + cli_opts: &mut SimulatorCLI, profile: &Profile, ) -> (u64, SimulatorEnv, InteractionPlan) { - if let Some(seed) = &cli_opts.load { - let seed = seed.parse::().expect("seed should be a number"); - let bugbase = bugbase.expect("BugBase must be enabled to load a bug"); - tracing::info!("seed={}", seed); - let bug = bugbase - .get_bug(seed) - .unwrap_or_else(|| panic!("bug '{seed}' not found in bug base")); - + if let Some(seed) = cli_opts.load { + let bugbase = bugbase + .as_mut() + .expect("BugBase must be enabled to load a bug"); let paths = bugbase.paths(seed); if !paths.base.exists() { std::fs::create_dir_all(&paths.base).unwrap(); } - let env = SimulatorEnv::new( - bug.seed(), - cli_opts, - paths, - SimulationType::Default, - profile, - ); - let plan = match bug { - Bug::Loaded(LoadedBug { plan, .. }) => plan.clone(), - Bug::Unloaded { seed } => { - let seed = *seed; - bugbase - .load_bug(seed) - .unwrap_or_else(|_| panic!("could not load bug '{seed}' in bug base")) - .plan - .clone() - } - }; + let bug = bugbase + .get_or_load_bug(seed) + .unwrap() + .unwrap_or_else(|| panic!("bug '{seed}' not found in bug base")); - std::fs::write(env.get_plan_path(), plan.to_string()).unwrap(); - std::fs::write( - env.get_plan_path().with_extension("json"), - serde_json::to_string_pretty(&plan).unwrap(), - ) - .unwrap(); - (seed, env, plan) - } else { - let seed = cli_opts.seed.unwrap_or_else(|| { - let mut rng = rand::rng(); - rng.next_u64() - }); - tracing::info!("seed={}", seed); - - let paths = if let Some(bugbase) = bugbase { - let paths = bugbase.paths(seed); - // Create the output directory if it doesn't exist - if !paths.base.exists() { - std::fs::create_dir_all(&paths.base) - .map_err(|e| format!("{e:?}")) - .unwrap(); - } - paths - } else { - let dir = std::env::current_dir().unwrap().join("simulator-output"); - std::fs::create_dir_all(&dir).unwrap(); - Paths::new(&dir) - }; - - let mut env = SimulatorEnv::new(seed, cli_opts, paths, SimulationType::Default, profile); - - tracing::info!("Generating database interaction plan..."); - - let plan = InteractionPlan::init_plan(&mut env); - - (seed, env, plan) + // run the simulation with the same CLI options as the loaded bug + *cli_opts = bug.last_cli_opts(); } -} + let seed = cli_opts.seed.unwrap_or_else(|| { + let mut rng = rand::rng(); + rng.next_u64() + }); + tracing::info!("seed={}", seed); + cli_opts.seed = Some(seed); + + let paths = if let Some(bugbase) = bugbase { + let paths = bugbase.paths(seed); + // Create the output directory if it doesn't exist + if !paths.base.exists() { + std::fs::create_dir_all(&paths.base) + .map_err(|e| format!("{e:?}")) + .unwrap(); + } + paths + } else { + let dir = std::env::current_dir().unwrap().join("simulator-output"); + std::fs::create_dir_all(&dir).unwrap(); + Paths::new(&dir) + }; + + let mut env = SimulatorEnv::new(seed, cli_opts, paths, SimulationType::Default, profile); + + tracing::info!("Generating database interaction plan..."); + + let plan = InteractionPlan::init_plan(&mut env); + + (seed, env, plan) +} fn run_simulation( env: Arc>, plan: impl InteractionPlanIterator, diff --git a/simulator/runner/bugbase.rs b/simulator/runner/bugbase.rs index 89ad25e71..ce8892b04 100644 --- a/simulator/runner/bugbase.rs +++ b/simulator/runner/bugbase.rs @@ -2,7 +2,7 @@ use std::{ collections::HashMap, env::current_dir, fs::File, - io::{self, Read, Write}, + io::Read, path::{Path, PathBuf}, time::SystemTime, }; @@ -11,60 +11,84 @@ use anyhow::{Context, anyhow}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use crate::{InteractionPlan, Paths}; +use crate::{Paths, model::interactions::InteractionPlan}; use super::cli::SimulatorCLI; +const READABLE_PLAN_PATH: &str = "plan.sql"; +const SHRUNK_READABLE_PLAN_PATH: &str = "shrunk.sql"; +const SEED_PATH: &str = "seed.txt"; +const RUNS_PATH: &str = "runs.json"; + /// A bug is a run that has been identified as buggy. #[derive(Clone)] -pub(crate) enum Bug { - Unloaded { seed: u64 }, - Loaded(LoadedBug), -} - -#[derive(Clone)] -pub struct LoadedBug { +pub struct Bug { /// The seed of the bug. pub seed: u64, + /// The plan of the bug. - pub plan: InteractionPlan, + /// TODO: currently plan is only saved to the .sql file, and that is not deserializable yet + /// so we cannot always store an interaction plan here + pub plan: Option, + /// The shrunk plan of the bug, if any. pub shrunk_plan: Option, + /// The runs of the bug. pub runs: Vec, } #[derive(Clone, Serialize, Deserialize)] -pub(crate) struct BugRun { +pub struct BugRun { /// Commit hash of the current version of Limbo. - pub(crate) hash: String, + pub hash: String, /// Timestamp of the run. #[serde(with = "chrono::serde::ts_seconds")] - pub(crate) timestamp: DateTime, + pub timestamp: DateTime, /// Error message of the run. - pub(crate) error: Option, + pub error: Option, /// Options - pub(crate) cli_options: SimulatorCLI, + pub cli_options: SimulatorCLI, /// Whether the run was a shrunk run. - pub(crate) shrunk: bool, + pub shrunk: bool, } impl Bug { - #[expect(dead_code)] - /// Check if the bug is loaded. - pub(crate) fn is_loaded(&self) -> bool { - match self { - Bug::Unloaded { .. } => false, - Bug::Loaded { .. } => true, + fn save_to_path(&self, path: impl AsRef) -> anyhow::Result<()> { + let path = path.as_ref(); + let bug_path = path.join(self.seed.to_string()); + std::fs::create_dir_all(&bug_path) + .with_context(|| "should be able to create bug directory")?; + + let seed_path = bug_path.join(SEED_PATH); + std::fs::write(&seed_path, self.seed.to_string()) + .with_context(|| "should be able to write seed file")?; + + if let Some(plan) = &self.plan { + let readable_plan_path = bug_path.join(READABLE_PLAN_PATH); + std::fs::write(&readable_plan_path, plan.to_string()) + .with_context(|| "should be able to write readable plan file")?; } + + if let Some(shrunk_plan) = &self.shrunk_plan { + let readable_shrunk_plan_path = bug_path.join(SHRUNK_READABLE_PLAN_PATH); + std::fs::write(&readable_shrunk_plan_path, shrunk_plan.to_string()) + .with_context(|| "should be able to write readable shrunk plan file")?; + } + + let runs_path = bug_path.join(RUNS_PATH); + std::fs::write( + &runs_path, + serde_json::to_string_pretty(&self.runs) + .with_context(|| "should be able to serialize runs")?, + ) + .with_context(|| "should be able to write runs file")?; + + Ok(()) } - /// Get the seed of the bug. - pub(crate) fn seed(&self) -> u64 { - match self { - Bug::Unloaded { seed } => *seed, - Bug::Loaded(LoadedBug { seed, .. }) => *seed, - } + pub fn last_cli_opts(&self) -> SimulatorCLI { + self.runs.last().unwrap().cli_options.clone() } } @@ -73,13 +97,14 @@ pub(crate) struct BugBase { /// Path to the bug base directory. path: PathBuf, /// The list of buggy runs, uniquely identified by their seed - bugs: HashMap, + bugs: HashMap>, } impl BugBase { /// Create a new bug base. fn new(path: PathBuf) -> anyhow::Result { let mut bugs = HashMap::new(); + // list all the bugs in the path as directories if let Ok(entries) = std::fs::read_dir(&path) { for entry in entries.flatten() { @@ -95,7 +120,7 @@ impl BugBase { entry.file_name().to_string_lossy() ) })?; - bugs.insert(seed, Bug::Unloaded { seed }); + bugs.insert(seed, None); } } } @@ -105,7 +130,7 @@ impl BugBase { /// Load the bug base from one of the potential paths. pub(crate) fn load() -> anyhow::Result { - let potential_paths = vec![ + let potential_paths = [ // limbo project directory BugBase::get_limbo_project_dir()?, // home directory @@ -132,57 +157,33 @@ impl BugBase { Err(anyhow!("failed to create bug base")) } - #[expect(dead_code)] - /// Load the bug base from one of the potential paths. - pub(crate) fn interactive_load() -> anyhow::Result { - let potential_paths = vec![ - // limbo project directory - BugBase::get_limbo_project_dir()?, - // home directory - dirs::home_dir().with_context(|| "should be able to get home directory")?, - // current directory - std::env::current_dir().with_context(|| "should be able to get current directory")?, - ]; + fn load_bug(&self, seed: u64) -> anyhow::Result { + let path = self.path.join(seed.to_string()).join(RUNS_PATH); - for path in potential_paths { - let path = path.join(".bugbase"); - if path.exists() { - return BugBase::new(path); - } - } - - println!("select bug base location:"); - println!("1. limbo project directory"); - println!("2. home directory"); - println!("3. current directory"); - print!("> "); - io::stdout().flush().unwrap(); - let mut choice = String::new(); - io::stdin() - .read_line(&mut choice) - .expect("failed to read line"); - - let choice = choice - .trim() - .parse::() - .with_context(|| format!("invalid choice {choice}"))?; - let path = match choice { - 1 => BugBase::get_limbo_project_dir()?.join(".bugbase"), - 2 => { - let home = std::env::var("HOME").with_context(|| "failed to get home directory")?; - PathBuf::from(home).join(".bugbase") - } - 3 => PathBuf::from(".bugbase"), - _ => anyhow::bail!(format!("invalid choice {choice}")), + let runs = if !path.exists() { + vec![] + } else { + std::fs::read_to_string(self.path.join(seed.to_string()).join(RUNS_PATH)) + .with_context(|| "should be able to read runs file") + .and_then(|runs| serde_json::from_str(&runs).map_err(|e| anyhow!("{}", e)))? }; - if path.exists() { - unreachable!("bug base already exists at {}", path.display()); - } else { - std::fs::create_dir_all(&path).with_context(|| "failed to create bug base")?; - tracing::info!("bug base created at {}", path.display()); - BugBase::new(path) - } + let bug = Bug { + seed, + plan: None, + shrunk_plan: None, + runs, + }; + Ok(bug) + } + + pub fn load_bugs(&self) -> anyhow::Result> { + let seeds = self.bugs.keys().copied().collect::>(); + + seeds + .iter() + .map(|seed| self.load_bug(*seed)) + .collect::, _>>() } /// Add a new bug to the bug base. @@ -193,12 +194,11 @@ impl BugBase { error: Option, cli_options: &SimulatorCLI, ) -> anyhow::Result<()> { - tracing::debug!("adding bug with seed {}", seed); - let bug = self.get_bug(seed); + let path = self.path.clone(); - if bug.is_some() { - let mut bug = self.load_bug(seed)?; - bug.plan = plan.clone(); + tracing::debug!("adding bug with seed {}", seed); + let bug = self.get_or_load_bug(seed)?; + let bug = if let Some(bug) = bug { bug.runs.push(BugRun { hash: Self::get_current_commit_hash()?, timestamp: SystemTime::now().into(), @@ -206,11 +206,13 @@ impl BugBase { cli_options: cli_options.clone(), shrunk: false, }); - self.bugs.insert(seed, Bug::Loaded(bug.clone())); + bug.plan = Some(plan); + bug } else { - let bug = LoadedBug { + let bug = Bug { seed, - plan: plan.clone(), + plan: Some(plan), + shrunk_plan: None, runs: vec![BugRun { hash: Self::get_current_commit_hash()?, timestamp: SystemTime::now().into(), @@ -218,172 +220,44 @@ impl BugBase { cli_options: cli_options.clone(), shrunk: false, }], - shrunk_plan: None, }; - self.bugs.insert(seed, Bug::Loaded(bug.clone())); - } + + self.bugs.insert(seed, Some(bug.clone())); + self.bugs.get_mut(&seed).unwrap().as_mut().unwrap() + }; + // Save the bug to the bug base. - self.save_bug(seed) + bug.save_to_path(&path) } - /// Get a bug from the bug base. - pub(crate) fn get_bug(&self, seed: u64) -> Option<&Bug> { - self.bugs.get(&seed) - } + pub fn get_or_load_bug(&mut self, seed: u64) -> anyhow::Result> { + // Check if the bug exists and is loaded + let needs_loading = match self.bugs.get(&seed) { + Some(Some(_)) => false, // Already loaded + Some(None) => true, // Exists but unloaded + None => return Ok(None), // Doesn't exist + }; - /// Save a bug to the bug base. - fn save_bug(&self, seed: u64) -> anyhow::Result<()> { - let bug = self.get_bug(seed); - - match bug { - None | Some(Bug::Unloaded { .. }) => { - unreachable!("save should only be called within add_bug"); - } - Some(Bug::Loaded(bug)) => { - let bug_path = self.path.join(seed.to_string()); - std::fs::create_dir_all(&bug_path) - .with_context(|| "should be able to create bug directory")?; - - let seed_path = bug_path.join("seed.txt"); - std::fs::write(&seed_path, seed.to_string()) - .with_context(|| "should be able to write seed file")?; - - let plan_path = bug_path.join("plan.json"); - std::fs::write( - &plan_path, - serde_json::to_string_pretty(&bug.plan) - .with_context(|| "should be able to serialize plan")?, - ) - .with_context(|| "should be able to write plan file")?; - - if let Some(shrunk_plan) = &bug.shrunk_plan { - let shrunk_plan_path = bug_path.join("shrunk.json"); - std::fs::write( - &shrunk_plan_path, - serde_json::to_string_pretty(shrunk_plan) - .with_context(|| "should be able to serialize shrunk plan")?, - ) - .with_context(|| "should be able to write shrunk plan file")?; - - let readable_shrunk_plan_path = bug_path.join("shrunk.sql"); - std::fs::write(&readable_shrunk_plan_path, shrunk_plan.to_string()) - .with_context(|| "should be able to write readable shrunk plan file")?; - } - - let readable_plan_path = bug_path.join("plan.sql"); - std::fs::write(&readable_plan_path, bug.plan.to_string()) - .with_context(|| "should be able to write readable plan file")?; - - let runs_path = bug_path.join("runs.json"); - std::fs::write( - &runs_path, - serde_json::to_string_pretty(&bug.runs) - .with_context(|| "should be able to serialize runs")?, - ) - .with_context(|| "should be able to write runs file")?; - } + if needs_loading { + let bug = self.load_bug(seed)?; + self.bugs.insert(seed, Some(bug)); } - Ok(()) + // Now get the mutable reference + Ok(self.bugs.get_mut(&seed).and_then(|opt| opt.as_mut())) } - pub(crate) fn load_bug(&mut self, seed: u64) -> anyhow::Result { - let seed_match = self.bugs.get(&seed); - - match seed_match { - None => anyhow::bail!("No bugs found for seed {}", seed), - Some(Bug::Unloaded { .. }) => { - let plan = - std::fs::read_to_string(self.path.join(seed.to_string()).join("plan.json")) - .with_context(|| { - format!( - "should be able to read plan file at {}", - self.path.join(seed.to_string()).join("plan.json").display() - ) - })?; - let plan: InteractionPlan = serde_json::from_str(&plan) - .with_context(|| "should be able to deserialize plan")?; - - let shrunk_plan: Option = - std::fs::read_to_string(self.path.join(seed.to_string()).join("shrunk.json")) - .with_context(|| "should be able to read shrunk plan file") - .and_then(|shrunk| { - serde_json::from_str(&shrunk).map_err(|e| anyhow!("{}", e)) - }) - .ok(); - - let shrunk_plan: Option = - shrunk_plan.and_then(|shrunk_plan| serde_json::from_str(&shrunk_plan).ok()); - - let runs = - std::fs::read_to_string(self.path.join(seed.to_string()).join("runs.json")) - .with_context(|| "should be able to read runs file") - .and_then(|runs| serde_json::from_str(&runs).map_err(|e| anyhow!("{}", e))) - .unwrap_or_default(); - - let bug = LoadedBug { - seed, - plan: plan.clone(), - runs, - shrunk_plan, - }; - - self.bugs.insert(seed, Bug::Loaded(bug.clone())); - tracing::debug!("Loaded bug with seed {}", seed); - Ok(bug) - } - Some(Bug::Loaded(bug)) => { - tracing::warn!( - "Bug with seed {} is already loaded, returning the existing plan", - seed - ); - Ok(bug.clone()) - } - } - } - - #[expect(dead_code)] - pub(crate) fn mark_successful_run( - &mut self, - seed: u64, - cli_options: &SimulatorCLI, - ) -> anyhow::Result<()> { - let bug = self.get_bug(seed); - match bug { - None => { - tracing::debug!("removing bug base entry for {}", seed); - std::fs::remove_dir_all(self.path.join(seed.to_string())) - .with_context(|| "should be able to remove bug directory")?; - } - Some(_) => { - let mut bug = self.load_bug(seed)?; - bug.runs.push(BugRun { - hash: Self::get_current_commit_hash()?, - timestamp: SystemTime::now().into(), - error: None, - cli_options: cli_options.clone(), - shrunk: false, - }); - self.bugs.insert(seed, Bug::Loaded(bug.clone())); - // Save the bug to the bug base. - self.save_bug(seed) - .with_context(|| "should be able to save bug")?; - tracing::debug!("Updated bug with seed {}", seed); - } - } - - Ok(()) - } - - pub(crate) fn make_shrunk( + pub(crate) fn save_shrunk( &mut self, seed: u64, cli_options: &SimulatorCLI, shrunk_plan: InteractionPlan, error: Option, ) -> anyhow::Result<()> { - let mut bug = self.load_bug(seed)?; - bug.shrunk_plan = Some(shrunk_plan); + let path = self.path.clone(); + let bug = self + .get_or_load_bug(seed)? + .expect("bug should have been loaded"); bug.runs.push(BugRun { hash: Self::get_current_commit_hash()?, timestamp: SystemTime::now().into(), @@ -391,27 +265,18 @@ impl BugBase { cli_options: cli_options.clone(), shrunk: true, }); - self.bugs.insert(seed, Bug::Loaded(bug.clone())); + bug.shrunk_plan = Some(shrunk_plan); + // Save the bug to the bug base. - self.save_bug(seed) + bug.save_to_path(path) .with_context(|| "should be able to save shrunk bug")?; Ok(()) } - pub(crate) fn load_bugs(&mut self) -> anyhow::Result> { - let seeds = self.bugs.keys().copied().collect::>(); - - seeds - .iter() - .map(|seed| self.load_bug(*seed)) - .collect::, _>>() - } - pub(crate) fn list_bugs(&mut self) -> anyhow::Result<()> { let bugs = self.load_bugs()?; for bug in bugs { println!("seed: {}", bug.seed); - println!("plan: {}", bug.plan.stats()); println!("runs:"); println!(" ------------------"); for run in &bug.runs { diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index 97062dd2d..f0b9f7093 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -55,7 +55,7 @@ pub struct SimulatorCLI { help = "load plan from the bug base", conflicts_with = "seed" )] - pub load: Option, + pub load: Option, #[clap( short = 'w', long, From a4f0f2364d576c61497ef6f2ed8a20708a127cdb Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 12:10:52 -0300 Subject: [PATCH 03/16] disable Watch Mode until we can properly serialize interaction plan --- simulator/main.rs | 59 +---------------------------- simulator/model/interactions.rs | 67 --------------------------------- simulator/runner/cli.rs | 3 ++ 3 files changed, 5 insertions(+), 124 deletions(-) diff --git a/simulator/main.rs b/simulator/main.rs index 1012a34d3..9b44a91fb 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -1,8 +1,6 @@ #![allow(clippy::arc_with_non_send_sync)] use anyhow::anyhow; use clap::Parser; -use notify::event::{DataChange, ModifyKind}; -use notify::{EventKind, RecursiveMode, Watcher}; use rand::prelude::*; use runner::bugbase::BugBase; use runner::cli::{SimulatorCLI, SimulatorCommand}; @@ -15,7 +13,7 @@ use std::fs::OpenOptions; use std::io::{IsTerminal, Write}; use std::path::Path; use std::rc::Rc; -use std::sync::{Arc, Mutex, mpsc}; +use std::sync::{Arc, Mutex}; use tracing_subscriber::EnvFilter; use tracing_subscriber::field::MakeExt; use tracing_subscriber::fmt::format; @@ -133,8 +131,7 @@ fn testing_main(cli_opts: &mut SimulatorCLI, profile: &Profile) -> anyhow::Resul let (seed, mut env, plans) = setup_simulation(bugbase.as_mut(), cli_opts, profile); if cli_opts.watch { - watch_mode(env).unwrap(); - return Ok(()); + anyhow::bail!("watch mode is disabled for now"); } let paths = env.paths.clone(); @@ -158,58 +155,6 @@ fn testing_main(cli_opts: &mut SimulatorCLI, profile: &Profile) -> anyhow::Resul result } -fn watch_mode(env: SimulatorEnv) -> notify::Result<()> { - let (tx, rx) = mpsc::channel::>(); - println!("watching {:?}", env.get_plan_path()); - // Use recommended_watcher() to automatically select the best implementation - // for your platform. The `EventHandler` passed to this constructor can be a - // closure, a `std::sync::mpsc::Sender`, a `crossbeam_channel::Sender`, or - // another type the trait is implemented for. - let mut watcher = notify::recommended_watcher(tx)?; - - // Add a path to be watched. All files and directories at that path and - // below will be monitored for changes. - watcher.watch(&env.get_plan_path(), RecursiveMode::NonRecursive)?; - // Block forever, printing out events as they come in - let last_execution = Arc::new(Mutex::new(Execution::new(0, 0))); - for res in rx { - match res { - Ok(event) => { - if let EventKind::Modify(ModifyKind::Data(DataChange::Content)) = event.kind { - tracing::info!("plan file modified, rerunning simulation"); - let env = env.clone_without_connections(); - let last_execution_ = last_execution.clone(); - let result = SandboxedResult::from( - std::panic::catch_unwind(move || { - let mut env = env; - let plan_path = env.get_plan_path(); - let plan = InteractionPlan::compute_via_diff(&plan_path); - env.clear(); - - let env = Arc::new(Mutex::new(env.clone_without_connections())); - run_simulation_default(env, plan, last_execution_.clone()) - }), - last_execution.clone(), - ); - match result { - SandboxedResult::Correct => { - tracing::info!("simulation succeeded"); - println!("simulation succeeded"); - } - SandboxedResult::Panicked { error, .. } - | SandboxedResult::FoundBug { error, .. } => { - tracing::error!("simulation failed: '{}'", error); - } - } - } - } - Err(e) => println!("watch error: {e:?}"), - } - } - - Ok(()) -} - fn run_simulator( mut bugbase: Option<&mut BugBase>, cli_opts: &SimulatorCLI, diff --git a/simulator/model/interactions.rs b/simulator/model/interactions.rs index 204167e61..588cb0a29 100644 --- a/simulator/model/interactions.rs +++ b/simulator/model/interactions.rs @@ -1,7 +1,6 @@ use std::{ fmt::{Debug, Display}, ops::{Deref, DerefMut}, - path::Path, rc::Rc, sync::Arc, }; @@ -115,72 +114,6 @@ impl InteractionPlan { self.len = self.new_len(); } - /// Compute via diff computes a a plan from a given `.plan` file without the need to parse - /// sql. This is possible because there are two versions of the plan file, one that is human - /// readable and one that is serialized as JSON. Under watch mode, the users will be able to - /// delete interactions from the human readable file, and this function uses the JSON file as - /// a baseline to detect with interactions were deleted and constructs the plan from the - /// remaining interactions. - pub(crate) fn compute_via_diff(plan_path: &Path) -> impl InteractionPlanIterator { - let interactions = std::fs::read_to_string(plan_path).unwrap(); - let interactions = interactions.lines().collect::>(); - - let plan: InteractionPlan = serde_json::from_str( - std::fs::read_to_string(plan_path.with_extension("json")) - .unwrap() - .as_str(), - ) - .unwrap(); - - let mut plan = plan - .plan - .into_iter() - .map(|i| i.interactions()) - .collect::>(); - - let (mut i, mut j) = (0, 0); - - while i < interactions.len() && j < plan.len() { - if interactions[i].starts_with("-- begin") - || interactions[i].starts_with("-- end") - || interactions[i].is_empty() - { - i += 1; - continue; - } - - // interactions[i] is the i'th line in the human readable plan - // plan[j][k] is the k'th interaction in the j'th property - let mut k = 0; - - while k < plan[j].len() { - if i >= interactions.len() { - let _ = plan.split_off(j + 1); - let _ = plan[j].split_off(k); - break; - } - tracing::error!("Comparing '{}' with '{}'", interactions[i], plan[j][k]); - if interactions[i].contains(plan[j][k].to_string().as_str()) { - i += 1; - k += 1; - } else { - plan[j].remove(k); - panic!("Comparing '{}' with '{}'", interactions[i], plan[j][k]); - } - } - - if plan[j].is_empty() { - plan.remove(j); - } else { - j += 1; - } - } - let _ = plan.split_off(j); - PlanIterator { - iter: plan.into_iter().flatten(), - } - } - pub fn interactions_list(&self) -> Vec { self.plan .clone() diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index f0b9f7093..31d0a4077 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -200,6 +200,9 @@ pub enum SimulatorCommand { impl SimulatorCLI { pub fn validate(&mut self) -> anyhow::Result<()> { + if self.watch { + anyhow::bail!("watch mode is disabled for now"); + } if self.minimum_tests > self.maximum_tests { tracing::warn!( "minimum size '{}' is greater than '{}' maximum size, setting both to '{}'", From 2fe39d40bb8ba4905e8ed040fe7c8335be8ea07c Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 12:13:49 -0300 Subject: [PATCH 04/16] add Span and PropertyMetadata structs --- simulator/model/interactions.rs | 39 ++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/simulator/model/interactions.rs b/simulator/model/interactions.rs index 588cb0a29..ec03d0a45 100644 --- a/simulator/model/interactions.rs +++ b/simulator/model/interactions.rs @@ -12,7 +12,10 @@ use turso_core::{Connection, Result, StepResult}; use crate::{ generation::Shadow, - model::{Query, ResultSet, property::Property}, + model::{ + Query, ResultSet, + property::{Property, PropertyDiscriminants}, + }, runner::env::{ShadowTablesMut, SimConnection, SimulationType, SimulatorEnv}, }; @@ -495,6 +498,40 @@ impl Display for Fault { } } +#[derive(Debug, Clone, Copy)] +pub enum Span { + Start, + End, + // Both start and end + StartEnd, +} + +impl Span { + fn start(&self) -> bool { + matches!(self, Self::Start | Self::StartEnd) + } + + fn end(&self) -> bool { + matches!(self, Self::End | Self::StartEnd) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct PropertyMetadata { + pub property: PropertyDiscriminants, + // If the query is an extension query + pub extension: bool, +} + +impl PropertyMetadata { + pub fn new(property: &Property, extension: bool) -> PropertyMetadata { + Self { + property: property.into(), + extension, + } + } +} + #[derive(Debug, Clone)] pub struct Interaction { pub connection_index: usize, From c088a653e63011c99a5c28145485c181d446203a Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 12:14:25 -0300 Subject: [PATCH 05/16] move interaction stats to metrics --- simulator/model/metrics.rs | 78 +++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/simulator/model/metrics.rs b/simulator/model/metrics.rs index 217e47f6f..2c7e21efd 100644 --- a/simulator/model/metrics.rs +++ b/simulator/model/metrics.rs @@ -1,6 +1,14 @@ +use std::fmt::Display; + use sql_generation::generation::GenerationContext; -use crate::{model::interactions::InteractionStats, profiles::query::QueryProfile}; +use crate::{ + model::{ + Query, + interactions::{Interaction, InteractionType}, + }, + profiles::query::QueryProfile, +}; #[derive(Debug)] pub struct Remaining { @@ -97,3 +105,71 @@ impl Remaining { } } } + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct InteractionStats { + pub select_count: u32, + pub insert_count: u32, + pub delete_count: u32, + pub update_count: u32, + pub create_count: u32, + pub create_index_count: u32, + pub drop_count: u32, + pub begin_count: u32, + pub commit_count: u32, + pub rollback_count: u32, + pub alter_table_count: u32, + pub drop_index_count: u32, + pub pragma_count: u32, +} + +impl InteractionStats { + pub fn update(&mut self, interaction: &Interaction) { + match &interaction.interaction { + InteractionType::Query(query) + | InteractionType::FsyncQuery(query) + | InteractionType::FaultyQuery(query) => self.query_stat(query), + _ => {} + } + } + + fn query_stat(&mut self, q: &Query) { + match q { + Query::Select(_) => self.select_count += 1, + Query::Insert(_) => self.insert_count += 1, + Query::Delete(_) => self.delete_count += 1, + Query::Create(_) => self.create_count += 1, + Query::Drop(_) => self.drop_count += 1, + Query::Update(_) => self.update_count += 1, + Query::CreateIndex(_) => self.create_index_count += 1, + Query::Begin(_) => self.begin_count += 1, + Query::Commit(_) => self.commit_count += 1, + Query::Rollback(_) => self.rollback_count += 1, + Query::AlterTable(_) => self.alter_table_count += 1, + Query::DropIndex(_) => self.drop_index_count += 1, + Query::Placeholder => {} + Query::Pragma(_) => self.pragma_count += 1, + } + } +} + +impl Display for InteractionStats { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Read: {}, Insert: {}, Delete: {}, Update: {}, Create: {}, CreateIndex: {}, Drop: {}, Begin: {}, Commit: {}, Rollback: {}, Alter Table: {}, Drop Index: {}", + self.select_count, + self.insert_count, + self.delete_count, + self.update_count, + self.create_count, + self.create_index_count, + self.drop_count, + self.begin_count, + self.commit_count, + self.rollback_count, + self.alter_table_count, + self.drop_index_count, + ) + } +} From 157a5cf10adfd1c6dc37ae076ad45b817af50875 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 12:14:25 -0300 Subject: [PATCH 06/16] - add Interaction Builder to simplify code for building an interaction. Modify `generation/property.rs` to use the Builder - add additional metadata to `Interaction` to give more context for shrinking and iterating over interactions that originated from the same interaction. - add Iterator like utilities for `InteractionPlan` to facilitate iterating over interactions that came from the same property: --- Cargo.lock | 32 ++ simulator/Cargo.toml | 1 + simulator/generation/plan.rs | 2 +- simulator/generation/property.rs | 195 ++++++----- simulator/model/interactions.rs | 576 +++++++++++++++---------------- simulator/model/property.rs | 52 +-- simulator/runner/execution.rs | 12 +- 7 files changed, 471 insertions(+), 399 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50f3f8047..8a936af6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,6 +1225,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.100", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.100", +] + [[package]] name = "derive_more" version = "2.0.1" @@ -2617,6 +2648,7 @@ dependencies = [ "bitmaps", "chrono", "clap", + "derive_builder", "dirs 6.0.0", "either", "garde", diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index 7fd5dbeff..43b4cceb2 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -47,3 +47,4 @@ similar = { workspace = true } similar-asserts = { workspace = true } bitmaps = { workspace = true } bitflags.workspace = true +derive_builder = "0.20.2" diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index ff7f570d6..97b842552 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -181,7 +181,7 @@ impl<'a, R: rand::Rng> PlanGenerator<'a, R> { let remaining_ = Remaining::new( env.opts.max_interactions, &env.profile.query, - &stats, + stats, env.profile.experimental_mvcc, &conn_ctx, ); diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 75343b13b..1db2cf0a7 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -5,6 +5,8 @@ //! we can generate queries that reference tables that do not exist. This is not a correctness issue, but more of //! an optimization issue that is good to point out for the future +use std::num::NonZeroUsize; + use rand::distr::{Distribution, weighted::WeightedIndex}; use sql_generation::{ generation::{Arbitrary, ArbitraryFrom, GenerationContext, pick, pick_index}, @@ -26,10 +28,12 @@ use turso_parser::ast::{self, Distinctness}; use crate::{ common::print_diff, - generation::{Shadow as _, WeightedDistribution, query::QueryDistribution}, + generation::{Shadow, WeightedDistribution, query::QueryDistribution}, model::{ Query, QueryCapabilities, QueryDiscriminants, ResultSet, - interactions::{Assertion, Interaction, InteractionType}, + interactions::{ + Assertion, Interaction, InteractionBuilder, InteractionType, PropertyMetadata, Span, + }, metrics::Remaining, property::{InteractiveQueryInfo, Property, PropertyDiscriminants}, }, @@ -231,7 +235,7 @@ impl Property { Property::SelectLimit { .. } | Property::SelectSelectOptimizer { .. } | Property::WhereTrueFalseNull { .. } - | Property::UNIONAllPreservesCardinality { .. } + | Property::UnionAllPreservesCardinality { .. } | Property::ReadYourUpdatesBack { .. } | Property::TableHasExpectedContent { .. } | Property::AllTableHaveExpectedContent { .. } => { @@ -243,8 +247,12 @@ impl Property { /// interactions construct a list of interactions, which is an executable representation of the property. /// the requirement of property -> vec conversion emerges from the need to serialize the property, /// and `interaction` cannot be serialized directly. - pub(crate) fn interactions(&self, connection_index: usize) -> Vec { - match self { + pub(crate) fn interactions( + &self, + connection_index: usize, + id: NonZeroUsize, + ) -> Vec { + let mut interactions: Vec = match self { Property::AllTableHaveExpectedContent { tables } => { assert_all_table_values(tables, connection_index).collect() } @@ -304,9 +312,9 @@ impl Property { )); vec![ - Interaction::new(connection_index, assumption), - Interaction::new(connection_index, select_interaction), - Interaction::new(connection_index, assertion), + InteractionBuilder::with_interaction(assumption), + InteractionBuilder::with_interaction(select_interaction), + InteractionBuilder::with_interaction(assertion), ] } Property::ReadYourUpdatesBack { update, select } => { @@ -368,10 +376,10 @@ impl Property { )); vec![ - Interaction::new(connection_index, assumption), - Interaction::new(connection_index, update_interaction), - Interaction::new(connection_index, select_interaction), - Interaction::new(connection_index, assertion), + InteractionBuilder::with_interaction(assumption), + InteractionBuilder::with_interaction(update_interaction), + InteractionBuilder::with_interaction(select_interaction), + InteractionBuilder::with_interaction(assertion), ] } Property::InsertValuesSelect { @@ -448,22 +456,20 @@ impl Property { )); let mut interactions = Vec::new(); - interactions.push(Interaction::new(connection_index, assumption)); - interactions.push(Interaction::new( - connection_index, + interactions.push(InteractionBuilder::with_interaction(assumption)); + interactions.push(InteractionBuilder::with_interaction( InteractionType::Query(Query::Insert(insert.clone())), )); - interactions.extend( - queries - .clone() - .into_iter() - .map(|q| Interaction::new(connection_index, InteractionType::Query(q))), - ); - interactions.push(Interaction::new( - connection_index, + interactions.extend(queries.clone().into_iter().map(|q| { + let mut builder = + InteractionBuilder::with_interaction(InteractionType::Query(q)); + builder.property_meta(PropertyMetadata::new(self, true)); + builder + })); + interactions.push(InteractionBuilder::with_interaction( InteractionType::Query(Query::Select(select.clone())), )); - interactions.push(Interaction::new(connection_index, assertion)); + interactions.push(InteractionBuilder::with_interaction(assertion)); interactions } @@ -505,16 +511,20 @@ impl Property { }) ); let mut interactions = Vec::new(); - interactions.push(Interaction::new(connection_index, assumption)); - interactions.push(Interaction::new(connection_index, cq1)); - interactions.extend( - queries - .clone() - .into_iter() - .map(|q| Interaction::new(connection_index, InteractionType::Query(q))), - ); - interactions.push(Interaction::new_ignore_error(connection_index, cq2)); - interactions.push(Interaction::new(connection_index, assertion)); + interactions.push(InteractionBuilder::with_interaction(assumption)); + interactions.push(InteractionBuilder::with_interaction(cq1)); + interactions.extend(queries.clone().into_iter().map(|q| { + let mut builder = + InteractionBuilder::with_interaction(InteractionType::Query(q)); + builder.property_meta(PropertyMetadata::new(self, true)); + builder + })); + interactions.push({ + let mut builder = InteractionBuilder::with_interaction(cq2); + builder.ignore_error(true); + builder + }); + interactions.push(InteractionBuilder::with_interaction(assertion)); interactions } @@ -574,12 +584,11 @@ impl Property { )); vec![ - Interaction::new(connection_index, assumption), - Interaction::new( - connection_index, - InteractionType::Query(Query::Select(select.clone())), - ), - Interaction::new(connection_index, assertion), + InteractionBuilder::with_interaction(assumption), + InteractionBuilder::with_interaction(InteractionType::Query(Query::Select( + select.clone(), + ))), + InteractionBuilder::with_interaction(assertion), ] } Property::DeleteSelect { @@ -643,16 +652,16 @@ impl Property { )); let mut interactions = Vec::new(); - interactions.push(Interaction::new(connection_index, assumption)); - interactions.push(Interaction::new(connection_index, delete)); - interactions.extend( - queries - .clone() - .into_iter() - .map(|q| Interaction::new(connection_index, InteractionType::Query(q))), - ); - interactions.push(Interaction::new(connection_index, select)); - interactions.push(Interaction::new(connection_index, assertion)); + interactions.push(InteractionBuilder::with_interaction(assumption)); + interactions.push(InteractionBuilder::with_interaction(delete)); + interactions.extend(queries.clone().into_iter().map(|q| { + let mut builder = + InteractionBuilder::with_interaction(InteractionType::Query(q)); + builder.property_meta(PropertyMetadata::new(self, true)); + builder + })); + interactions.push(InteractionBuilder::with_interaction(select)); + interactions.push(InteractionBuilder::with_interaction(assertion)); interactions } @@ -715,16 +724,20 @@ impl Property { let mut interactions = Vec::new(); - interactions.push(Interaction::new(connection_index, assumption)); - interactions.push(Interaction::new(connection_index, drop)); - interactions.extend( - queries - .clone() - .into_iter() - .map(|q| Interaction::new(connection_index, InteractionType::Query(q))), - ); - interactions.push(Interaction::new_ignore_error(connection_index, select)); - interactions.push(Interaction::new(connection_index, assertion)); + interactions.push(InteractionBuilder::with_interaction(assumption)); + interactions.push(InteractionBuilder::with_interaction(drop)); + interactions.extend(queries.clone().into_iter().map(|q| { + let mut builder = + InteractionBuilder::with_interaction(InteractionType::Query(q)); + builder.property_meta(PropertyMetadata::new(self, true)); + builder + })); + interactions.push({ + let mut builder = InteractionBuilder::with_interaction(select); + builder.ignore_error(true); + builder + }); + interactions.push(InteractionBuilder::with_interaction(assertion)); interactions } @@ -811,15 +824,14 @@ impl Property { )); vec![ - Interaction::new(connection_index, assumption), - Interaction::new(connection_index, select1), - Interaction::new(connection_index, select2), - Interaction::new(connection_index, assertion), + InteractionBuilder::with_interaction(assumption), + InteractionBuilder::with_interaction(select1), + InteractionBuilder::with_interaction(select2), + InteractionBuilder::with_interaction(assertion), ] } Property::FsyncNoWait { query } => { - vec![Interaction::new( - connection_index, + vec![InteractionBuilder::with_interaction( InteractionType::FsyncQuery(query.clone()), )] } @@ -855,7 +867,7 @@ impl Property { InteractionType::Assertion(assert), ] .into_iter() - .map(|i| Interaction::new(connection_index, i)) + .map(InteractionBuilder::with_interaction) .collect() } Property::WhereTrueFalseNull { select, predicate } => { @@ -1010,13 +1022,13 @@ impl Property { )); vec![ - Interaction::new(connection_index, assumption), - Interaction::new(connection_index, select), - Interaction::new(connection_index, select_tlp), - Interaction::new(connection_index, assertion), + InteractionBuilder::with_interaction(assumption), + InteractionBuilder::with_interaction(select), + InteractionBuilder::with_interaction(select_tlp), + InteractionBuilder::with_interaction(assertion), ] } - Property::UNIONAllPreservesCardinality { + Property::UnionAllPreservesCardinality { select, where_clause, } => { @@ -1062,21 +1074,42 @@ impl Property { } }, )), - ].into_iter().map(|i| Interaction::new(connection_index, i)).collect() + ].into_iter().map(InteractionBuilder::with_interaction).collect() } Property::Queries { queries } => queries .clone() .into_iter() - .map(|query| Interaction::new(connection_index, InteractionType::Query(query))) + .map(|query| InteractionBuilder::with_interaction(InteractionType::Query(query))) .collect(), - } + }; + + assert!(!interactions.is_empty()); + + // Add a span to the interactions that matter + if interactions.len() == 1 { + interactions.first_mut().unwrap().span(Span::StartEnd); + } else { + interactions.first_mut().unwrap().span(Span::Start); + interactions.last_mut().unwrap().span(Span::End); + }; + + interactions + .into_iter() + .map(|mut builder| { + if !builder.has_property_meta() { + builder.property_meta(PropertyMetadata::new(self, false)); + } + builder.connection_index(connection_index).id(id); + builder.build().unwrap() + }) + .collect() } } fn assert_all_table_values( tables: &[String], connection_index: usize, -) -> impl Iterator + use<'_> { +) -> impl Iterator + use<'_> { tables.iter().flat_map(move |table| { let select = InteractionType::Query(Query::Select(Select::simple( table.clone(), @@ -1131,7 +1164,7 @@ fn assert_all_table_values( } } })); - [select, assertion].into_iter().map(move |i| Interaction::new(connection_index, i)) + [select, assertion].into_iter().map(InteractionBuilder::with_interaction) }) } @@ -1411,7 +1444,7 @@ fn property_union_all_preserves_cardinality( Distinctness::All, ); - Property::UNIONAllPreservesCardinality { + Property::UnionAllPreservesCardinality { select, where_clause: p2, } @@ -1460,7 +1493,7 @@ impl PropertyDiscriminants { PropertyDiscriminants::DropSelect => property_drop_select, PropertyDiscriminants::SelectSelectOptimizer => property_select_select_optimizer, PropertyDiscriminants::WhereTrueFalseNull => property_where_true_false_null, - PropertyDiscriminants::UNIONAllPreservesCardinality => { + PropertyDiscriminants::UnionAllPreservesCardinality => { property_union_all_preserves_cardinality } PropertyDiscriminants::FsyncNoWait => property_fsync_no_wait, @@ -1543,7 +1576,7 @@ impl PropertyDiscriminants { 0 } } - PropertyDiscriminants::UNIONAllPreservesCardinality => { + PropertyDiscriminants::UnionAllPreservesCardinality => { if opts.indexes && !env.opts.disable_union_all_preserves_cardinality && !ctx.tables().is_empty() @@ -1607,7 +1640,7 @@ impl PropertyDiscriminants { } PropertyDiscriminants::SelectSelectOptimizer => QueryCapabilities::SELECT, PropertyDiscriminants::WhereTrueFalseNull => QueryCapabilities::SELECT, - PropertyDiscriminants::UNIONAllPreservesCardinality => QueryCapabilities::SELECT, + PropertyDiscriminants::UnionAllPreservesCardinality => QueryCapabilities::SELECT, PropertyDiscriminants::FsyncNoWait => QueryCapabilities::all(), PropertyDiscriminants::FaultyQuery => QueryCapabilities::all(), PropertyDiscriminants::Queries => panic!("queries property should not be generated"), diff --git a/simulator/model/interactions.rs b/simulator/model/interactions.rs index ec03d0a45..580ecda1c 100644 --- a/simulator/model/interactions.rs +++ b/simulator/model/interactions.rs @@ -1,11 +1,15 @@ use std::{ fmt::{Debug, Display}, - ops::{Deref, DerefMut}, + marker::PhantomData, + num::NonZeroUsize, + ops::{Deref, DerefMut, Range}, + panic::RefUnwindSafe, rc::Rc, sync::Arc, }; -use indexmap::IndexSet; +use either::Either; +use itertools::Itertools; use serde::{Deserialize, Serialize}; use sql_generation::model::table::SimValue; use turso_core::{Connection, Result, StepResult}; @@ -14,224 +18,262 @@ use crate::{ generation::Shadow, model::{ Query, ResultSet, + metrics::InteractionStats, property::{Property, PropertyDiscriminants}, }, runner::env::{ShadowTablesMut, SimConnection, SimulationType, SimulatorEnv}, }; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] pub(crate) struct InteractionPlan { - plan: Vec, + plan: Vec, + stats: InteractionStats, + // In the future, this should probably be a stack of interactions + // so we can have nested properties + last_interactions: Option, pub mvcc: bool, - // Len should not count transactions statements, just so we can generate more meaningful interactions per run - len: usize, + + /// Counts [Interactions]. Should not count transactions statements, just so we can generate more meaningful interactions per run + /// This field is only necessary and valid when generating interactions. For static iteration, we do not care about this field + len_properties: usize, + next_interaction_id: NonZeroUsize, } impl InteractionPlan { pub(crate) fn new(mvcc: bool) -> Self { Self { plan: Vec::new(), + stats: InteractionStats::default(), + last_interactions: None, mvcc, - len: 0, + len_properties: 0, + next_interaction_id: NonZeroUsize::new(1).unwrap(), } } - pub fn new_with(plan: Vec, mvcc: bool) -> Self { - let len = plan - .iter() - .filter(|interaction| !interaction.ignore()) - .count(); - Self { plan, mvcc, len } - } - - #[inline] - fn new_len(&self) -> usize { - self.plan - .iter() - .filter(|interaction| !interaction.ignore()) - .count() - } - - /// Length of interactions that are not transaction statements + /// Count of interactions #[inline] pub fn len(&self) -> usize { - self.len + self.plan.len() } + /// Count of properties #[inline] - pub fn plan(&self) -> &[Interactions] { - &self.plan + pub fn len_properties(&self) -> usize { + self.len_properties } - pub fn push(&mut self, interactions: Interactions) { + pub fn next_property_id(&mut self) -> NonZeroUsize { + let id = self.next_interaction_id; + self.next_interaction_id = self + .next_interaction_id + .checked_add(1) + .expect("Generated too many interactions, that overflowed ID generation"); + id + } + + pub fn last_interactions(&self) -> Option<&Interactions> { + self.last_interactions.as_ref() + } + + pub fn push_interactions(&mut self, interactions: Interactions) { if !interactions.ignore() { - self.len += 1; + self.len_properties += 1; } - self.plan.push(interactions); + self.last_interactions = Some(interactions); } - pub fn remove(&mut self, index: usize) -> Interactions { - let interactions = self.plan.remove(index); - if !interactions.ignore() { - self.len -= 1; - } - interactions + pub fn push(&mut self, interaction: Interaction) { + self.plan.push(interaction); } + /// Finds the range of interactions that are contained between the start and end spans for a given ID. + pub fn find_interactions_range(&self, id: NonZeroUsize) -> Range { + let interactions = self.interactions_list(); + let idx = interactions + .binary_search_by_key(&id, |interaction| interaction.id()) + .map_err(|_| format!("Interaction containing id `{id}` should be present")) + .unwrap(); + let interaction = &interactions[idx]; + + let backward = || -> usize { + interactions + .iter() + .rev() + .skip(interactions.len() - idx) + .position(|interaction| { + interaction.id() == id + && interaction + .span + .is_some_and(|span| matches!(span, Span::Start)) + }) + .map(|idx| (interactions.len() - 1) - idx - 1) + .expect("A start span should have been emitted") + }; + + let forward = || -> usize { + interactions + .iter() + .skip(idx + 1) + .position(|interaction| interaction.id() != id) + .map(|idx| idx - 1) + .unwrap_or(interactions.len() - 1) + // It can happen we do not have an end Span as we can fail in the middle of a property + }; + + if let Some(span) = interaction.span { + match span { + Span::Start => { + // go forward and find the end span + let end_idx = forward(); + idx..end_idx + 1 + } + Span::End => { + // go backward and find the start span + let start_idx = backward(); + start_idx..idx + 1 + } + Span::StartEnd => idx..idx + 1, + } + } else { + // go backward and find the start span + let start_idx = backward(); + // go forward and find the end span + let end_idx = forward(); + start_idx..end_idx + 1 + } + } + + /// Truncates up to a particular interaction pub fn truncate(&mut self, len: usize) { self.plan.truncate(len); - self.len = self.new_len(); } - pub fn retain_mut(&mut self, mut f: F) + /// Used to remove a particular [Interactions] + pub fn remove_property(&mut self, id: NonZeroUsize) { + let range = self.find_interactions_range(id); + // Consume the drain iterator just to be sure + for _interaction in self.plan.drain(range) {} + } + + pub fn retain_mut(&mut self, f: F) where - F: FnMut(&mut Interactions) -> bool, + F: FnMut(&mut Interaction) -> bool, { - let f = |t: &mut Interactions| { - let ignore = t.ignore(); - let retain = f(t); - // removed an interaction that was not previously ignored - if !retain && !ignore { - self.len -= 1; - } - retain - }; self.plan.retain_mut(f); } - #[expect(dead_code)] - pub fn retain(&mut self, mut f: F) - where - F: FnMut(&Interactions) -> bool, - { - let f = |t: &Interactions| { - let ignore = t.ignore(); - let retain = f(t); - // removed an interaction that was not previously ignored - if !retain && !ignore { - self.len -= 1; - } - retain - }; - self.plan.retain(f); - self.len = self.new_len(); + #[inline] + pub fn interactions_list(&self) -> &[Interaction] { + &self.plan } - pub fn interactions_list(&self) -> Vec { - self.plan - .clone() - .into_iter() - .flat_map(|interactions| interactions.interactions().into_iter()) - .collect() - } - - pub fn interactions_list_with_secondary_index(&self) -> Vec<(usize, Interaction)> { - self.plan - .clone() - .into_iter() - .enumerate() - .flat_map(|(idx, interactions)| { - interactions - .interactions() - .into_iter() - .map(move |interaction| (idx, interaction)) - }) - .collect() - } - - pub(crate) fn stats(&self) -> InteractionStats { - let mut stats = InteractionStats::default(); - - fn query_stat(q: &Query, stats: &mut InteractionStats) { - match q { - Query::Select(_) => stats.select_count += 1, - Query::Insert(_) => stats.insert_count += 1, - Query::Delete(_) => stats.delete_count += 1, - Query::Create(_) => stats.create_count += 1, - Query::Drop(_) => stats.drop_count += 1, - Query::Update(_) => stats.update_count += 1, - Query::CreateIndex(_) => stats.create_index_count += 1, - Query::Begin(_) => stats.begin_count += 1, - Query::Commit(_) => stats.commit_count += 1, - Query::Rollback(_) => stats.rollback_count += 1, - Query::AlterTable(_) => stats.alter_table_count += 1, - Query::DropIndex(_) => stats.drop_index_count += 1, - Query::Placeholder => {} - Query::Pragma(_) => stats.pragma_count += 1, - } - } - for interactions in &self.plan { - match &interactions.interactions { - InteractionsType::Property(property) => { - if matches!(property, Property::AllTableHaveExpectedContent { .. }) { - // Skip Property::AllTableHaveExpectedContent when counting stats - // this allows us to generate more relevant interactions as we count less Select's to the Stats - continue; - } - for interaction in &property.interactions(interactions.connection_index) { - if let InteractionType::Query(query) = &interaction.interaction { - query_stat(query, &mut stats); - } - } - } - InteractionsType::Query(query) => { - query_stat(query, &mut stats); - } - InteractionsType::Fault(_) => {} - } + pub fn iter_properties( + &self, + ) -> IterProperty< + std::iter::Peekable>>, + Forward, + > { + IterProperty { + iter: self.interactions_list().iter().enumerate().peekable(), + _direction: PhantomData, } + } - stats + pub fn rev_iter_properties( + &self, + ) -> IterProperty< + std::iter::Peekable< + std::iter::Enumerate>>, + >, + Backward, + > { + IterProperty { + iter: self.interactions_list().iter().rev().enumerate().peekable(), + _direction: PhantomData, + } + } + + pub fn stats(&self) -> &InteractionStats { + &self.stats + } + + pub fn stats_mut(&mut self) -> &mut InteractionStats { + &mut self.stats } pub fn static_iterator(&self) -> impl InteractionPlanIterator { PlanIterator { - iter: self.interactions_list().into_iter(), + iter: self.interactions_list().to_vec().into_iter(), } } } -impl Deref for InteractionPlan { - type Target = Vec; +pub struct Forward; +pub struct Backward; - fn deref(&self) -> &Self::Target { - &self.plan +pub struct IterProperty { + iter: I, + _direction: PhantomData, +} + +impl<'a, I> IterProperty +where + I: Iterator + itertools::PeekingNext + std::fmt::Debug, +{ + pub fn next_property(&mut self) -> Option> { + let (idx, interaction) = self.iter.next()?; + let id = interaction.id(); + // get interactions from a particular property + let span = interaction + .span + .expect("we should loop on interactions that have a span"); + + let first = std::iter::once((idx, interaction)); + + let property_interactions = match span { + Span::Start => Either::Left( + first.chain( + self.iter + .peeking_take_while(move |(_idx, interaction)| interaction.id() == id), + ), + ), + Span::End => panic!("we should always be at the start of an interaction"), + Span::StartEnd => Either::Right(first), + }; + + Some(property_interactions) } } -impl DerefMut for InteractionPlan { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.plan - } -} +impl<'a, I> IterProperty +where + I: Iterator + + DoubleEndedIterator + + itertools::PeekingNext + + std::fmt::Debug, +{ + pub fn next_property(&mut self) -> Option> { + let (idx, interaction) = self.iter.next()?; + let id = interaction.id(); + // get interactions from a particular property + let span = interaction + .span + .expect("we should loop on interactions that have a span"); -impl IntoIterator for InteractionPlan { - type Item = Interactions; + let first = std::iter::once((idx, interaction)); - type IntoIter = as IntoIterator>::IntoIter; + let property_interactions = match span { + Span::Start => panic!("we should always be at the end of an interaction"), + Span::End => Either::Left( + self.iter + .peeking_take_while(move |(_idx, interaction)| interaction.id() == id) + .chain(first), + ), + Span::StartEnd => Either::Right(first), + }; - fn into_iter(self) -> Self::IntoIter { - self.plan.into_iter() - } -} - -impl<'a> IntoIterator for &'a InteractionPlan { - type Item = &'a Interactions; - - type IntoIter = <&'a Vec as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.plan.iter() - } -} - -impl<'a> IntoIterator for &'a mut InteractionPlan { - type Item = &'a mut Interactions; - - type IntoIter = <&'a mut Vec as IntoIterator>::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.plan.iter_mut() + Some(property_interactions.into_iter()) } } @@ -284,13 +326,6 @@ impl Interactions { } } - pub fn get_extensional_queries(&mut self) -> Option<&mut Vec> { - match &mut self.interactions { - InteractionsType::Property(property) => property.get_extensional_queries(), - InteractionsType::Query(..) | InteractionsType::Fault(..) => None, - } - } - /// Whether the interaction needs to check the database tables pub fn check_tables(&self) -> bool { match &self.interactions { @@ -341,75 +376,28 @@ impl InteractionsType { } } -impl Interactions { - pub(crate) fn interactions(&self) -> Vec { - match &self.interactions { - InteractionsType::Property(property) => property.interactions(self.connection_index), - InteractionsType::Query(query) => vec![Interaction::new( - self.connection_index, - InteractionType::Query(query.clone()), - )], - InteractionsType::Fault(fault) => vec![Interaction::new( - self.connection_index, - InteractionType::Fault(*fault), - )], - } - } - - pub(crate) fn dependencies(&self) -> IndexSet { - match &self.interactions { - InteractionsType::Property(property) => property - .interactions(self.connection_index) - .iter() - .fold(IndexSet::new(), |mut acc, i| match &i.interaction { - InteractionType::Query(q) => { - acc.extend(q.dependencies()); - acc - } - _ => acc, - }), - InteractionsType::Query(query) => query.dependencies(), - InteractionsType::Fault(_) => IndexSet::new(), - } - } - - pub(crate) fn uses(&self) -> Vec { - match &self.interactions { - InteractionsType::Property(property) => property - .interactions(self.connection_index) - .iter() - .fold(vec![], |mut acc, i| match &i.interaction { - InteractionType::Query(q) => { - acc.extend(q.uses()); - acc - } - _ => acc, - }), - InteractionsType::Query(query) => query.uses(), - InteractionsType::Fault(_) => vec![], - } - } -} - -// FIXME: for the sql display come back and add connection index as a comment impl Display for InteractionPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for interactions in &self.plan { - match &interactions.interactions { - InteractionsType::Property(property) => { - let name = property.name(); - writeln!(f, "-- begin testing '{name}'")?; - for interaction in property.interactions(interactions.connection_index) { - writeln!(f, "\t{interaction}")?; - } - writeln!(f, "-- end testing '{name}'")?; - } - InteractionsType::Fault(fault) => { - writeln!(f, "-- FAULT '{fault}'")?; - } - InteractionsType::Query(query) => { - writeln!(f, "{query}; -- {}", interactions.connection_index)?; - } + const PAD: usize = 4; + let mut indentation_level = 0; + for interaction in &self.plan { + if let Some(name) = interaction.property_meta.map(|p| p.property.name()) + && interaction.span.is_some_and(|span| span.start()) + { + indentation_level += 1; + writeln!(f, "-- begin testing '{name}'")?; + } + + if indentation_level > 0 { + let padding = " ".repeat(indentation_level * PAD); + f.pad(&padding)?; + } + writeln!(f, "{interaction}")?; + if let Some(name) = interaction.property_meta.map(|p| p.property.name()) + && interaction.span.is_some_and(|span| span.end()) + { + indentation_level -= 1; + writeln!(f, "-- end testing '{name}'")?; } } @@ -417,45 +405,8 @@ impl Display for InteractionPlan { } } -#[derive(Debug, Clone, Copy, Default)] -pub(crate) struct InteractionStats { - pub select_count: u32, - pub insert_count: u32, - pub delete_count: u32, - pub update_count: u32, - pub create_count: u32, - pub create_index_count: u32, - pub drop_count: u32, - pub begin_count: u32, - pub commit_count: u32, - pub rollback_count: u32, - pub alter_table_count: u32, - pub drop_index_count: u32, - pub pragma_count: u32, -} - -impl Display for InteractionStats { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Read: {}, Insert: {}, Delete: {}, Update: {}, Create: {}, CreateIndex: {}, Drop: {}, Begin: {}, Commit: {}, Rollback: {}, Alter Table: {}, Drop Index: {}", - self.select_count, - self.insert_count, - self.delete_count, - self.update_count, - self.create_count, - self.create_index_count, - self.drop_count, - self.begin_count, - self.commit_count, - self.rollback_count, - self.alter_table_count, - self.drop_index_count, - ) - } -} - -type AssertionFunc = dyn Fn(&Vec, &mut SimulatorEnv) -> Result>; +type AssertionFunc = + dyn Fn(&Vec, &mut SimulatorEnv) -> Result> + RefUnwindSafe; #[derive(Clone)] pub struct Assertion { @@ -474,7 +425,9 @@ impl Debug for Assertion { impl Assertion { pub fn new(name: String, func: F) -> Self where - F: Fn(&Vec, &mut SimulatorEnv) -> Result> + 'static, + F: Fn(&Vec, &mut SimulatorEnv) -> Result> + + 'static + + RefUnwindSafe, { Self { func: Rc::new(func), @@ -532,11 +485,61 @@ impl PropertyMetadata { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, derive_builder::Builder)] +#[builder(build_fn(validate = "Self::validate"))] pub struct Interaction { pub connection_index: usize, pub interaction: InteractionType, + #[builder(default)] pub ignore_error: bool, + #[builder(setter(strip_option), default)] + pub property_meta: Option, + #[builder(setter(strip_option), default)] + pub span: Option, + /// 0 id means the ID was not set + id: NonZeroUsize, +} + +impl InteractionBuilder { + pub fn from_interaction(interaction: &Interaction) -> Self { + let mut builder = Self::default(); + builder + .connection_index(interaction.connection_index) + .id(interaction.id()) + .ignore_error(interaction.ignore_error) + .interaction(interaction.interaction.clone()); + if let Some(property_meta) = interaction.property_meta { + builder.property_meta(property_meta); + } + if let Some(span) = interaction.span { + builder.span(span); + } + builder + } + + pub fn with_interaction(interaction: InteractionType) -> Self { + let mut builder = Self::default(); + builder.interaction(interaction); + builder + } + + /// Checks to see if the property metadata was already set + pub fn has_property_meta(&self) -> bool { + self.property_meta.is_some() + } + + fn validate(&self) -> Result<(), InteractionBuilderError> { + // Cannot have span and property_meta.extension being true at the same time + if let Some(property_meta) = self.property_meta.flatten() + && property_meta.extension + && self.span.flatten().is_some() + { + return Err(InteractionBuilderError::ValidationError( + "cannot have a span set with an extension query".to_string(), + )); + } + Ok(()) + } } impl Deref for Interaction { @@ -554,19 +557,16 @@ impl DerefMut for Interaction { } impl Interaction { - pub fn new(connection_index: usize, interaction: InteractionType) -> Self { - Self { - connection_index, - interaction, - ignore_error: false, - } + pub fn id(&self) -> NonZeroUsize { + self.id } - pub fn new_ignore_error(connection_index: usize, interaction: InteractionType) -> Self { - Self { - connection_index, - interaction, - ignore_error: true, + pub fn uses(&self) -> Vec { + match &self.interaction { + InteractionType::Query(query) + | InteractionType::FsyncQuery(query) + | InteractionType::FaultyQuery(query) => query.uses(), + _ => vec![], } } } diff --git a/simulator/model/property.rs b/simulator/model/property.rs index 7dff89577..e11949ab3 100644 --- a/simulator/model/property.rs +++ b/simulator/model/property.rs @@ -5,8 +5,9 @@ use crate::model::Query; /// Properties are representations of executable specifications /// about the database behavior. -#[derive(Debug, Clone, Serialize, Deserialize, strum::EnumDiscriminants)] -#[strum_discriminants(derive(strum::EnumIter))] +#[derive(Debug, Clone, Serialize, Deserialize, strum::EnumDiscriminants, strum::IntoStaticStr)] +#[strum_discriminants(derive(strum::EnumIter, strum::IntoStaticStr))] +#[strum(serialize_all = "Train-Case")] pub enum Property { /// Insert-Select is a property in which the inserted row /// must be in the resulting rows of a select query that has a @@ -161,7 +162,7 @@ pub enum Property { /// should return the same number of rows as `SELECT FROM WHERE `. /// > The property is succesfull when the UNION ALL of 2 select queries returns the same number of rows /// > as the sum of the two select queries. - UNIONAllPreservesCardinality { + UnionAllPreservesCardinality { select: Select, where_clause: Predicate, }, @@ -192,25 +193,6 @@ pub struct InteractiveQueryInfo { } impl Property { - pub(crate) fn name(&self) -> &str { - match self { - Property::InsertValuesSelect { .. } => "Insert-Values-Select", - Property::ReadYourUpdatesBack { .. } => "Read-Your-Updates-Back", - Property::TableHasExpectedContent { .. } => "Table-Has-Expected-Content", - Property::AllTableHaveExpectedContent { .. } => "All-Tables-Have-Expected-Content", - Property::DoubleCreateFailure { .. } => "Double-Create-Failure", - Property::SelectLimit { .. } => "Select-Limit", - Property::DeleteSelect { .. } => "Delete-Select", - Property::DropSelect { .. } => "Drop-Select", - Property::SelectSelectOptimizer { .. } => "Select-Select-Optimizer", - Property::WhereTrueFalseNull { .. } => "Where-True-False-Null", - Property::FsyncNoWait { .. } => "FsyncNoWait", - Property::FaultyQuery { .. } => "FaultyQuery", - Property::UNIONAllPreservesCardinality { .. } => "UNION-All-Preserves-Cardinality", - Property::Queries { .. } => "Queries", - } - } - /// Property Does some sort of fault injection pub fn check_tables(&self) -> bool { matches!( @@ -219,6 +201,17 @@ impl Property { ) } + pub fn has_extensional_queries(&self) -> bool { + matches!( + self, + Property::InsertValuesSelect { .. } + | Property::DoubleCreateFailure { .. } + | Property::DeleteSelect { .. } + | Property::DropSelect { .. } + | Property::Queries { .. } + ) + } + pub fn get_extensional_queries(&mut self) -> Option<&mut Vec> { match self { Property::InsertValuesSelect { queries, .. } @@ -230,10 +223,23 @@ impl Property { Property::SelectLimit { .. } | Property::SelectSelectOptimizer { .. } | Property::WhereTrueFalseNull { .. } - | Property::UNIONAllPreservesCardinality { .. } + | Property::UnionAllPreservesCardinality { .. } | Property::ReadYourUpdatesBack { .. } | Property::TableHasExpectedContent { .. } | Property::AllTableHaveExpectedContent { .. } => None, } } } + +impl PropertyDiscriminants { + pub fn name(&self) -> &'static str { + self.into() + } + + pub fn check_tables(&self) -> bool { + matches!( + self, + Self::AllTableHaveExpectedContent | Self::TableHasExpectedContent + ) + } +} diff --git a/simulator/runner/execution.rs b/simulator/runner/execution.rs index 148a996fb..ca59237e9 100644 --- a/simulator/runner/execution.rs +++ b/simulator/runner/execution.rs @@ -9,8 +9,8 @@ use crate::{ model::{ Query, ResultSet, interactions::{ - ConnectionState, Interaction, InteractionPlanIterator, InteractionPlanState, - InteractionType, + ConnectionState, Interaction, InteractionBuilder, InteractionPlanIterator, + InteractionPlanState, InteractionType, }, }, }; @@ -199,10 +199,10 @@ pub fn execute_interaction_turso( stack.push(results); - let query_interaction = Interaction::new( - interaction.connection_index, - InteractionType::Query(query.clone()), - ); + let query_interaction = InteractionBuilder::from_interaction(interaction) + .interaction(InteractionType::Query(query.clone())) + .build() + .unwrap(); execute_interaction(env, &query_interaction, stack)?; } From a21f7675dd2b0869e01353b1a2e88da089ef9a6e Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 12:14:25 -0300 Subject: [PATCH 07/16] - update interaction stats on demand instead of reading the entire plan to calculate metrics per generation step - simplify generation as we now only store `Interaction`. So now we can funnel most of the logic for interaction generation, metric update, and Interaction append in the `PlanGenerator::next`. --- simulator/generation/plan.rs | 276 +++++++++++++++++++---------------- 1 file changed, 149 insertions(+), 127 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 97b842552..a02b673dc 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -1,4 +1,4 @@ -use std::vec; +use std::{num::NonZeroUsize, vec}; use sql_generation::{ generation::{Arbitrary, ArbitraryFrom, GenerationContext, frequency}, @@ -18,28 +18,27 @@ use crate::{ model::{ Query, interactions::{ - Fault, Interaction, InteractionPlan, InteractionPlanIterator, InteractionStats, - InteractionType, Interactions, InteractionsType, + Fault, Interaction, InteractionBuilder, InteractionPlan, InteractionPlanIterator, + InteractionType, Interactions, InteractionsType, Span, }, - metrics::Remaining, + metrics::{InteractionStats, Remaining}, property::Property, }, }; impl InteractionPlan { - pub fn init_plan(env: &mut SimulatorEnv) -> Self { - let mut plan = InteractionPlan::new(env.profile.experimental_mvcc); - - // First create at least one table - let create_query = Create::arbitrary(&mut env.rng.clone(), &env.connection_context(0)); - - // initial query starts at 0th connection - plan.push(Interactions::new( - 0, - InteractionsType::Query(Query::Create(create_query)), - )); - - plan + pub fn generator<'a>( + &'a mut self, + rng: &'a mut impl rand::Rng, + ) -> impl InteractionPlanIterator { + let interactions = self.interactions_list().to_vec(); + let iter = interactions.into_iter(); + PlanGenerator { + plan: self, + peek: None, + iter, + rng, + } } /// Appends a new [Interactions] and outputs the next set of [Interaction] to take @@ -47,10 +46,21 @@ impl InteractionPlan { &mut self, rng: &mut impl rand::Rng, env: &mut SimulatorEnv, - ) -> Option> { + ) -> Option { + // First interaction + if self.len_properties() == 0 { + // First create at least one table + let create_query = Create::arbitrary(&mut env.rng.clone(), &env.connection_context(0)); + + // initial query starts at 0th connection + let interactions = + Interactions::new(0, InteractionsType::Query(Query::Create(create_query))); + return Some(interactions); + } + let num_interactions = env.opts.max_interactions as usize; // If last interaction needs to check all db tables, generate the Property to do so - if let Some(i) = self.plan().last() + if let Some(i) = self.last_interactions() && i.check_tables() { let check_all_tables = Interactions::new( @@ -65,13 +75,10 @@ impl InteractionPlan { }), ); - let out_interactions = check_all_tables.interactions(); - - self.push(check_all_tables); - return Some(out_interactions); + return Some(check_all_tables); } - if self.len() < num_interactions { + if self.len_properties() < num_interactions { let conn_index = env.choose_conn(rng); let interactions = if self.mvcc && !env.conn_in_transaction(conn_index) { let query = Query::Begin(Begin::Concurrent); @@ -88,58 +95,20 @@ impl InteractionPlan { Interactions::arbitrary_from(rng, conn_ctx, (env, self.stats(), conn_index)) }; - tracing::debug!("Generating interaction {}/{}", self.len(), num_interactions); + tracing::debug!( + "Generating interaction {}/{}", + self.len_properties(), + num_interactions + ); - let mut out_interactions = interactions.interactions(); - - assert!(!out_interactions.is_empty()); - - let out_interactions = if self.mvcc - && out_interactions - .iter() - .any(|interaction| interaction.is_ddl()) - { - // DDL statements must be serial, so commit all connections and then execute the DDL - let mut commit_interactions = (0..env.connections.len()) - .filter(|&idx| env.conn_in_transaction(idx)) - .map(|idx| { - let query = Query::Commit(Commit); - let interaction = Interactions::new(idx, InteractionsType::Query(query)); - let out_interactions = interaction.interactions(); - self.push(interaction); - out_interactions - }) - .fold( - Vec::with_capacity(env.connections.len()), - |mut accum, mut curr| { - accum.append(&mut curr); - accum - }, - ); - commit_interactions.append(&mut out_interactions); - commit_interactions - } else { - out_interactions - }; - - self.push(interactions); - Some(out_interactions) + Some(interactions) } else { - None - } - } - - pub fn generator<'a>( - &'a mut self, - rng: &'a mut impl rand::Rng, - ) -> impl InteractionPlanIterator { - let interactions = self.interactions_list(); - let iter = interactions.into_iter(); - PlanGenerator { - plan: self, - peek: None, - iter, - rng, + // after we generated all interactions if some connection is still in a transaction, commit + (0..env.connections.len()) + .find(|idx| env.conn_in_transaction(*idx)) + .map(|conn_index| { + Interactions::new(conn_index, InteractionsType::Query(Query::Commit(Commit))) + }) } } } @@ -159,12 +128,18 @@ impl<'a, R: rand::Rng> PlanGenerator<'a, R> { // Iterator ended, try to create a new iterator // This will not be an infinte sequence because generate_next_interaction will eventually // stop generating - let mut iter = self - .plan - .generate_next_interaction(self.rng, env) - .map_or(Vec::new().into_iter(), |interactions| { - interactions.into_iter() - }); + let interactions = self.plan.generate_next_interaction(self.rng, env)?; + + let id = self.plan.next_property_id(); + + let iter = interactions.interactions(id); + + assert!(!iter.is_empty()); + + let mut iter = iter.into_iter(); + + self.plan.push_interactions(interactions); + let next = iter.next(); self.iter = iter; @@ -186,8 +161,11 @@ impl<'a, R: rand::Rng> PlanGenerator<'a, R> { &conn_ctx, ); - let InteractionsType::Property(property) = - &mut self.plan.last_mut().unwrap().interactions + let Some(InteractionsType::Property(property)) = self + .plan + .last_interactions() + .as_ref() + .map(|interactions| &interactions.interactions) else { unreachable!("only properties have extensional queries"); }; @@ -205,20 +183,15 @@ impl<'a, R: rand::Rng> PlanGenerator<'a, R> { if let Some(new_query) = (query_gen)(self.rng, &conn_ctx, &query_distr, property) { - let queries = property.get_extensional_queries().unwrap(); - let query = queries - .iter_mut() - .find(|query| matches!(query, Query::Placeholder)) - .expect("Placeholder should be present in extensional queries"); - *query = new_query.clone(); break new_query; } count += 1; }; - Interaction::new( - interaction.connection_index, - InteractionType::Query(new_query), - ) + + InteractionBuilder::from_interaction(&interaction) + .interaction(InteractionType::Query(new_query)) + .build() + .unwrap() } else { interaction } @@ -237,36 +210,22 @@ impl<'a, R: rand::Rng> InteractionPlanIterator for PlanGenerator<'a, R> { /// try to generate the next [Interactions] and store it fn next(&mut self, env: &mut SimulatorEnv) -> Option { let mvcc = self.plan.mvcc; - match self.peek(env) { + let mut next_interaction = || match self.peek(env) { Some(peek_interaction) => { if mvcc && peek_interaction.is_ddl() { - // try to commit a transaction as we cannot execute DDL statements in concurrent mode + // if any connection is in a transaction, + // try to commit the transaction as we cannot execute DDL statements in concurrent mode - let commit_connection = (0..env.connections.len()) - .find(|idx| env.conn_in_transaction(*idx)) - .map(|conn_index| { - let query = Query::Commit(Commit); - let interaction = Interactions::new( - conn_index, - InteractionsType::Query(query.clone()), - ); - - // Connections are queued for commit on `generate_next_interaction` if Interactions::Query or Interactions::Property produce a DDL statement. - // This means that the only way we will reach here, is if the DDL statement was created later in the extensional query of a Property - let queries = self - .plan - .last_mut() - .unwrap() - .get_extensional_queries() - .unwrap(); - queries.insert(0, query.clone()); - - self.plan.push(interaction); - - Interaction::new(conn_index, InteractionType::Query(query)) - }); - if commit_connection.is_some() { - return commit_connection; + if let Some(conn_index) = + (0..env.connections.len()).find(|idx| env.conn_in_transaction(*idx)) + { + return Some( + InteractionBuilder::from_interaction(peek_interaction) + .interaction(InteractionType::Query(Query::Commit(Commit))) + .connection_index(conn_index) + .build() + .unwrap(), + ); } } @@ -274,18 +233,81 @@ impl<'a, R: rand::Rng> InteractionPlanIterator for PlanGenerator<'a, R> { } None => { // after we generated all interactions if some connection is still in a transaction, commit - (0..env.connections.len()) + let commit = (0..env.connections.len()) .find(|idx| env.conn_in_transaction(*idx)) .map(|conn_index| { let query = Query::Commit(Commit); - let interaction = + let interactions = Interactions::new(conn_index, InteractionsType::Query(query)); - self.plan.push(interaction); - Interaction::new(conn_index, InteractionType::Query(Query::Commit(Commit))) - }) + let interaction = InteractionBuilder::with_interaction( + InteractionType::Query(Query::Commit(Commit)), + ) + .connection_index(conn_index) + .id(self.plan.next_property_id()) + .span(Span::StartEnd) + .build() + .unwrap(); + + self.plan.push_interactions(interactions); + + interaction + }); + + #[cfg(debug_assertions)] + if commit.is_none() { + // Do a final sanity check to make sure that all interactions are sorted by ids + assert!(self.plan.interactions_list().is_sorted_by_key(|a| a.id())); + } + commit } + }; + let next_interaction = next_interaction(); + // intercept interaction to update metrics + if let Some(next_interaction) = next_interaction.as_ref() { + // Skip counting queries that come from Properties that only exist to check tables + if let Some(property_meta) = next_interaction.property_meta + && !property_meta.property.check_tables() + { + self.plan.stats_mut().update(next_interaction); + } + self.plan.push(next_interaction.clone()); } + + next_interaction + } +} + +impl Interactions { + pub(crate) fn interactions(&self, id: NonZeroUsize) -> Vec { + let ret = match &self.interactions { + InteractionsType::Property(property) => { + property.interactions(self.connection_index, id) + } + InteractionsType::Query(query) => { + let mut builder = + InteractionBuilder::with_interaction(InteractionType::Query(query.clone())); + builder + .connection_index(self.connection_index) + .id(id) + .span(Span::StartEnd); + let interaction = builder.build().unwrap(); + vec![interaction] + } + InteractionsType::Fault(fault) => { + let mut builder = + InteractionBuilder::with_interaction(InteractionType::Fault(*fault)); + builder + .connection_index(self.connection_index) + .id(id) + .span(Span::StartEnd); + let interaction = builder.build().unwrap(); + vec![interaction] + } + }; + + assert!(!ret.is_empty()); + ret } } @@ -303,16 +325,16 @@ fn random_fault( Interactions::new(conn_index, InteractionsType::Fault(fault)) } -impl ArbitraryFrom<(&SimulatorEnv, InteractionStats, usize)> for Interactions { +impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats, usize)> for Interactions { fn arbitrary_from( rng: &mut R, conn_ctx: &C, - (env, stats, conn_index): (&SimulatorEnv, InteractionStats, usize), + (env, stats, conn_index): (&SimulatorEnv, &InteractionStats, usize), ) -> Self { let remaining_ = Remaining::new( env.opts.max_interactions, &env.profile.query, - &stats, + stats, env.profile.experimental_mvcc, conn_ctx, ); From 2c8754985beaf060bdd7ae48bf24759651c90751 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 12:14:25 -0300 Subject: [PATCH 08/16] refactor shrinking to use utilities in the `InteractionPlan` to iterate over properties, instead of handrolling property iteration --- simulator/shrink/plan.rs | 497 +++++++++++++++------------------------ 1 file changed, 186 insertions(+), 311 deletions(-) diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 0ea62d45a..63616a0eb 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -4,28 +4,19 @@ use crate::{ SandboxedResult, SimulatorEnv, model::{ Query, - interactions::{InteractionPlan, InteractionType, Interactions, InteractionsType}, - property::Property, + interactions::{InteractionPlan, InteractionType}, + property::PropertyDiscriminants, }, run_simulation, runner::execution::Execution, }; use std::{ collections::HashMap, + num::NonZeroUsize, + ops::Range, sync::{Arc, Mutex}, }; -fn retain_relevant_queries( - extensional_queries: &mut Vec, - depending_tables: &IndexSet, -) { - extensional_queries.retain(|query| { - query.is_transaction() - || (!matches!(query, Query::Select(..)) - && query.uses().iter().any(|t| depending_tables.contains(t))) - }); -} - impl InteractionPlan { /// Create a smaller interaction plan by deleting a property pub(crate) fn shrink_interaction_plan(&self, failing_execution: &Execution) -> InteractionPlan { @@ -34,54 +25,85 @@ impl InteractionPlan { // - Shrink properties by removing their extensions, or shrinking their values let mut plan = self.clone(); - let all_interactions = self.interactions_list_with_secondary_index(); - let secondary_interactions_index = all_interactions[failing_execution.interaction_index].0; + let all_interactions = self.interactions_list(); + let failing_interaction = &all_interactions[failing_execution.interaction_index]; - // Index of the parent property where the interaction originated from - let failing_property = &self[secondary_interactions_index]; - let mut depending_tables = failing_property.dependencies(); + let range = self.find_interactions_range(failing_interaction.id()); - { - let mut idx = failing_execution.interaction_index; - loop { - if all_interactions[idx].0 != secondary_interactions_index { - // Stop when we reach a different property - break; - } - match &all_interactions[idx].1.interaction { + // Interactions that are part of the failing overall property + let mut failing_property = all_interactions + [range.start..=failing_execution.interaction_index] + .iter() + .rev(); + + let depending_tables = failing_property + .find_map(|interaction| { + match &interaction.interaction { InteractionType::Query(query) | InteractionType::FaultyQuery(query) => { - depending_tables = query.dependencies(); - break; - } - // Fault does not depend on - InteractionType::Fault(..) => break, - _ => { - // In principle we should never fail this checked_sub. - // But if there is a bug in how we count the secondary index - // we may panic if we do not use a checked_sub. - if let Some(new_idx) = idx.checked_sub(1) { - idx = new_idx; - } else { - tracing::warn!("failed to find error query"); - break; - } + Some(query.dependencies()) } + // Fault does not depend on tables + InteractionType::Fault(..) => None, + _ => None, } - } - } + }) + .unwrap_or_else(IndexSet::new); let before = self.len(); // Remove all properties after the failing one - plan.truncate(secondary_interactions_index + 1); + plan.truncate(failing_execution.interaction_index + 1); // means we errored in some fault on transaction statement so just maintain the statements from before the failing one if !depending_tables.is_empty() { - plan.remove_properties(&depending_tables, secondary_interactions_index); + plan.remove_properties(&depending_tables, range); } let after = plan.len(); + tracing::info!( + "Shrinking interaction plan from {} to {} interactions", + before, + after + ); + + plan + } + + /// Create a smaller interaction plan by deleting a property + pub(crate) fn brute_shrink_interaction_plan( + &self, + result: &SandboxedResult, + env: Arc>, + ) -> InteractionPlan { + let failing_execution = match result { + SandboxedResult::Panicked { + error: _, + last_execution: e, + } => e, + SandboxedResult::FoundBug { + error: _, + history: _, + last_execution: e, + } => e, + SandboxedResult::Correct => { + unreachable!("shrink is never called on correct result") + } + }; + + let mut plan = self.clone(); + let all_interactions = self.interactions_list(); + let property_id = all_interactions[failing_execution.interaction_index].id(); + + let before = self.len_properties(); + + plan.truncate(failing_execution.interaction_index + 1); + + // phase 2: shrink the entire plan + plan = Self::iterative_shrink(&plan, failing_execution, result, env, property_id); + + let after = plan.len_properties(); + tracing::info!( "Shrinking interaction plan from {} to {} properties", before, @@ -91,97 +113,127 @@ impl InteractionPlan { plan } + /// shrink a plan by removing one interaction at a time (and its deps) while preserving the error + fn iterative_shrink( + plan: &InteractionPlan, + failing_execution: &Execution, + old_result: &SandboxedResult, + env: Arc>, + failing_property_id: NonZeroUsize, + ) -> InteractionPlan { + let mut iter_properties = plan.rev_iter_properties(); + + let mut ret_plan = plan.clone(); + + while let Some(property_interactions) = iter_properties.next_property() { + // get the overall property id and try to remove it + // need to consume the iterator, to advance outer iterator + if let Some((_, interaction)) = property_interactions.last() + && interaction.id() != failing_property_id + { + // try to remove the property + let mut test_plan = ret_plan.clone(); + test_plan.remove_property(interaction.id()); + if Self::test_shrunk_plan(&test_plan, failing_execution, old_result, env.clone()) { + ret_plan = test_plan; + } + } + } + + ret_plan + } + + fn test_shrunk_plan( + test_plan: &InteractionPlan, + failing_execution: &Execution, + old_result: &SandboxedResult, + env: Arc>, + ) -> bool { + let last_execution = Arc::new(Mutex::new(*failing_execution)); + let result = SandboxedResult::from( + std::panic::catch_unwind(|| { + let plan = test_plan.static_iterator(); + + run_simulation(env.clone(), plan, last_execution.clone()) + }), + last_execution, + ); + match (old_result, &result) { + ( + SandboxedResult::Panicked { error: e1, .. }, + SandboxedResult::Panicked { error: e2, .. }, + ) + | ( + SandboxedResult::FoundBug { error: e1, .. }, + SandboxedResult::FoundBug { error: e2, .. }, + ) => e1 == e2, + _ => false, + } + } + /// Remove all properties that do not use the failing tables fn remove_properties( &mut self, depending_tables: &IndexSet, - failing_interaction_index: usize, + failing_interaction_range: Range, ) { - let mut idx = 0; - // Remove all properties that do not use the failing tables - self.retain_mut(|interactions| { - let retain = if idx == failing_interaction_index { - true - } else { - let mut has_table = interactions - .uses() - .iter() - .any(|t| depending_tables.contains(t)); - - if has_table { - // will contain extensional queries that reference the depending tables - let mut extensional_queries = Vec::new(); - - // Remove the extensional parts of the properties - if let InteractionsType::Property(p) = &mut interactions.interactions { - match p { - Property::InsertValuesSelect { queries, .. } - | Property::DoubleCreateFailure { queries, .. } - | Property::DeleteSelect { queries, .. } - | Property::DropSelect { queries, .. } - | Property::Queries { queries } => { - // Remove placeholder queries - queries.retain(|query| !matches!(query, Query::Placeholder)); - extensional_queries.append(queries); - } - Property::AllTableHaveExpectedContent { tables } => { - tables.retain(|table| depending_tables.contains(table)); - } - Property::FsyncNoWait { .. } | Property::FaultyQuery { .. } => {} - Property::SelectLimit { .. } - | Property::SelectSelectOptimizer { .. } - | Property::WhereTrueFalseNull { .. } - | Property::UNIONAllPreservesCardinality { .. } - | Property::ReadYourUpdatesBack { .. } - | Property::TableHasExpectedContent { .. } => {} - } - } - // Check again after query clear if the interactions still uses the failing table - has_table = interactions + // First pass - mark indexes that should be retained + let mut retain_map = Vec::with_capacity(self.len()); + let mut iter_properties = self.iter_properties(); + while let Some(property_interactions) = iter_properties.next_property() { + for (idx, interaction) in property_interactions { + let retain = if failing_interaction_range.contains(&idx) { + true + } else { + let has_table = interaction .uses() .iter() .any(|t| depending_tables.contains(t)); - // means the queries in the original property are present in the depending tables regardless of the extensional queries - if has_table { - if let Some(queries) = interactions.get_extensional_queries() { - retain_relevant_queries(&mut extensional_queries, depending_tables); - queries.append(&mut extensional_queries); - } + let is_fault = matches!(&interaction.interaction, InteractionType::Fault(..)); + let is_transaction = matches!( + &interaction.interaction, + InteractionType::Query(Query::Begin(..)) + | InteractionType::Query(Query::Commit(..)) + | InteractionType::Query(Query::Rollback(..)) + ); + let is_assertion = matches!( + &interaction.interaction, + InteractionType::Assertion(..) | InteractionType::Assumption(..) + ); + + let skip_interaction = if let Some(property_meta) = interaction.property_meta + && matches!( + property_meta.property, + PropertyDiscriminants::AllTableHaveExpectedContent + | PropertyDiscriminants::SelectLimit + | PropertyDiscriminants::SelectSelectOptimizer + | PropertyDiscriminants::TableHasExpectedContent + | PropertyDiscriminants::UnionAllPreservesCardinality + | PropertyDiscriminants::WhereTrueFalseNull + ) { + // Theses properties only emit select queries, so they can be discarded entirely + true } else { - // original property without extensional queries does not reference the tables so convert the property to - // `Property::Queries` if `extensional_queries` is not empty - retain_relevant_queries(&mut extensional_queries, depending_tables); - if !extensional_queries.is_empty() { - has_table = true; - *interactions = Interactions::new( - interactions.connection_index, - InteractionsType::Property(Property::Queries { - queries: extensional_queries, - }), - ); - } - } - } - let is_fault = matches!(interactions.interactions, InteractionsType::Fault(..)); - let is_transaction = matches!( - interactions.interactions, - InteractionsType::Query(Query::Begin(..)) - | InteractionsType::Query(Query::Commit(..)) - | InteractionsType::Query(Query::Rollback(..)) - ); - is_fault - || is_transaction - || (has_table - && !matches!( - interactions.interactions, - InteractionsType::Query(Query::Select(_)) - | InteractionsType::Property(Property::SelectLimit { .. }) - | InteractionsType::Property( - Property::SelectSelectOptimizer { .. } - ) - )) - }; + // Standalone Select query + matches!( + &interaction.interaction, + InteractionType::Query(Query::Select(_)) + ) + }; + + !skip_interaction && (is_fault || is_transaction || is_assertion || has_table) + }; + retain_map.push(retain); + } + } + + debug_assert_eq!(self.len(), retain_map.len()); + + let mut idx = 0; + // Remove all properties that do not use the failing tables + self.retain_mut(|_| { + let retain = retain_map[idx]; idx += 1; retain }); @@ -191,23 +243,23 @@ impl InteractionPlan { // Comprises of idxs of Commit and Rollback intereactions let mut end_tx_idx: HashMap> = HashMap::new(); - for (idx, interactions) in self.iter().enumerate() { - match &interactions.interactions { - InteractionsType::Query(Query::Begin(..)) => { + for (idx, interaction) in self.interactions_list().iter().enumerate() { + match &interaction.interaction { + InteractionType::Query(Query::Begin(..)) => { begin_idx - .entry(interactions.connection_index) + .entry(interaction.connection_index) .or_insert_with(|| vec![idx]); } - InteractionsType::Query(Query::Commit(..)) - | InteractionsType::Query(Query::Rollback(..)) => { + InteractionType::Query(Query::Commit(..)) + | InteractionType::Query(Query::Rollback(..)) => { let last_begin = begin_idx - .get(&interactions.connection_index) + .get(&interaction.connection_index) .and_then(|list| list.last()) .unwrap() + 1; if last_begin == idx { end_tx_idx - .entry(interactions.connection_index) + .entry(interaction.connection_index) .or_insert_with(|| vec![idx]); } } @@ -241,181 +293,4 @@ impl InteractionPlan { retain }); } - - /// Create a smaller interaction plan by deleting a property - pub(crate) fn brute_shrink_interaction_plan( - &self, - result: &SandboxedResult, - env: Arc>, - ) -> InteractionPlan { - let failing_execution = match result { - SandboxedResult::Panicked { - error: _, - last_execution: e, - } => e, - SandboxedResult::FoundBug { - error: _, - history: _, - last_execution: e, - } => e, - SandboxedResult::Correct => { - unreachable!("shrink is never called on correct result") - } - }; - - let mut plan = self.clone(); - let all_interactions = self.interactions_list_with_secondary_index(); - let secondary_interactions_index = all_interactions[failing_execution.interaction_index].0; - - { - let mut idx = failing_execution.interaction_index; - loop { - if all_interactions[idx].0 != secondary_interactions_index { - // Stop when we reach a different property - break; - } - match &all_interactions[idx].1.interaction { - // Fault does not depend on - InteractionType::Fault(..) => break, - _ => { - // In principle we should never fail this checked_sub. - // But if there is a bug in how we count the secondary index - // we may panic if we do not use a checked_sub. - if let Some(new_idx) = idx.checked_sub(1) { - idx = new_idx; - } else { - tracing::warn!("failed to find error query"); - break; - } - } - } - } - } - - let before = self.len(); - - plan.truncate(secondary_interactions_index + 1); - - // phase 1: shrink extensions - for interaction in &mut plan { - if let InteractionsType::Property(property) = &mut interaction.interactions { - match property { - Property::InsertValuesSelect { queries, .. } - | Property::DoubleCreateFailure { queries, .. } - | Property::DeleteSelect { queries, .. } - | Property::DropSelect { queries, .. } - | Property::Queries { queries } => { - let mut temp_plan = InteractionPlan::new_with( - queries - .iter() - .map(|q| { - Interactions::new( - interaction.connection_index, - InteractionsType::Query(q.clone()), - ) - }) - .collect(), - self.mvcc, - ); - - temp_plan = InteractionPlan::iterative_shrink( - temp_plan, - failing_execution, - result, - env.clone(), - secondary_interactions_index, - ); - //temp_plan = Self::shrink_queries(temp_plan, failing_execution, result, env); - - *queries = temp_plan - .into_iter() - .filter_map(|i| match i.interactions { - InteractionsType::Query(q) => Some(q), - _ => None, - }) - .collect(); - } - Property::WhereTrueFalseNull { .. } - | Property::UNIONAllPreservesCardinality { .. } - | Property::SelectLimit { .. } - | Property::SelectSelectOptimizer { .. } - | Property::FaultyQuery { .. } - | Property::FsyncNoWait { .. } - | Property::ReadYourUpdatesBack { .. } - | Property::TableHasExpectedContent { .. } - | Property::AllTableHaveExpectedContent { .. } => {} - } - } - } - - // phase 2: shrink the entire plan - plan = Self::iterative_shrink( - plan, - failing_execution, - result, - env, - secondary_interactions_index, - ); - - let after = plan.len(); - - tracing::info!( - "Shrinking interaction plan from {} to {} properties", - before, - after - ); - - plan - } - - /// shrink a plan by removing one interaction at a time (and its deps) while preserving the error - fn iterative_shrink( - mut plan: InteractionPlan, - failing_execution: &Execution, - old_result: &SandboxedResult, - env: Arc>, - secondary_interaction_index: usize, - ) -> InteractionPlan { - for i in (0..plan.len()).rev() { - if i == secondary_interaction_index { - continue; - } - let mut test_plan = plan.clone(); - - test_plan.remove(i); - - if Self::test_shrunk_plan(&test_plan, failing_execution, old_result, env.clone()) { - plan = test_plan; - } - } - plan - } - - fn test_shrunk_plan( - test_plan: &InteractionPlan, - failing_execution: &Execution, - old_result: &SandboxedResult, - env: Arc>, - ) -> bool { - let last_execution = Arc::new(Mutex::new(*failing_execution)); - let result = SandboxedResult::from( - std::panic::catch_unwind(|| { - let plan = test_plan.static_iterator(); - - run_simulation(env.clone(), plan, last_execution.clone()) - }), - last_execution, - ); - match (old_result, &result) { - ( - SandboxedResult::Panicked { error: e1, .. }, - SandboxedResult::Panicked { error: e2, .. }, - ) - | ( - SandboxedResult::FoundBug { error: e1, .. }, - SandboxedResult::FoundBug { error: e2, .. }, - ) => e1 == e2, - _ => false, - } - } } From 836d115853425f91e487469363c2b133c83b3862 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 13:51:26 -0300 Subject: [PATCH 09/16] create interaction plan correct in `main.rs` --- simulator/main.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/simulator/main.rs b/simulator/main.rs index 9b44a91fb..2d5fdd1be 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -289,7 +289,7 @@ fn run_simulator( tracing::trace!( "adding bug to bugbase, seed: {}, plan: {}, error: {}", env.opts.seed, - plan.len(), + plan.len_properties(), error ); bugbase @@ -455,14 +455,15 @@ fn setup_simulation( Paths::new(&dir) }; - let mut env = SimulatorEnv::new(seed, cli_opts, paths, SimulationType::Default, profile); + let env = SimulatorEnv::new(seed, cli_opts, paths, SimulationType::Default, profile); tracing::info!("Generating database interaction plan..."); - let plan = InteractionPlan::init_plan(&mut env); + let plan = InteractionPlan::new(env.profile.experimental_mvcc); (seed, env, plan) } + fn run_simulation( env: Arc>, plan: impl InteractionPlanIterator, From 087d5f59a1bdae01d9b394065d0b025dfa6b6dd6 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 14:03:35 -0300 Subject: [PATCH 10/16] fix execution ticks not ticking enough --- simulator/runner/env.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs index 22022c8ae..fd330b93d 100644 --- a/simulator/runner/env.rs +++ b/simulator/runner/env.rs @@ -302,8 +302,7 @@ impl SimulatorEnv { let mut opts = SimulatorOpts { seed, - ticks: rng - .random_range(cli_opts.minimum_tests as usize..=cli_opts.maximum_tests as usize), + ticks: usize::MAX, disable_select_optimizer: cli_opts.disable_select_optimizer, disable_insert_values_select: cli_opts.disable_insert_values_select, disable_double_create_failure: cli_opts.disable_double_create_failure, From af31e74d9f03b824ccb20b055009b53647658061 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 16:32:12 -0300 Subject: [PATCH 11/16] add depending tables to assertions to delete them if needed in shrinking --- simulator/generation/property.rs | 30 +++++++++++++++++++++++++++--- simulator/model/interactions.rs | 24 +++++++++++++++++++----- simulator/shrink/plan.rs | 11 +++++------ 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 1db2cf0a7..7c8ba06d9 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -258,6 +258,7 @@ impl Property { } Property::TableHasExpectedContent { table } => { let table = table.to_string(); + let table_dependency = table.clone(); let table_name = table.clone(); let assumption = InteractionType::Assumption(Assertion::new( format!("table {} exists", table.clone()), @@ -269,6 +270,7 @@ impl Property { Ok(Err(format!("table {table_name} does not exist"))) } }, + vec![table_dependency.clone()], )); let select_interaction = InteractionType::Query(Query::Select(Select::simple( @@ -309,6 +311,7 @@ impl Property { } Ok(Ok(())) }, + vec![table_dependency.clone()], )); vec![ @@ -319,6 +322,7 @@ impl Property { } Property::ReadYourUpdatesBack { update, select } => { let table = update.table().to_string(); + let table_dependency = table.clone(); let assumption = InteractionType::Assumption(Assertion::new( format!("table {} exists", table.clone()), move |_: &Vec, env: &mut SimulatorEnv| { @@ -329,6 +333,7 @@ impl Property { Ok(Err(format!("table {} does not exist", table.clone()))) } }, + vec![table_dependency.clone()], )); let update_interaction = InteractionType::Query(Query::Update(update.clone())); @@ -373,6 +378,7 @@ impl Property { Err(err) => Err(LimboError::InternalError(err.to_string())), } }, + vec![table_dependency], )); vec![ @@ -419,6 +425,7 @@ impl Property { } } }, + vec![insert.table().to_string()], )); let assertion = InteractionType::Assertion(Assertion::new( @@ -453,6 +460,7 @@ impl Property { Err(err) => Err(LimboError::InternalError(err.to_string())), } }, + vec![insert.table().to_string()], )); let mut interactions = Vec::new(); @@ -475,6 +483,7 @@ impl Property { } Property::DoubleCreateFailure { create, queries } => { let table_name = create.table.name.clone(); + let table_dependency = table_name.clone(); let assumption = InteractionType::Assumption(Assertion::new( "Double-Create-Failure should not be called on an existing table".to_string(), @@ -486,6 +495,7 @@ impl Property { Ok(Err(format!("table {table_name} already exists"))) } }, + vec![table_dependency.clone()], )); let cq1 = InteractionType::Query(Query::Create(create.clone())); @@ -508,7 +518,7 @@ impl Property { } } } - }) ); + }, vec![table_dependency],) ); let mut interactions = Vec::new(); interactions.push(InteractionBuilder::with_interaction(assumption)); @@ -556,6 +566,7 @@ impl Property { } } }, + select.dependencies().into_iter().collect(), )); let limit = select @@ -581,6 +592,7 @@ impl Property { Err(_) => Ok(Ok(())), } }, + select.dependencies().into_iter().collect(), )); vec![ @@ -615,6 +627,7 @@ impl Property { } } }, + vec![table.clone()], )); let delete = InteractionType::Query(Query::Delete(Delete { @@ -649,6 +662,7 @@ impl Property { Err(err) => Err(LimboError::InternalError(err.to_string())), } }, + vec![table.clone()], )); let mut interactions = Vec::new(); @@ -689,6 +703,7 @@ impl Property { } } }, + vec![table.clone()], )); let table_name = table.clone(); @@ -714,6 +729,7 @@ impl Property { }, } }, + vec![table.clone()], )); let drop = InteractionType::Query(Query::Drop(Drop { @@ -761,6 +777,7 @@ impl Property { } } }, + vec![table.clone()], )); let select1 = InteractionType::Query(Query::Select(Select::single( @@ -821,6 +838,7 @@ impl Property { } } }, + vec![table.clone()], )); vec![ @@ -861,6 +879,7 @@ impl Property { } } }, + query.dependencies().into_iter().collect(), ); [ InteractionType::FaultyQuery(query.clone()), @@ -871,6 +890,7 @@ impl Property { .collect() } Property::WhereTrueFalseNull { select, predicate } => { + let tables_dependencies = select.dependencies().into_iter().collect::>(); let assumption = InteractionType::Assumption(Assertion::new( format!( "tables ({}) exists", @@ -898,6 +918,7 @@ impl Property { } } }, + tables_dependencies.clone(), )); let old_predicate = select.body.select.where_clause.clone(); @@ -1019,6 +1040,7 @@ impl Property { } } }, + tables_dependencies, )); vec![ @@ -1073,7 +1095,9 @@ impl Property { } } }, - )), + s3.dependencies().into_iter().collect() + ) + ), ].into_iter().map(InteractionBuilder::with_interaction).collect() } Property::Queries { queries } => queries @@ -1163,7 +1187,7 @@ fn assert_all_table_values( Err(err) => Err(LimboError::InternalError(format!("{err}"))), } } - })); + }, vec![table.clone()])); [select, assertion].into_iter().map(InteractionBuilder::with_interaction) }) } diff --git a/simulator/model/interactions.rs b/simulator/model/interactions.rs index 580ecda1c..4547e9a2e 100644 --- a/simulator/model/interactions.rs +++ b/simulator/model/interactions.rs @@ -9,6 +9,7 @@ use std::{ }; use either::Either; +use indexmap::IndexSet; use itertools::Itertools; use serde::{Deserialize, Serialize}; use sql_generation::model::table::SimValue; @@ -379,12 +380,12 @@ impl InteractionsType { impl Display for InteractionPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { const PAD: usize = 4; - let mut indentation_level = 0; + let mut indentation_level: usize = 0; for interaction in &self.plan { if let Some(name) = interaction.property_meta.map(|p| p.property.name()) && interaction.span.is_some_and(|span| span.start()) { - indentation_level += 1; + indentation_level = indentation_level.saturating_add(1); writeln!(f, "-- begin testing '{name}'")?; } @@ -396,7 +397,7 @@ impl Display for InteractionPlan { if let Some(name) = interaction.property_meta.map(|p| p.property.name()) && interaction.span.is_some_and(|span| span.end()) { - indentation_level -= 1; + indentation_level = indentation_level.saturating_sub(1); writeln!(f, "-- end testing '{name}'")?; } } @@ -411,7 +412,8 @@ type AssertionFunc = #[derive(Clone)] pub struct Assertion { pub func: Rc, - pub name: String, // For display purposes in the plan + pub name: String, // For display purposes in the plan + pub tables: Vec, // Tables it depends on } impl Debug for Assertion { @@ -423,7 +425,7 @@ impl Debug for Assertion { } impl Assertion { - pub fn new(name: String, func: F) -> Self + pub fn new(name: String, func: F, tables: Vec) -> Self where F: Fn(&Vec, &mut SimulatorEnv) -> Result> + 'static @@ -432,8 +434,17 @@ impl Assertion { Self { func: Rc::new(func), name, + tables, } } + + pub fn dependencies(&self) -> IndexSet { + IndexSet::from_iter(self.tables.clone()) + } + + pub fn uses(&self) -> Vec { + self.tables.clone() + } } #[derive(Debug, Clone, Copy, Serialize, Deserialize)] @@ -566,6 +577,9 @@ impl Interaction { InteractionType::Query(query) | InteractionType::FsyncQuery(query) | InteractionType::FaultyQuery(query) => query.uses(), + InteractionType::Assertion(assert) | InteractionType::Assumption(assert) => { + assert.uses() + } _ => vec![], } } diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 63616a0eb..2a319d39b 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -44,6 +44,9 @@ impl InteractionPlan { } // Fault does not depend on tables InteractionType::Fault(..) => None, + InteractionType::Assertion(assert) | InteractionType::Assumption(assert) => { + (!assert.tables.is_empty()).then(|| assert.dependencies()) + } _ => None, } }) @@ -182,7 +185,7 @@ impl InteractionPlan { let mut iter_properties = self.iter_properties(); while let Some(property_interactions) = iter_properties.next_property() { for (idx, interaction) in property_interactions { - let retain = if failing_interaction_range.contains(&idx) { + let retain = if failing_interaction_range.end == idx { true } else { let has_table = interaction @@ -197,10 +200,6 @@ impl InteractionPlan { | InteractionType::Query(Query::Commit(..)) | InteractionType::Query(Query::Rollback(..)) ); - let is_assertion = matches!( - &interaction.interaction, - InteractionType::Assertion(..) | InteractionType::Assumption(..) - ); let skip_interaction = if let Some(property_meta) = interaction.property_meta && matches!( @@ -222,7 +221,7 @@ impl InteractionPlan { ) }; - !skip_interaction && (is_fault || is_transaction || is_assertion || has_table) + !skip_interaction && (is_fault || is_transaction || has_table) }; retain_map.push(retain); } From 9d439556cadb65a185723f273bebc253d9cf8925 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 17:32:18 -0300 Subject: [PATCH 12/16] if table changed names, add its previous names to depending tables when shrinking --- simulator/shrink/plan.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 2a319d39b..b7ffa6d01 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -1,4 +1,5 @@ use indexmap::IndexSet; +use sql_generation::model::query::alter_table::{AlterTable, AlterTableType}; use crate::{ SandboxedResult, SimulatorEnv, @@ -36,7 +37,7 @@ impl InteractionPlan { .iter() .rev(); - let depending_tables = failing_property + let mut depending_tables = failing_property .find_map(|interaction| { match &interaction.interaction { InteractionType::Query(query) | InteractionType::FaultyQuery(query) => { @@ -52,6 +53,30 @@ impl InteractionPlan { }) .unwrap_or_else(IndexSet::new); + // Iterate over the rest of the interactions to identify if the depending tables ever changed names + all_interactions[..range.start] + .iter() + .rev() + .for_each(|interaction| match &interaction.interaction { + InteractionType::Query(query) + | InteractionType::FsyncQuery(query) + | InteractionType::FaultyQuery(query) => { + if let Query::AlterTable(AlterTable { + table_name, + alter_table_type: AlterTableType::RenameTo { new_name }, + }) = query + { + if depending_tables.contains(new_name) + || depending_tables.contains(table_name) + { + depending_tables.insert(new_name.clone()); + depending_tables.insert(table_name.clone()); + } + } + } + _ => {} + }); + let before = self.len(); // Remove all properties after the failing one From 4fd0896538b060fb44907c1c4a21e73443971377 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 18 Oct 2025 19:41:46 -0300 Subject: [PATCH 13/16] remove extension queries from other types of properties --- simulator/shrink/plan.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index b7ffa6d01..d38c8c0fd 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -213,6 +213,8 @@ impl InteractionPlan { let retain = if failing_interaction_range.end == idx { true } else { + let is_part_of_property = failing_interaction_range.contains(&idx); + let has_table = interaction .uses() .iter() @@ -226,8 +228,8 @@ impl InteractionPlan { | InteractionType::Query(Query::Rollback(..)) ); - let skip_interaction = if let Some(property_meta) = interaction.property_meta - && matches!( + let skip_interaction = if let Some(property_meta) = interaction.property_meta { + if matches!( property_meta.property, PropertyDiscriminants::AllTableHaveExpectedContent | PropertyDiscriminants::SelectLimit @@ -236,17 +238,24 @@ impl InteractionPlan { | PropertyDiscriminants::UnionAllPreservesCardinality | PropertyDiscriminants::WhereTrueFalseNull ) { - // Theses properties only emit select queries, so they can be discarded entirely - true + // Theses properties only emit select queries, so they can be discarded entirely + true + } else { + property_meta.extension + && matches!( + &interaction.interaction, + InteractionType::Query(Query::Select(..)) + ) + } } else { - // Standalone Select query matches!( &interaction.interaction, - InteractionType::Query(Query::Select(_)) + InteractionType::Query(Query::Select(..)) ) }; - !skip_interaction && (is_fault || is_transaction || has_table) + (is_part_of_property || !skip_interaction) + && (is_fault || is_transaction || has_table) }; retain_map.push(retain); } From 2aab33b714b93e8d447aef833a19576757df3e53 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 1 Nov 2025 12:11:30 -0300 Subject: [PATCH 14/16] `find_interactions_range` only check for interaction id to determine membership --- simulator/model/interactions.rs | 87 +++++++++++---------------------- 1 file changed, 28 insertions(+), 59 deletions(-) diff --git a/simulator/model/interactions.rs b/simulator/model/interactions.rs index 4547e9a2e..4c389daf4 100644 --- a/simulator/model/interactions.rs +++ b/simulator/model/interactions.rs @@ -8,7 +8,6 @@ use std::{ sync::Arc, }; -use either::Either; use indexmap::IndexSet; use itertools::Itertools; use serde::{Deserialize, Serialize}; @@ -100,49 +99,37 @@ impl InteractionPlan { let backward = || -> usize { interactions .iter() + .enumerate() .rev() .skip(interactions.len() - idx) - .position(|interaction| { - interaction.id() == id - && interaction - .span - .is_some_and(|span| matches!(span, Span::Start)) - }) - .map(|idx| (interactions.len() - 1) - idx - 1) - .expect("A start span should have been emitted") + .find(|(_, interaction)| interaction.id() != id) + .map(|(idx, _)| idx.saturating_add(1)) + .unwrap_or(idx) }; let forward = || -> usize { interactions .iter() + .enumerate() .skip(idx + 1) - .position(|interaction| interaction.id() != id) - .map(|idx| idx - 1) - .unwrap_or(interactions.len() - 1) - // It can happen we do not have an end Span as we can fail in the middle of a property + .find(|(_, interaction)| interaction.id() != id) + .map(|(idx, _)| idx.saturating_sub(1)) + .unwrap_or(idx) }; - if let Some(span) = interaction.span { - match span { - Span::Start => { - // go forward and find the end span - let end_idx = forward(); - idx..end_idx + 1 - } - Span::End => { - // go backward and find the start span - let start_idx = backward(); - start_idx..idx + 1 - } - Span::StartEnd => idx..idx + 1, - } - } else { - // go backward and find the start span + let range = if interaction.property_meta.is_some() { + // go backward and find the interaction that is not the same id let start_idx = backward(); - // go forward and find the end span + // go forward and find the interaction that is not the same id let end_idx = forward(); + start_idx..end_idx + 1 - } + } else { + idx..idx + 1 + }; + + assert!(!range.is_empty()); + range } /// Truncates up to a particular interaction @@ -225,23 +212,13 @@ where pub fn next_property(&mut self) -> Option> { let (idx, interaction) = self.iter.next()?; let id = interaction.id(); - // get interactions from a particular property - let span = interaction - .span - .expect("we should loop on interactions that have a span"); - + // get interactions with a particular property let first = std::iter::once((idx, interaction)); - let property_interactions = match span { - Span::Start => Either::Left( - first.chain( - self.iter - .peeking_take_while(move |(_idx, interaction)| interaction.id() == id), - ), - ), - Span::End => panic!("we should always be at the start of an interaction"), - Span::StartEnd => Either::Right(first), - }; + let property_interactions = first.chain( + self.iter + .peeking_take_while(move |(_idx, interaction)| interaction.id() == id), + ); Some(property_interactions) } @@ -257,22 +234,14 @@ where pub fn next_property(&mut self) -> Option> { let (idx, interaction) = self.iter.next()?; let id = interaction.id(); - // get interactions from a particular property - let span = interaction - .span - .expect("we should loop on interactions that have a span"); + // get interactions with a particular id let first = std::iter::once((idx, interaction)); - let property_interactions = match span { - Span::Start => panic!("we should always be at the end of an interaction"), - Span::End => Either::Left( - self.iter - .peeking_take_while(move |(_idx, interaction)| interaction.id() == id) - .chain(first), - ), - Span::StartEnd => Either::Right(first), - }; + let property_interactions = self + .iter + .peeking_take_while(move |(_idx, interaction)| interaction.id() == id) + .chain(first); Some(property_interactions.into_iter()) } From f09b73c768e20be9fa1301489c32ebfa18edc025 Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 1 Nov 2025 14:13:47 -0300 Subject: [PATCH 15/16] remove Span, as interaction ID is enough to determine membership of a property --- simulator/generation/plan.rs | 13 ++---- simulator/generation/property.rs | 12 +---- simulator/model/interactions.rs | 77 ++++++++++---------------------- 3 files changed, 28 insertions(+), 74 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index a02b673dc..f4a265b61 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -19,7 +19,7 @@ use crate::{ Query, interactions::{ Fault, Interaction, InteractionBuilder, InteractionPlan, InteractionPlanIterator, - InteractionType, Interactions, InteractionsType, Span, + InteractionType, Interactions, InteractionsType, }, metrics::{InteractionStats, Remaining}, property::Property, @@ -245,7 +245,6 @@ impl<'a, R: rand::Rng> InteractionPlanIterator for PlanGenerator<'a, R> { ) .connection_index(conn_index) .id(self.plan.next_property_id()) - .span(Span::StartEnd) .build() .unwrap(); @@ -287,20 +286,14 @@ impl Interactions { InteractionsType::Query(query) => { let mut builder = InteractionBuilder::with_interaction(InteractionType::Query(query.clone())); - builder - .connection_index(self.connection_index) - .id(id) - .span(Span::StartEnd); + builder.connection_index(self.connection_index).id(id); let interaction = builder.build().unwrap(); vec![interaction] } InteractionsType::Fault(fault) => { let mut builder = InteractionBuilder::with_interaction(InteractionType::Fault(*fault)); - builder - .connection_index(self.connection_index) - .id(id) - .span(Span::StartEnd); + builder.connection_index(self.connection_index).id(id); let interaction = builder.build().unwrap(); vec![interaction] } diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 7c8ba06d9..070676f0d 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -32,7 +32,7 @@ use crate::{ model::{ Query, QueryCapabilities, QueryDiscriminants, ResultSet, interactions::{ - Assertion, Interaction, InteractionBuilder, InteractionType, PropertyMetadata, Span, + Assertion, Interaction, InteractionBuilder, InteractionType, PropertyMetadata, }, metrics::Remaining, property::{InteractiveQueryInfo, Property, PropertyDiscriminants}, @@ -252,7 +252,7 @@ impl Property { connection_index: usize, id: NonZeroUsize, ) -> Vec { - let mut interactions: Vec = match self { + let interactions: Vec = match self { Property::AllTableHaveExpectedContent { tables } => { assert_all_table_values(tables, connection_index).collect() } @@ -1109,14 +1109,6 @@ impl Property { assert!(!interactions.is_empty()); - // Add a span to the interactions that matter - if interactions.len() == 1 { - interactions.first_mut().unwrap().span(Span::StartEnd); - } else { - interactions.first_mut().unwrap().span(Span::Start); - interactions.last_mut().unwrap().span(Span::End); - }; - interactions .into_iter() .map(|mut builder| { diff --git a/simulator/model/interactions.rs b/simulator/model/interactions.rs index 4c389daf4..e69887604 100644 --- a/simulator/model/interactions.rs +++ b/simulator/model/interactions.rs @@ -350,24 +350,30 @@ impl Display for InteractionPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { const PAD: usize = 4; let mut indentation_level: usize = 0; - for interaction in &self.plan { - if let Some(name) = interaction.property_meta.map(|p| p.property.name()) - && interaction.span.is_some_and(|span| span.start()) - { - indentation_level = indentation_level.saturating_add(1); - writeln!(f, "-- begin testing '{name}'")?; - } + let mut iter = self.iter_properties(); + while let Some(property) = iter.next_property() { + let mut property = property.peekable(); + let mut start = true; + while let Some((_, interaction)) = property.next() { + if let Some(name) = interaction.property_meta.map(|p| p.property.name()) + && start + { + indentation_level = indentation_level.saturating_add(1); + writeln!(f, "-- begin testing '{name}'")?; + start = false; + } - if indentation_level > 0 { - let padding = " ".repeat(indentation_level * PAD); - f.pad(&padding)?; - } - writeln!(f, "{interaction}")?; - if let Some(name) = interaction.property_meta.map(|p| p.property.name()) - && interaction.span.is_some_and(|span| span.end()) - { - indentation_level = indentation_level.saturating_sub(1); - writeln!(f, "-- end testing '{name}'")?; + if indentation_level > 0 { + let padding = " ".repeat(indentation_level * PAD); + f.pad(&padding)?; + } + writeln!(f, "{interaction}")?; + if let Some(name) = interaction.property_meta.map(|p| p.property.name()) + && property.peek().is_none() + { + indentation_level = indentation_level.saturating_sub(1); + writeln!(f, "-- end testing '{name}'")?; + } } } @@ -431,24 +437,6 @@ impl Display for Fault { } } -#[derive(Debug, Clone, Copy)] -pub enum Span { - Start, - End, - // Both start and end - StartEnd, -} - -impl Span { - fn start(&self) -> bool { - matches!(self, Self::Start | Self::StartEnd) - } - - fn end(&self) -> bool { - matches!(self, Self::End | Self::StartEnd) - } -} - #[derive(Debug, Clone, Copy)] pub struct PropertyMetadata { pub property: PropertyDiscriminants, @@ -466,7 +454,6 @@ impl PropertyMetadata { } #[derive(Debug, Clone, derive_builder::Builder)] -#[builder(build_fn(validate = "Self::validate"))] pub struct Interaction { pub connection_index: usize, pub interaction: InteractionType, @@ -474,8 +461,6 @@ pub struct Interaction { pub ignore_error: bool, #[builder(setter(strip_option), default)] pub property_meta: Option, - #[builder(setter(strip_option), default)] - pub span: Option, /// 0 id means the ID was not set id: NonZeroUsize, } @@ -491,9 +476,6 @@ impl InteractionBuilder { if let Some(property_meta) = interaction.property_meta { builder.property_meta(property_meta); } - if let Some(span) = interaction.span { - builder.span(span); - } builder } @@ -507,19 +489,6 @@ impl InteractionBuilder { pub fn has_property_meta(&self) -> bool { self.property_meta.is_some() } - - fn validate(&self) -> Result<(), InteractionBuilderError> { - // Cannot have span and property_meta.extension being true at the same time - if let Some(property_meta) = self.property_meta.flatten() - && property_meta.extension - && self.span.flatten().is_some() - { - return Err(InteractionBuilderError::ValidationError( - "cannot have a span set with an extension query".to_string(), - )); - } - Ok(()) - } } impl Deref for Interaction { From c2be60b00758435347072040318f96381ec7716b Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Sat, 1 Nov 2025 17:44:44 -0300 Subject: [PATCH 16/16] add pragma to shrinking --- simulator/shrink/plan.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index d38c8c0fd..5a6ef5f83 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -227,6 +227,10 @@ impl InteractionPlan { | InteractionType::Query(Query::Commit(..)) | InteractionType::Query(Query::Rollback(..)) ); + let is_pragma = matches!( + &interaction.interaction, + InteractionType::Query(Query::Pragma(..)) + ); let skip_interaction = if let Some(property_meta) = interaction.property_meta { if matches!( @@ -255,7 +259,7 @@ impl InteractionPlan { }; (is_part_of_property || !skip_interaction) - && (is_fault || is_transaction || has_table) + && (is_fault || is_transaction || is_pragma || has_table) }; retain_map.push(retain); }