diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index ec6ee471e..5193feccf 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -273,12 +273,14 @@ impl Debug for Assertion { #[derive(Debug, Clone, Serialize, Deserialize)] pub(crate) enum Fault { Disconnect, + ReopenDatabase, } impl Display for Fault { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Fault::Disconnect => write!(f, "DISCONNECT"), + Fault::ReopenDatabase => write!(f, "REOPEN_DATABASE"), } } } @@ -634,6 +636,31 @@ impl Interaction { } env.connections[conn_index] = SimConnection::Disconnected; } + Fault::ReopenDatabase => { + // 1. Close all connections without default checkpoint-on-close behavior + // to expose bugs related to how we handle WAL + let num_conns = env.connections.len(); + env.connections.clear(); + + // 2. Re-open database + let db_path = env.db_path.clone(); + let db = match limbo_core::Database::open_file( + env.io.clone(), + &db_path, + false, + ) { + Ok(db) => db, + Err(e) => { + panic!("error opening simulator test file {:?}: {:?}", db_path, e); + } + }; + env.db = db; + + for _ in 0..num_conns { + env.connections + .push(SimConnection::LimboConnection(env.db.connect().unwrap())); + } + } } Ok(()) } @@ -674,8 +701,14 @@ fn random_create_index(rng: &mut R, env: &SimulatorEnv) -> Option< ))) } -fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Interactions { - Interactions::Fault(Fault::Disconnect) +fn random_fault(rng: &mut R, env: &SimulatorEnv) -> Interactions { + let faults = if env.opts.disable_reopen_database { + vec![Fault::Disconnect] + } else { + vec![Fault::Disconnect, Fault::ReopenDatabase] + }; + let fault = faults[rng.gen_range(0..faults.len())].clone(); + Interactions::Fault(fault) } impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index c76d7367d..3b3bd6452 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -82,6 +82,8 @@ pub struct SimulatorCLI { default_value_t = false )] pub disable_select_optimizer: bool, + #[clap(long, help = "disable Reopen-Database fault", default_value_t = false)] + pub disable_reopen_database: bool, } #[derive(Parser, Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord)] diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs index 78ff3c605..a5616d2cc 100644 --- a/simulator/runner/env.rs +++ b/simulator/runner/env.rs @@ -20,6 +20,7 @@ pub(crate) struct SimulatorEnv { pub(crate) io: Arc, pub(crate) db: Arc, pub(crate) rng: ChaCha8Rng, + pub(crate) db_path: String, } impl SimulatorEnv { @@ -33,6 +34,7 @@ impl SimulatorEnv { io: self.io.clone(), db: self.db.clone(), rng: self.rng.clone(), + db_path: self.db_path.clone(), } } } @@ -118,6 +120,7 @@ impl SimulatorEnv { page_size: 4096, // TODO: randomize this too max_interactions: rng.gen_range(cli_opts.minimum_tests..=cli_opts.maximum_tests), max_time_simulation: cli_opts.maximum_time, + disable_reopen_database: cli_opts.disable_reopen_database, }; let io = Arc::new(SimulatorIO::new(seed, opts.page_size).unwrap()); @@ -150,6 +153,7 @@ impl SimulatorEnv { rng, io, db, + db_path: db_path.to_str().unwrap().to_string(), } } } @@ -227,6 +231,7 @@ pub(crate) struct SimulatorOpts { pub(crate) disable_select_limit: bool, pub(crate) disable_delete_select: bool, pub(crate) disable_drop_select: bool, + pub(crate) disable_reopen_database: bool, pub(crate) max_interactions: usize, pub(crate) page_size: usize,