sim: add Fault::ReopenDatabase

Add fault type that reopens the DB and reconnects the connections.
It purposefully does not call conn.close(), as the default behavior
of that method is to checkpoint the database.

This easily exposes multiple manifestations of DB/WAL corruption caused by
this issue: https://github.com/tursodatabase/limbo/issues/1725

in fact, in my testing, every run fails when this fault is enabled. Hence I've
added the --disable-reopen-database flag if you want to try to find some other
bugs.

simulator/generation/plan.rs

@@ -273,12 +273,14 @@ impl Debug for Assertion {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub(crate) enum Fault {
     Disconnect,
+    ReopenDatabase,
 }
 
 impl Display for Fault {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Fault::Disconnect => write!(f, "DISCONNECT"),
+            Fault::ReopenDatabase => write!(f, "REOPEN_DATABASE"),
         }
     }
 }
@@ -634,6 +636,31 @@ impl Interaction {
                         }
                         env.connections[conn_index] = SimConnection::Disconnected;
                     }
+                    Fault::ReopenDatabase => {
+                        // 1. Close all connections without default checkpoint-on-close behavior
+                        // to expose bugs related to how we handle WAL
+                        let num_conns = env.connections.len();
+                        env.connections.clear();
+
+                        // 2. Re-open database
+                        let db_path = env.db_path.clone();
+                        let db = match limbo_core::Database::open_file(
+                            env.io.clone(),
+                            &db_path,
+                            false,
+                        ) {
+                            Ok(db) => db,
+                            Err(e) => {
+                                panic!("error opening simulator test file {:?}: {:?}", db_path, e);
+                            }
+                        };
+                        env.db = db;
+
+                        for _ in 0..num_conns {
+                            env.connections
+                                .push(SimConnection::LimboConnection(env.db.connect().unwrap()));
+                        }
+                    }
                 }
                 Ok(())
             }
@@ -674,8 +701,14 @@ fn random_create_index<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Option<
     )))
 }
 
-fn random_fault<R: rand::Rng>(_rng: &mut R, _env: &SimulatorEnv) -> Interactions {
-    Interactions::Fault(Fault::Disconnect)
+fn random_fault<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Interactions {
+    let faults = if env.opts.disable_reopen_database {
+        vec![Fault::Disconnect]
+    } else {
+        vec![Fault::Disconnect, Fault::ReopenDatabase]
+    };
+    let fault = faults[rng.gen_range(0..faults.len())].clone();
+    Interactions::Fault(fault)
 }
 
 impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions {

simulator/runner/cli.rs

@@ -82,6 +82,8 @@ pub struct SimulatorCLI {
         default_value_t = false
     )]
     pub disable_select_optimizer: bool,
+    #[clap(long, help = "disable Reopen-Database fault", default_value_t = false)]
+    pub disable_reopen_database: bool,
 }
 
 #[derive(Parser, Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord)]

simulator/runner/env.rs

@@ -20,6 +20,7 @@ pub(crate) struct SimulatorEnv {
     pub(crate) io: Arc<SimulatorIO>,
     pub(crate) db: Arc<Database>,
     pub(crate) rng: ChaCha8Rng,
+    pub(crate) db_path: String,
 }
 
 impl SimulatorEnv {
@@ -33,6 +34,7 @@ impl SimulatorEnv {
             io: self.io.clone(),
             db: self.db.clone(),
             rng: self.rng.clone(),
+            db_path: self.db_path.clone(),
         }
     }
 }
@@ -118,6 +120,7 @@ impl SimulatorEnv {
             page_size: 4096, // TODO: randomize this too
             max_interactions: rng.gen_range(cli_opts.minimum_tests..=cli_opts.maximum_tests),
             max_time_simulation: cli_opts.maximum_time,
+            disable_reopen_database: cli_opts.disable_reopen_database,
         };
 
         let io = Arc::new(SimulatorIO::new(seed, opts.page_size).unwrap());
@@ -150,6 +153,7 @@ impl SimulatorEnv {
             rng,
             io,
             db,
+            db_path: db_path.to_str().unwrap().to_string(),
         }
     }
 }
@@ -227,6 +231,7 @@ pub(crate) struct SimulatorOpts {
     pub(crate) disable_select_limit: bool,
     pub(crate) disable_delete_select: bool,
     pub(crate) disable_drop_select: bool,
+    pub(crate) disable_reopen_database: bool,
 
     pub(crate) max_interactions: usize,
     pub(crate) page_size: usize,