From 9aae1ff00a8177d2cb792ed2a32ab8ccd766a3a1 Mon Sep 17 00:00:00 2001
From: Jussi Saurio <jussi.saurio@gmail.com>
Date: Tue, 17 Jun 2025 11:51:28 +0300
Subject: [PATCH] sim: add Fault::ReopenDatabase

Add fault type that reopens the DB and reconnects the connections.
It purposefully does not call conn.close(), as the default behavior
of that method is to checkpoint the database.

This easily exposes multiple manifestations of DB/WAL corruption caused by
this issue: https://github.com/tursodatabase/limbo/issues/1725

in fact, in my testing, every run fails when this fault is enabled. Hence I've
added the --disable-reopen-database flag if you want to try to find some other
bugs.
---
 simulator/generation/plan.rs | 37 ++++++++++++++++++++++++++++++++++--
 simulator/runner/cli.rs      |  2 ++
 simulator/runner/env.rs      |  5 +++++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs
index ec6ee471e..5193feccf 100644
--- a/simulator/generation/plan.rs
+++ b/simulator/generation/plan.rs
@@ -273,12 +273,14 @@ impl Debug for Assertion {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub(crate) enum Fault {
     Disconnect,
+    ReopenDatabase,
 }
 
 impl Display for Fault {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Fault::Disconnect => write!(f, "DISCONNECT"),
+            Fault::ReopenDatabase => write!(f, "REOPEN_DATABASE"),
         }
     }
 }
@@ -634,6 +636,31 @@ impl Interaction {
                         }
                         env.connections[conn_index] = SimConnection::Disconnected;
                     }
+                    Fault::ReopenDatabase => {
+                        // 1. Close all connections without default checkpoint-on-close behavior
+                        // to expose bugs related to how we handle WAL
+                        let num_conns = env.connections.len();
+                        env.connections.clear();
+
+                        // 2. Re-open database
+                        let db_path = env.db_path.clone();
+                        let db = match limbo_core::Database::open_file(
+                            env.io.clone(),
+                            &db_path,
+                            false,
+                        ) {
+                            Ok(db) => db,
+                            Err(e) => {
+                                panic!("error opening simulator test file {:?}: {:?}", db_path, e);
+                            }
+                        };
+                        env.db = db;
+
+                        for _ in 0..num_conns {
+                            env.connections
+                                .push(SimConnection::LimboConnection(env.db.connect().unwrap()));
+                        }
+                    }
                 }
                 Ok(())
             }
@@ -674,8 +701,14 @@ fn random_create_index<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Option<
     )))
 }
 
-fn random_fault<R: rand::Rng>(_rng: &mut R, _env: &SimulatorEnv) -> Interactions {
-    Interactions::Fault(Fault::Disconnect)
+fn random_fault<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Interactions {
+    let faults = if env.opts.disable_reopen_database {
+        vec![Fault::Disconnect]
+    } else {
+        vec![Fault::Disconnect, Fault::ReopenDatabase]
+    };
+    let fault = faults[rng.gen_range(0..faults.len())].clone();
+    Interactions::Fault(fault)
 }
 
 impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions {
diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs
index c76d7367d..3b3bd6452 100644
--- a/simulator/runner/cli.rs
+++ b/simulator/runner/cli.rs
@@ -82,6 +82,8 @@ pub struct SimulatorCLI {
         default_value_t = false
     )]
     pub disable_select_optimizer: bool,
+    #[clap(long, help = "disable Reopen-Database fault", default_value_t = false)]
+    pub disable_reopen_database: bool,
 }
 
 #[derive(Parser, Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord)]
diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs
index 78ff3c605..a5616d2cc 100644
--- a/simulator/runner/env.rs
+++ b/simulator/runner/env.rs
@@ -20,6 +20,7 @@ pub(crate) struct SimulatorEnv {
     pub(crate) io: Arc<SimulatorIO>,
     pub(crate) db: Arc<Database>,
     pub(crate) rng: ChaCha8Rng,
+    pub(crate) db_path: String,
 }
 
 impl SimulatorEnv {
@@ -33,6 +34,7 @@ impl SimulatorEnv {
             io: self.io.clone(),
             db: self.db.clone(),
             rng: self.rng.clone(),
+            db_path: self.db_path.clone(),
         }
     }
 }
@@ -118,6 +120,7 @@ impl SimulatorEnv {
             page_size: 4096, // TODO: randomize this too
             max_interactions: rng.gen_range(cli_opts.minimum_tests..=cli_opts.maximum_tests),
             max_time_simulation: cli_opts.maximum_time,
+            disable_reopen_database: cli_opts.disable_reopen_database,
         };
 
         let io = Arc::new(SimulatorIO::new(seed, opts.page_size).unwrap());
@@ -150,6 +153,7 @@ impl SimulatorEnv {
             rng,
             io,
             db,
+            db_path: db_path.to_str().unwrap().to_string(),
         }
     }
 }
@@ -227,6 +231,7 @@ pub(crate) struct SimulatorOpts {
     pub(crate) disable_select_limit: bool,
     pub(crate) disable_delete_select: bool,
     pub(crate) disable_drop_select: bool,
+    pub(crate) disable_reopen_database: bool,
 
     pub(crate) max_interactions: usize,
     pub(crate) page_size: usize,