aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-02-23 17:05:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'simulator: updates to bug base, refactors' from Alperen Keleş

Browse Source 
- Fixes https://github.com/tursodatabase/limbo/issues/1200
- Fixes https://github.com/tursodatabase/limbo/issues/1306
- Adds detailed run information to bug base
- Fixes a bug in the differential testing with SQLite

Closes #1314
This commit is contained in:
Pekka Enberg
2025-04-11 18:52:28 +03:00
parent 1fd9a7ad9c 7a8fb34004
commit df11562f69
11 changed files with 517 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.lock generated
View File

@@ -344,6 +344,7 @@ dependencies = [
"iana-time-zone",
"js-sys",
"num-traits",
"serde",
"wasm-bindgen",
"windows-link",
]
@@ -1866,6 +1867,7 @@ name = "limbo_sim"
version = "0.0.19-pre.4"
dependencies = [
"anarchist-readable-name-generator-lib",
"chrono",
"clap",
"dirs 6.0.0",
"env_logger 0.10.2",
@@ -2297,9 +2299,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.21.1"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "onig"

1
simulator/Cargo.toml
View File

@@ -31,3 +31,4 @@ serde_json = { version = "1.0" }
notify = "8.0.0"
rusqlite = { version = "0.34", features = ["bundled"] }
dirs = "6.0.0"
chrono = { version = "0.4.40", features = ["serde"] }

23
simulator/generation/plan.rs
View File

@@ -11,7 +11,7 @@ use crate::{
},
table::Value,
},
runner::env::{SimConnection, SimulatorEnvTrait},
runner::env::SimConnection,
SimulatorEnv,
};
@@ -238,7 +238,7 @@ impl Display for Interaction {
}
}
type AssertionFunc = dyn Fn(&Vec<ResultSet>, &dyn SimulatorEnvTrait) -> Result<bool>;
type AssertionFunc = dyn Fn(&Vec<ResultSet>, &SimulatorEnv) -> Result<bool>;
enum AssertionAST {
Pick(),
@@ -523,7 +523,7 @@ impl Interaction {
pub(crate) fn execute_assertion(
&self,
stack: &Vec<ResultSet>,
env: &impl SimulatorEnvTrait,
env: &SimulatorEnv,
) -> Result<()> {
match self {
Self::Query(_) => {
@@ -554,7 +554,7 @@ impl Interaction {
pub(crate) fn execute_assumption(
&self,
stack: &Vec<ResultSet>,
env: &dyn SimulatorEnvTrait,
env: &SimulatorEnv,
) -> Result<()> {
match self {
Self::Query(_) => {
@@ -596,15 +596,12 @@ impl Interaction {
Self::Fault(fault) => {
match fault {
Fault::Disconnect => {
match env.connections[conn_index] {
SimConnection::Connected(ref mut conn) => {
conn.close()?;
}
SimConnection::Disconnected => {
return Err(limbo_core::LimboError::InternalError(
"Tried to disconnect a disconnected connection".to_string(),
));
}
if env.connections[conn_index].is_connected() {
env.connections[conn_index].disconnect();
} else {
return Err(limbo_core::LimboError::InternalError(
"connection already disconnected".into(),
));
}
env.connections[conn_index] = SimConnection::Disconnected;
}

40
simulator/generation/property.rs
View File

@@ -9,7 +9,7 @@ use crate::{
},
table::Value,
},
runner::env::{SimulatorEnv, SimulatorEnvTrait},
runner::env::SimulatorEnv,
};
use super::{
@@ -170,8 +170,8 @@ impl Property {
message: format!("table {} exists", insert.table()),
func: Box::new({
let table_name = table.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table_name))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table_name))
}
}),
});
@@ -182,7 +182,7 @@ impl Property {
row.iter().map(|v| v.to_string()).collect::<Vec<String>>(),
insert.table(),
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let rows = stack.last().unwrap();
match rows {
Ok(rows) => Ok(rows.iter().any(|r| r == &row)),
@@ -206,8 +206,8 @@ impl Property {
let assumption = Interaction::Assumption(Assertion {
message: "Double-Create-Failure should not be called on an existing table"
.to_string(),
func: Box::new(move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(!env.tables().iter().any(|t| t.name == table_name))
func: Box::new(move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(!env.tables.iter().any(|t| t.name == table_name))
}),
});
@@ -220,11 +220,11 @@ impl Property {
message:
"creating two tables with the name should result in a failure for the second query"
.to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let last = stack.last().unwrap();
match last {
Ok(_) => Ok(false),
Err(e) => Ok(e.to_string().contains(&format!("Table {table_name} already exists"))),
Err(e) => Ok(e.to_string().to_lowercase().contains(&format!("table {table_name} already exists"))),
}
}),
});
@@ -245,8 +245,8 @@ impl Property {
message: format!("table {} exists", table_name),
func: Box::new({
let table_name = table_name.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table_name))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table_name))
}
}),
});
@@ -257,7 +257,7 @@ impl Property {
let assertion = Interaction::Assertion(Assertion {
message: "select query should respect the limit clause".to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let last = stack.last().unwrap();
match last {
Ok(rows) => Ok(limit >= rows.len()),
@@ -281,8 +281,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
}
}),
});
@@ -292,7 +292,7 @@ impl Property {
"select '{}' should return no values for table '{}'",
predicate, table,
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let rows = stack.last().unwrap();
match rows {
Ok(rows) => Ok(rows.is_empty()),
@@ -332,8 +332,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
}
}),
});
@@ -345,7 +345,7 @@ impl Property {
"select query should result in an error for table '{}'",
table
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let last = stack.last().unwrap();
match last {
Ok(_) => Ok(false),
@@ -377,8 +377,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
}
}),
});
@@ -401,7 +401,7 @@ impl Property {
let assertion = Interaction::Assertion(Assertion {
message: "select queries should return the same amount of results".to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let select_star = stack.last().unwrap();
let select_predicate = stack.get(stack.len() - 2).unwrap();
match (select_predicate, select_star) {

134
simulator/main.rs
View File

@@ -5,7 +5,7 @@ use generation::ArbitraryFrom;
use notify::event::{DataChange, ModifyKind};
use notify::{EventKind, RecursiveMode, Watcher};
use rand::prelude::*;
use runner::bugbase::{Bug, BugBase};
use runner::bugbase::{Bug, BugBase, LoadedBug};
use runner::cli::SimulatorCLI;
use runner::env::SimulatorEnv;
use runner::execution::{execute_plans, Execution, ExecutionHistory, ExecutionResult};
@@ -28,6 +28,7 @@ struct Paths {
history: PathBuf,
doublecheck_db: PathBuf,
shrunk_db: PathBuf,
diff_db: PathBuf,
}
impl Paths {
@@ -40,6 +41,7 @@ impl Paths {
history: PathBuf::from(output_dir).join("history.txt"),
doublecheck_db: PathBuf::from(output_dir).join("double.db"),
shrunk_db: PathBuf::from(output_dir).join("shrunk.db"),
diff_db: PathBuf::from(output_dir).join("diff.db"),
}
}
}
@@ -52,7 +54,6 @@ fn main() -> Result<(), String> {
let mut bugbase = BugBase::load().map_err(|e| format!("{:?}", e))?;
banner();
// let paths = Paths::new(&output_dir, cli_opts.doublecheck);
let last_execution = Arc::new(Mutex::new(Execution::new(0, 0, 0)));
let (seed, env, plans) = setup_simulation(&mut bugbase, &cli_opts, |p| &p.plan, |p| &p.db);
@@ -66,8 +67,18 @@ fn main() -> Result<(), String> {
if cli_opts.watch {
watch_mode(seed, &cli_opts, &paths, last_execution.clone()).unwrap();
} else if cli_opts.differential {
differential_testing(env, plans, last_execution.clone())
return Ok(());
}
let result = if cli_opts.differential {
differential_testing(
seed,
&mut bugbase,
&cli_opts,
&paths,
plans,
last_execution.clone(),
)
} else {
run_simulator(
seed,
@@ -77,13 +88,14 @@ fn main() -> Result<(), String> {
env,
plans,
last_execution.clone(),
);
}
)
};
// Print the seed, the locations of the database and the plan file at the end again for easily accessing them.
println!("seed: {}", seed);
println!("path: {}", paths.base.display());
Ok(())
result
}
fn watch_mode(
@@ -120,7 +132,7 @@ fn watch_mode(
i.shadow(&mut env);
});
});
let env = Arc::new(Mutex::new(env.clone()));
let env = Arc::new(Mutex::new(env.clone_without_connections()));
watch::run_simulation(env, &mut [plan], last_execution.clone())
}),
last_execution.clone(),
@@ -133,7 +145,6 @@ fn watch_mode(
SandboxedResult::Panicked { error, .. }
| SandboxedResult::FoundBug { error, .. } => {
log::error!("simulation failed: '{}'", error);
println!("simulation failed: '{}'", error);
}
}
}
@@ -153,7 +164,7 @@ fn run_simulator(
env: SimulatorEnv,
plans: Vec<InteractionPlan>,
last_execution: Arc<Mutex<Execution>>,
) {
) -> Result<(), String> {
std::panic::set_hook(Box::new(move |info| {
log::error!("panic occurred");
@@ -181,15 +192,15 @@ fn run_simulator(
if cli_opts.doublecheck {
let env = SimulatorEnv::new(seed, cli_opts, &paths.doublecheck_db);
let env = Arc::new(Mutex::new(env));
doublecheck(env, paths, &plans, last_execution.clone(), result);
doublecheck(env, paths, &plans, last_execution.clone(), result)
} else {
// No doublecheck, run shrinking if panicking or found a bug.
match &result {
SandboxedResult::Correct => {
log::info!("simulation succeeded");
println!("simulation succeeded");
// remove the bugbase entry
bugbase.remove_bug(seed).unwrap();
bugbase.mark_successful_run(seed, cli_opts).unwrap();
Ok(())
}
SandboxedResult::Panicked {
error,
@@ -217,8 +228,6 @@ fn run_simulator(
}
log::error!("simulation failed: '{}'", error);
println!("simulation failed: '{}'", error);
log::info!("Starting to shrink");
let shrunk_plans = plans
@@ -260,12 +269,21 @@ fn run_simulator(
) => {
if e1 != e2 {
log::error!("shrinking failed, the error was not properly reproduced");
bugbase.add_bug(seed, plans[0].clone()).unwrap();
bugbase
.add_bug(seed, plans[0].clone(), Some(error.clone()), cli_opts)
.unwrap();
Err(format!("failed with error: '{}'", error))
} else {
log::info!("shrinking succeeded");
println!("shrinking succeeded");
log::info!(
"shrinking succeeded, reduced the plan from {} to {}",
plans[0].plan.len(),
shrunk_plans[0].plan.len()
);
// Save the shrunk database
bugbase.add_bug(seed, shrunk_plans[0].clone()).unwrap();
bugbase
.add_bug(seed, shrunk_plans[0].clone(), Some(e1.clone()), cli_opts)
.unwrap();
Err(format!("failed with error: '{}'", e1))
}
}
(_, SandboxedResult::Correct) => {
@@ -273,7 +291,10 @@ fn run_simulator(
}
_ => {
log::error!("shrinking failed, the error was not properly reproduced");
bugbase.add_bug(seed, plans[0].clone()).unwrap();
bugbase
.add_bug(seed, plans[0].clone(), Some(error.clone()), cli_opts)
.unwrap();
Err(format!("failed with error: '{}'", error))
}
}
}
@@ -287,7 +308,7 @@ fn doublecheck(
plans: &[InteractionPlan],
last_execution: Arc<Mutex<Execution>>,
result: SandboxedResult,
) {
) -> Result<(), String> {
// Run the simulation again
let result2 = SandboxedResult::from(
std::panic::catch_unwind(|| {
@@ -299,29 +320,47 @@ fn doublecheck(
match (result, result2) {
(SandboxedResult::Correct, SandboxedResult::Panicked { .. }) => {
log::error!("doublecheck failed! first run succeeded, but second run panicked.");
Err("doublecheck failed! first run succeeded, but second run panicked.".to_string())
}
(SandboxedResult::FoundBug { .. }, SandboxedResult::Panicked { .. }) => {
log::error!(
"doublecheck failed! first run failed an assertion, but second run panicked."
);
Err(
"doublecheck failed! first run failed an assertion, but second run panicked."
.to_string(),
)
}
(SandboxedResult::Panicked { .. }, SandboxedResult::Correct) => {
log::error!("doublecheck failed! first run panicked, but second run succeeded.");
Err("doublecheck failed! first run panicked, but second run succeeded.".to_string())
}
(SandboxedResult::Panicked { .. }, SandboxedResult::FoundBug { .. }) => {
log::error!(
"doublecheck failed! first run panicked, but second run failed an assertion."
);
Err(
"doublecheck failed! first run panicked, but second run failed an assertion."
.to_string(),
)
}
(SandboxedResult::Correct, SandboxedResult::FoundBug { .. }) => {
log::error!(
"doublecheck failed! first run succeeded, but second run failed an assertion."
);
Err(
"doublecheck failed! first run succeeded, but second run failed an assertion."
.to_string(),
)
}
(SandboxedResult::FoundBug { .. }, SandboxedResult::Correct) => {
log::error!(
"doublecheck failed! first run failed an assertion, but second run succeeded."
);
Err(
"doublecheck failed! first run failed an assertion, but second run succeeded."
.to_string(),
)
}
(SandboxedResult::Correct, SandboxedResult::Correct)
| (SandboxedResult::FoundBug { .. }, SandboxedResult::FoundBug { .. })
@@ -331,33 +370,62 @@ fn doublecheck(
let doublecheck_db_bytes = std::fs::read(&paths.doublecheck_db).unwrap();
if db_bytes != doublecheck_db_bytes {
log::error!("doublecheck failed! database files are different.");
log::error!("current: {}", paths.db.display());
log::error!("doublecheck: {}", paths.doublecheck_db.display());
Err(
"doublecheck failed! database files are different, check binary diffs for more details.".to_string()
)
} else {
log::info!("doublecheck succeeded! database files are the same.");
println!("doublecheck succeeded! database files are the same.");
Ok(())
}
}
}
}
fn differential_testing(
env: SimulatorEnv,
seed: u64,
bugbase: &mut BugBase,
cli_opts: &SimulatorCLI,
paths: &Paths,
plans: Vec<InteractionPlan>,
last_execution: Arc<Mutex<Execution>>,
) {
let env = Arc::new(Mutex::new(env));
) -> Result<(), String> {
let env = Arc::new(Mutex::new(SimulatorEnv::new(seed, cli_opts, &paths.db)));
let rusqlite_env = Arc::new(Mutex::new(SimulatorEnv::new(
seed,
cli_opts,
&paths.diff_db,
)));
let result = SandboxedResult::from(
std::panic::catch_unwind(|| {
let plan = plans[0].clone();
differential::run_simulation(env, &mut [plan], last_execution.clone())
differential::run_simulation(
env,
rusqlite_env,
&|| rusqlite::Connection::open(paths.diff_db.clone()).unwrap(),
&mut [plan],
last_execution.clone(),
)
}),
last_execution.clone(),
);
if let SandboxedResult::Correct = result {
log::info!("simulation succeeded");
println!("simulation succeeded");
} else {
log::error!("simulation failed");
println!("simulation failed");
match result {
SandboxedResult::Correct => {
log::info!("simulation succeeded, output of Limbo conforms to SQLite");
println!("simulation succeeded, output of Limbo conforms to SQLite");
Ok(())
}
SandboxedResult::Panicked { error, .. } | SandboxedResult::FoundBug { error, .. } => {
log::error!("simulation failed: '{}'", error);
bugbase
.add_bug(seed, plans[0].clone(), Some(error.clone()), cli_opts)
.unwrap();
Err(format!("simulation failed: '{}'", error))
}
}
}
@@ -433,12 +501,14 @@ fn setup_simulation(
let env = SimulatorEnv::new(bug.seed(), cli_opts, db_path(&paths));
let plan = match bug {
Bug::Loaded { plan, .. } => plan.clone(),
Bug::Loaded(LoadedBug { plan, .. }) => plan.clone(),
Bug::Unloaded { seed } => {
let seed = *seed;
bugbase
.load_bug(seed)
.unwrap_or_else(|_| panic!("could not load bug '{}' in bug base", seed))
.plan
.clone()
}
};

222
simulator/runner/bugbase.rs
View File

@@ -3,15 +3,44 @@ use std::{
io::{self, Write},
path::PathBuf,
process::Command,
time::SystemTime,
};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use crate::{InteractionPlan, Paths};
use super::cli::SimulatorCLI;
/// A bug is a run that has been identified as buggy.
#[derive(Clone)]
pub(crate) enum Bug {
Unloaded { seed: u64 },
Loaded { seed: u64, plan: InteractionPlan },
Loaded(LoadedBug),
}
#[derive(Clone)]
pub struct LoadedBug {
/// The seed of the bug.
pub seed: u64,
/// The plan of the bug.
pub plan: InteractionPlan,
/// The runs of the bug.
pub runs: Vec<BugRun>,
}
#[derive(Clone, Serialize, Deserialize)]
pub(crate) struct BugRun {
/// Commit hash of the current version of Limbo.
hash: String,
/// Timestamp of the run.
#[serde(with = "chrono::serde::ts_seconds")]
timestamp: DateTime<Utc>,
/// Error message of the run.
error: Option<String>,
/// Options
cli_options: SimulatorCLI,
}
impl Bug {
@@ -27,7 +56,7 @@ impl Bug {
pub(crate) fn seed(&self) -> u64 {
match self {
Bug::Unloaded { seed } => *seed,
Bug::Loaded { seed, .. } => *seed,
Bug::Loaded(LoadedBug { seed, .. }) => *seed,
}
}
}
@@ -77,6 +106,36 @@ impl BugBase {
.or(Err("should be able to get current directory".to_string()))?,
];
for path in &potential_paths {
let path = path.join(".bugbase");
if path.exists() {
return BugBase::new(path);
}
}
for path in potential_paths {
let path = path.join(".bugbase");
if std::fs::create_dir_all(&path).is_ok() {
log::info!("bug base created at {}", path.display());
return BugBase::new(path);
}
}
Err("failed to create bug base".to_string())
}
/// Load the bug base from one of the potential paths.
pub(crate) fn interactive_load() -> Result<Self, String> {
let potential_paths = vec![
// limbo project directory
BugBase::get_limbo_project_dir()?,
// home directory
dirs::home_dir().ok_or("should be able to get home directory".to_string())?,
// current directory
std::env::current_dir()
.or(Err("should be able to get current directory".to_string()))?,
];
for path in potential_paths {
let path = path.join(".bugbase");
if path.exists() {
@@ -119,14 +178,41 @@ impl BugBase {
}
/// Add a new bug to the bug base.
pub(crate) fn add_bug(&mut self, seed: u64, plan: InteractionPlan) -> Result<(), String> {
pub(crate) fn add_bug(
&mut self,
seed: u64,
plan: InteractionPlan,
error: Option<String>,
cli_options: &SimulatorCLI,
) -> Result<(), String> {
log::debug!("adding bug with seed {}", seed);
if self.bugs.contains_key(&seed) {
return Err(format!("Bug with hash {} already exists", seed));
let bug = self.get_bug(seed);
if bug.is_some() {
let mut bug = self.load_bug(seed)?;
bug.plan = plan.clone();
bug.runs.push(BugRun {
hash: Self::get_current_commit_hash()?,
timestamp: SystemTime::now().into(),
error,
cli_options: cli_options.clone(),
});
self.bugs.insert(seed, Bug::Loaded(bug.clone()));
} else {
let bug = LoadedBug {
seed,
plan: plan.clone(),
runs: vec![BugRun {
hash: Self::get_current_commit_hash()?,
timestamp: SystemTime::now().into(),
error,
cli_options: cli_options.clone(),
}],
};
self.bugs.insert(seed, Bug::Loaded(bug.clone()));
}
self.save_bug(seed, &plan)?;
self.bugs.insert(seed, Bug::Loaded { seed, plan });
Ok(())
// Save the bug to the bug base.
self.save_bug(seed)
}
/// Get a bug from the bug base.
@@ -135,36 +221,48 @@ impl BugBase {
}
/// Save a bug to the bug base.
pub(crate) fn save_bug(&self, seed: u64, plan: &InteractionPlan) -> Result<(), String> {
let bug_path = self.path.join(seed.to_string());
std::fs::create_dir_all(&bug_path)
.or(Err("should be able to create bug directory".to_string()))?;
fn save_bug(&self, seed: u64) -> Result<(), String> {
let bug = self.get_bug(seed);
let seed_path = bug_path.join("seed.txt");
std::fs::write(&seed_path, seed.to_string())
.or(Err("should be able to write seed file".to_string()))?;
match bug {
None | Some(Bug::Unloaded { .. }) => {
unreachable!("save should only be called within add_bug");
}
Some(Bug::Loaded(bug)) => {
let bug_path = self.path.join(seed.to_string());
std::fs::create_dir_all(&bug_path)
.or(Err("should be able to create bug directory".to_string()))?;
// At some point we might want to save the commit hash of the current
// version of Limbo.
// let commit_hash = Self::get_current_commit_hash()?;
// let commit_hash_path = bug_path.join("commit_hash.txt");
// std::fs::write(&commit_hash_path, commit_hash)
// .or(Err("should be able to write commit hash file".to_string()))?;
let seed_path = bug_path.join("seed.txt");
std::fs::write(&seed_path, seed.to_string())
.or(Err("should be able to write seed file".to_string()))?;
let plan_path = bug_path.join("plan.json");
std::fs::write(
&plan_path,
serde_json::to_string(plan).or(Err("should be able to serialize plan".to_string()))?,
)
.or(Err("should be able to write plan file".to_string()))?;
let plan_path = bug_path.join("plan.json");
std::fs::write(
&plan_path,
serde_json::to_string_pretty(&bug.plan)
.or(Err("should be able to serialize plan".to_string()))?,
)
.or(Err("should be able to write plan file".to_string()))?;
let readable_plan_path = bug_path.join("plan.sql");
std::fs::write(&readable_plan_path, bug.plan.to_string())
.or(Err("should be able to write readable plan file".to_string()))?;
let runs_path = bug_path.join("runs.json");
std::fs::write(
&runs_path,
serde_json::to_string_pretty(&bug.runs)
.or(Err("should be able to serialize runs".to_string()))?,
)
.or(Err("should be able to write runs file".to_string()))?;
}
}
let readable_plan_path = bug_path.join("plan.sql");
std::fs::write(&readable_plan_path, plan.to_string())
.or(Err("should be able to write readable plan file".to_string()))?;
Ok(())
}
pub(crate) fn load_bug(&mut self, seed: u64) -> Result<InteractionPlan, String> {
pub(crate) fn load_bug(&mut self, seed: u64) -> Result<LoadedBug, String> {
let seed_match = self.bugs.get(&seed);
match seed_match {
@@ -176,30 +274,60 @@ impl BugBase {
let plan: InteractionPlan = serde_json::from_str(&plan)
.or(Err("should be able to deserialize plan".to_string()))?;
let bug = Bug::Loaded {
let runs =
std::fs::read_to_string(self.path.join(seed.to_string()).join("runs.json"))
.or(Err("should be able to read runs file".to_string()))?;
let runs: Vec<BugRun> = serde_json::from_str(&runs)
.or(Err("should be able to deserialize runs".to_string()))?;
let bug = LoadedBug {
seed,
plan: plan.clone(),
runs,
};
self.bugs.insert(seed, bug);
self.bugs.insert(seed, Bug::Loaded(bug.clone()));
log::debug!("Loaded bug with seed {}", seed);
Ok(plan)
Ok(bug)
}
Some(Bug::Loaded { plan, .. }) => {
Some(Bug::Loaded(bug)) => {
log::warn!(
"Bug with seed {} is already loaded, returning the existing plan",
seed
);
Ok(plan.clone())
Ok(bug.clone())
}
}
}
pub(crate) fn remove_bug(&mut self, seed: u64) -> Result<(), String> {
self.bugs.remove(&seed);
std::fs::remove_dir_all(self.path.join(seed.to_string()))
.or(Err("should be able to remove bug directory".to_string()))?;
pub(crate) fn mark_successful_run(
&mut self,
seed: u64,
cli_options: &SimulatorCLI,
) -> Result<(), String> {
let bug = self.get_bug(seed);
match bug {
None => {
log::debug!("removing bug base entry for {}", seed);
std::fs::remove_dir_all(self.path.join(seed.to_string()))
.or(Err("should be able to remove bug directory".to_string()))?;
}
Some(_) => {
let mut bug = self.load_bug(seed)?;
bug.runs.push(BugRun {
hash: Self::get_current_commit_hash()?,
timestamp: SystemTime::now().into(),
error: None,
cli_options: cli_options.clone(),
});
self.bugs.insert(seed, Bug::Loaded(bug.clone()));
// Save the bug to the bug base.
self.save_bug(seed)
.or(Err("should be able to save bug".to_string()))?;
log::debug!("Updated bug with seed {}", seed);
}
}
log::debug!("Removed bug with seed {}", seed);
Ok(())
}
}
@@ -223,6 +351,18 @@ impl BugBase {
}
impl BugBase {
pub(crate) fn get_current_commit_hash() -> Result<String, String> {
let output = Command::new("git")
.args(["rev-parse", "HEAD"])
.output()
.or(Err("should be able to get the commit hash".to_string()))?;
let commit_hash = String::from_utf8(output.stdout)
.or(Err("commit hash should be valid utf8".to_string()))?
.trim()
.to_string();
Ok(commit_hash)
}
pub(crate) fn get_limbo_project_dir() -> Result<PathBuf, String> {
Ok(PathBuf::from(
String::from_utf8(

3
simulator/runner/cli.rs
View File

@@ -1,6 +1,7 @@
use clap::{command, Parser};
use serde::{Deserialize, Serialize};
#[derive(Parser)]
#[derive(Parser, Clone, Serialize, Deserialize)]
#[command(name = "limbo-simulator")]
#[command(author, version, about, long_about = None)]
pub struct SimulatorCLI {

238
simulator/runner/differential.rs
View File

@@ -5,54 +5,20 @@ use crate::{
pick_index,
plan::{Interaction, InteractionPlanState, ResultSet},
},
model::{
query::Query,
table::{Table, Value},
},
model::{query::Query, table::Value},
runner::execution::ExecutionContinuation,
InteractionPlan,
};
use super::{
env::{ConnectionTrait, SimConnection, SimulatorEnv, SimulatorEnvTrait},
env::{SimConnection, SimulatorEnv},
execution::{execute_interaction, Execution, ExecutionHistory, ExecutionResult},
};
pub(crate) struct SimulatorEnvRusqlite {
pub(crate) tables: Vec<Table>,
pub(crate) connections: Vec<RusqliteConnection>,
}
pub(crate) enum RusqliteConnection {
Connected(rusqlite::Connection),
Disconnected,
}
impl ConnectionTrait for RusqliteConnection {
fn is_connected(&self) -> bool {
match self {
RusqliteConnection::Connected(_) => true,
RusqliteConnection::Disconnected => false,
}
}
fn disconnect(&mut self) {
*self = RusqliteConnection::Disconnected;
}
}
impl SimulatorEnvTrait for SimulatorEnvRusqlite {
fn tables(&self) -> &Vec<Table> {
&self.tables
}
fn tables_mut(&mut self) -> &mut Vec<Table> {
&mut self.tables
}
}
pub(crate) fn run_simulation(
env: Arc<Mutex<SimulatorEnv>>,
rusqlite_env: Arc<Mutex<SimulatorEnv>>,
rusqlite_conn: &dyn Fn() -> rusqlite::Connection,
plans: &mut [InteractionPlan],
last_execution: Arc<Mutex<Execution>>,
) -> ExecutionResult {
@@ -66,14 +32,7 @@ pub(crate) fn run_simulation(
secondary_pointer: 0,
})
.collect::<Vec<_>>();
let env = env.lock().unwrap();
let rusqlite_env = SimulatorEnvRusqlite {
tables: env.tables.clone(),
connections: (0..env.connections.len())
.map(|_| RusqliteConnection::Connected(rusqlite::Connection::open_in_memory().unwrap()))
.collect::<Vec<_>>(),
};
let mut rusqlite_states = plans
.iter()
.map(|_| InteractionPlanState {
@@ -84,16 +43,15 @@ pub(crate) fn run_simulation(
.collect::<Vec<_>>();
let result = execute_plans(
Arc::new(Mutex::new(env.clone())),
env,
rusqlite_env,
rusqlite_conn,
plans,
&mut states,
&mut rusqlite_states,
last_execution,
);
env.io.print_stats();
log::info!("Simulation completed");
result
@@ -148,7 +106,8 @@ fn execute_query_rusqlite(
pub(crate) fn execute_plans(
env: Arc<Mutex<SimulatorEnv>>,
mut rusqlite_env: SimulatorEnvRusqlite,
rusqlite_env: Arc<Mutex<SimulatorEnv>>,
rusqlite_conn: &dyn Fn() -> rusqlite::Connection,
plans: &mut [InteractionPlan],
states: &mut [InteractionPlanState],
rusqlite_states: &mut [InteractionPlanState],
@@ -158,6 +117,8 @@ pub(crate) fn execute_plans(
let now = std::time::Instant::now();
let mut env = env.lock().unwrap();
let mut rusqlite_env = rusqlite_env.lock().unwrap();
for _tick in 0..env.opts.ticks {
// Pick the connection to interact with
let connection_index = pick_index(env.connections.len(), &mut env.rng);
@@ -176,6 +137,7 @@ pub(crate) fn execute_plans(
match execute_plan(
&mut env,
&mut rusqlite_env,
rusqlite_conn,
connection_index,
plans,
states,
@@ -202,13 +164,15 @@ pub(crate) fn execute_plans(
fn execute_plan(
env: &mut SimulatorEnv,
rusqlite_env: &mut SimulatorEnvRusqlite,
rusqlite_env: &mut SimulatorEnv,
rusqlite_conn: &dyn Fn() -> rusqlite::Connection,
connection_index: usize,
plans: &mut [InteractionPlan],
states: &mut [InteractionPlanState],
rusqlite_states: &mut [InteractionPlanState],
) -> limbo_core::Result<()> {
let connection = &env.connections[connection_index];
let rusqlite_connection = &rusqlite_env.connections[connection_index];
let plan = &mut plans[connection_index];
let state = &mut states[connection_index];
let rusqlite_state = &mut rusqlite_states[connection_index];
@@ -218,83 +182,141 @@ fn execute_plan(
let interaction = &plan.plan[state.interaction_pointer].interactions()[state.secondary_pointer];
if let SimConnection::Disconnected = connection {
log::debug!("connecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect().unwrap());
} else {
let limbo_result =
execute_interaction(env, connection_index, interaction, &mut state.stack);
let ruqlite_result = execute_interaction_rusqlite(
rusqlite_env,
connection_index,
interaction,
&mut rusqlite_state.stack,
);
match (connection, rusqlite_connection) {
(SimConnection::Disconnected, SimConnection::Disconnected) => {
log::debug!("connecting {}", connection_index);
env.connections[connection_index] =
SimConnection::LimboConnection(env.db.connect().unwrap());
rusqlite_env.connections[connection_index] =
SimConnection::SQLiteConnection(rusqlite_conn());
}
(SimConnection::LimboConnection(_), SimConnection::SQLiteConnection(_)) => {
let limbo_result =
execute_interaction(env, connection_index, interaction, &mut state.stack);
let ruqlite_result = execute_interaction_rusqlite(
rusqlite_env,
connection_index,
interaction,
&mut rusqlite_state.stack,
);
match (limbo_result, ruqlite_result) {
(Ok(next_execution), Ok(next_execution_rusqlite)) => {
if next_execution != next_execution_rusqlite {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
match (limbo_result, ruqlite_result) {
(Ok(next_execution), Ok(next_execution_rusqlite)) => {
if next_execution != next_execution_rusqlite {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
log::debug!("connection {} processed", connection_index);
// Move to the next interaction or property
match next_execution {
ExecutionContinuation::NextInteraction => {
if state.secondary_pointer + 1
>= plan.plan[state.interaction_pointer].interactions().len()
{
// If we have reached the end of the interactions for this property, move to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
} else {
// Otherwise, move to the next interaction
state.secondary_pointer += 1;
let limbo_values = state.stack.last();
let rusqlite_values = rusqlite_state.stack.last();
match (limbo_values, rusqlite_values) {
(Some(limbo_values), Some(rusqlite_values)) => {
match (limbo_values, rusqlite_values) {
(Ok(limbo_values), Ok(rusqlite_values)) => {
if limbo_values != rusqlite_values {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
}
(Err(limbo_err), Err(rusqlite_err)) => {
log::warn!(
"limbo and rusqlite both fail, requires manual check"
);
log::warn!("limbo error {}", limbo_err);
log::warn!("rusqlite error {}", rusqlite_err);
}
(Ok(limbo_result), Err(rusqlite_err)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo values {:?}", limbo_result);
log::error!("rusqlite error {}", rusqlite_err);
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
(Err(limbo_err), Ok(_)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo error {}", limbo_err);
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
}
}
(None, None) => {}
_ => {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
}
ExecutionContinuation::NextProperty => {
// Skip to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
// Move to the next interaction or property
match next_execution {
ExecutionContinuation::NextInteraction => {
if state.secondary_pointer + 1
>= plan.plan[state.interaction_pointer].interactions().len()
{
// If we have reached the end of the interactions for this property, move to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
} else {
// Otherwise, move to the next interaction
state.secondary_pointer += 1;
}
}
ExecutionContinuation::NextProperty => {
// Skip to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
}
}
}
}
(Err(err), Ok(_)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo error {}", err);
return Err(err);
}
(Ok(_), Err(err)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("rusqlite error {}", err);
return Err(err);
}
(Err(err), Err(err_rusqlite)) => {
log::error!("limbo and rusqlite both fail, requires manual check");
log::error!("limbo error {}", err);
log::error!("rusqlite error {}", err_rusqlite);
return Err(err);
(Err(err), Ok(_)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo error {}", err);
return Err(err);
}
(Ok(val), Err(err)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo {:?}", val);
log::error!("rusqlite error {}", err);
return Err(err);
}
(Err(err), Err(err_rusqlite)) => {
log::error!("limbo and rusqlite both fail, requires manual check");
log::error!("limbo error {}", err);
log::error!("rusqlite error {}", err_rusqlite);
return Err(err);
}
}
}
_ => unreachable!("{} vs {}", connection, rusqlite_connection),
}
Ok(())
}
fn execute_interaction_rusqlite(
env: &mut SimulatorEnvRusqlite,
env: &mut SimulatorEnv,
connection_index: usize,
interaction: &Interaction,
stack: &mut Vec<ResultSet>,
) -> limbo_core::Result<ExecutionContinuation> {
log::info!("executing in rusqlite: {}", interaction);
log::trace!(
"execute_interaction_rusqlite(connection_index={}, interaction={})",
connection_index,
interaction
);
match interaction {
Interaction::Query(query) => {
let conn = match &mut env.connections[connection_index] {
RusqliteConnection::Connected(conn) => conn,
RusqliteConnection::Disconnected => unreachable!(),
SimConnection::SQLiteConnection(conn) => conn,
SimConnection::LimboConnection(_) => unreachable!(),
SimConnection::Disconnected => unreachable!(),
};
log::debug!("{}", interaction);
@@ -318,7 +340,7 @@ fn execute_interaction_rusqlite(
}
}
Interaction::Fault(_) => {
log::debug!("faults are not supported in differential testing mode");
interaction.execute_fault(env, connection_index)?;
}
}

83
simulator/runner/env.rs
View File

@@ -1,8 +1,10 @@
use std::fmt::Display;
use std::mem;
use std::path::Path;
use std::rc::Rc;
use std::sync::Arc;
use limbo_core::{Connection, Database};
use limbo_core::Database;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha8Rng;
@@ -12,12 +14,6 @@ use crate::runner::io::SimulatorIO;
use super::cli::SimulatorCLI;
pub trait SimulatorEnvTrait {
fn tables(&self) -> &Vec<Table>;
fn tables_mut(&mut self) -> &mut Vec<Table>;
}
#[derive(Clone)]
pub(crate) struct SimulatorEnv {
pub(crate) opts: SimulatorOpts,
pub(crate) tables: Vec<Table>,
@@ -27,13 +23,18 @@ pub(crate) struct SimulatorEnv {
pub(crate) rng: ChaCha8Rng,
}
impl SimulatorEnvTrait for SimulatorEnv {
fn tables(&self) -> &Vec<Table> {
&self.tables
}
fn tables_mut(&mut self) -> &mut Vec<Table> {
&mut self.tables
impl SimulatorEnv {
pub(crate) fn clone_without_connections(&self) -> Self {
SimulatorEnv {
opts: self.opts.clone(),
tables: self.tables.clone(),
connections: (0..self.connections.len())
.map(|_| SimConnection::Disconnected)
.collect(),
io: self.io.clone(),
db: self.db.clone(),
rng: self.rng.clone(),
}
}
}
@@ -85,7 +86,11 @@ impl SimulatorEnv {
// Remove existing database file if it exists
if db_path.exists() {
std::fs::remove_file(db_path).unwrap();
std::fs::remove_file(db_path.with_extension("db-wal")).unwrap();
}
let wal_path = db_path.with_extension("db-wal");
if wal_path.exists() {
std::fs::remove_file(wal_path).unwrap();
}
let db = match Database::open_file(io.clone(), db_path.to_str().unwrap(), false) {
@@ -95,7 +100,9 @@ impl SimulatorEnv {
}
};
let connections = vec![SimConnection::Disconnected; opts.max_connections];
let connections = (0..opts.max_connections)
.map(|_| SimConnection::Disconnected)
.collect::<Vec<_>>();
SimulatorEnv {
opts,
@@ -108,27 +115,55 @@ impl SimulatorEnv {
}
}
pub trait ConnectionTrait {
pub trait ConnectionTrait
where
Self: std::marker::Sized + Clone,
{
fn is_connected(&self) -> bool;
fn disconnect(&mut self);
}
#[derive(Clone)]
pub(crate) enum SimConnection {
Connected(Rc<Connection>),
LimboConnection(Rc<limbo_core::Connection>),
SQLiteConnection(rusqlite::Connection),
Disconnected,
}
impl ConnectionTrait for SimConnection {
fn is_connected(&self) -> bool {
impl SimConnection {
pub(crate) fn is_connected(&self) -> bool {
match self {
SimConnection::Connected(_) => true,
SimConnection::LimboConnection(_) | SimConnection::SQLiteConnection(_) => true,
SimConnection::Disconnected => false,
}
}
pub(crate) fn disconnect(&mut self) {
let conn = mem::replace(self, SimConnection::Disconnected);
fn disconnect(&mut self) {
*self = SimConnection::Disconnected;
match conn {
SimConnection::LimboConnection(conn) => {
conn.close().unwrap();
}
SimConnection::SQLiteConnection(conn) => {
conn.close().unwrap();
}
SimConnection::Disconnected => {}
}
}
}
impl Display for SimConnection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SimConnection::LimboConnection(_) => {
write!(f, "LimboConnection")
}
SimConnection::SQLiteConnection(_) => {
write!(f, "SQLiteConnection")
}
SimConnection::Disconnected => {
write!(f, "Disconnected")
}
}
}
}

9
simulator/runner/execution.rs
View File

@@ -122,8 +122,10 @@ fn execute_plan(
if let SimConnection::Disconnected = connection {
log::debug!("connecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect().unwrap());
env.connections[connection_index] =
SimConnection::LimboConnection(env.db.connect().unwrap());
} else {
log::debug!("connection {} already connected", connection_index);
match execute_interaction(env, connection_index, interaction, &mut state.stack) {
Ok(next_execution) => {
interaction.shadow(env);
@@ -163,7 +165,7 @@ fn execute_plan(
/// `execute_interaction` uses this type in conjunction with a result, where
/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case
/// indicates the next step in the plan.
#[derive(PartialEq)]
#[derive(PartialEq, Debug)]
pub(crate) enum ExecutionContinuation {
/// Default continuation, execute the next interaction.
NextInteraction,
@@ -185,7 +187,8 @@ pub(crate) fn execute_interaction(
match interaction {
Interaction::Query(_) => {
let conn = match &mut env.connections[connection_index] {
SimConnection::Connected(conn) => conn,
SimConnection::LimboConnection(conn) => conn,
SimConnection::SQLiteConnection(_) => unreachable!(),
SimConnection::Disconnected => unreachable!(),
};

3
simulator/runner/watch.rs
View File

@@ -98,7 +98,8 @@ fn execute_plan(
if let SimConnection::Disconnected = connection {
log::debug!("connecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect().unwrap());
env.connections[connection_index] =
SimConnection::LimboConnection(env.db.connect().unwrap());
} else {
match execute_interaction(env, connection_index, interaction, &mut state.stack) {
Ok(next_execution) => {