From 003ad6cc64974e6a09d8fb8c7f6a3b1e31284b62 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Thu, 26 Dec 2024 15:10:03 -0500 Subject: [PATCH 1/7] allow failure assertions in the simulator, add creating the same table two times to the list of checked properties as a failure property example --- Cargo.lock | 1 + simulator/Cargo.toml | 1 + simulator/generation/plan.rs | 38 +++++++++++++++++++++++++++++++++--- simulator/main.rs | 2 +- simulator/model/query.rs | 2 +- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6053a3e93..ced458793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1171,6 +1171,7 @@ dependencies = [ "log", "rand", "rand_chacha", + "regex", "tempfile", ] diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index 31a54f1e6..dcb948bbe 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -23,3 +23,4 @@ tempfile = "3.0.7" env_logger = "0.10.1" anarchist-readable-name-generator-lib = "0.1.2" clap = { version = "4.5", features = ["derive"] } +regex = "1.10.5" \ No newline at end of file diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 61b115f01..2a39ff759 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -16,7 +16,7 @@ use crate::generation::{frequency, Arbitrary, ArbitraryFrom}; use super::{pick, pick_index}; -pub(crate) type ResultSet = Vec>; +pub(crate) type ResultSet = Result>>; pub(crate) struct InteractionPlan { pub(crate) plan: Vec, @@ -195,7 +195,7 @@ impl ArbitraryFrom for InteractionPlan { } impl Interaction { - pub(crate) fn execute_query(&self, conn: &mut Rc) -> Result { + pub(crate) fn execute_query(&self, conn: &mut Rc) -> ResultSet { match self { Interaction::Query(query) => { let query_str = query.to_string(); @@ -340,13 +340,41 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Inte ), func: Box::new(move |stack: &Vec| { let rows = stack.last().unwrap(); - rows.iter().any(|r| r == &row) + match rows { + Ok(rows) => rows.iter().any(|r| r == &row), + Err(_) => false, + } }), }); Interactions(vec![insert_query, select_query, assertion]) } +fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) -> Interactions { + let create_query = Create::arbitrary(rng); + let cq1 = Interaction::Query(Query::Create(create_query.clone())); + let cq2 = Interaction::Query(Query::Create(create_query.clone())); + + let assertion = Interaction::Assertion(Assertion { + message: + "creating two tables with the name should result in a failure for the second query" + .to_string(), + func: Box::new(move |stack: &Vec| { + let last = stack.last().unwrap(); + println!("last: {:?}", last); + match last { + Ok(_) => false, + Err(e) => { + let re = regex::Regex::new("Table .* already exists").unwrap(); + re.is_match(&e.to_string()) + } + } + }), + }); + + Interactions(vec![cq1, cq2, assertion]) +} + fn create_table(rng: &mut R, env: &SimulatorEnv) -> Interactions { let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng))); Interactions(vec![create_query]) @@ -399,6 +427,10 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { Box::new(|rng: &mut R| create_table(rng, env)), ), (1, Box::new(|rng: &mut R| random_fault(rng, env))), + ( + 1, + Box::new(|rng: &mut R| property_double_create_failure(rng, env)), + ), ], rng, ) diff --git a/simulator/main.rs b/simulator/main.rs index 49f738c56..ec5dc845f 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -269,7 +269,7 @@ fn execute_interaction( }; log::debug!("{}", interaction); - let results = interaction.execute_query(conn)?; + let results = interaction.execute_query(conn); log::debug!("{:?}", results); stack.push(results); } diff --git a/simulator/model/query.rs b/simulator/model/query.rs index eeec68d08..e948c3318 100644 --- a/simulator/model/query.rs +++ b/simulator/model/query.rs @@ -61,7 +61,7 @@ pub(crate) enum Query { Delete(Delete), } -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct Create { pub(crate) table: Table, } From 12fee4df3760d6ff86eb87f1d60eae8116c5beb6 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Thu, 26 Dec 2024 15:20:19 -0500 Subject: [PATCH 2/7] remove the regex dependency as functionality is possible without it --- Cargo.lock | 1 - simulator/Cargo.toml | 3 +-- simulator/generation/plan.rs | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da64f8e3b..0f034d89d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1175,7 +1175,6 @@ dependencies = [ "log", "rand", "rand_chacha", - "regex", "tempfile", ] diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index dcb948bbe..462ceaec7 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -22,5 +22,4 @@ log = "0.4.20" tempfile = "3.0.7" env_logger = "0.10.1" anarchist-readable-name-generator-lib = "0.1.2" -clap = { version = "4.5", features = ["derive"] } -regex = "1.10.5" \ No newline at end of file +clap = { version = "4.5", features = ["derive"] } \ No newline at end of file diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 2a39ff759..7be3ad420 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -352,6 +352,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Inte fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) -> Interactions { let create_query = Create::arbitrary(rng); + let table_name = create_query.table.name.clone(); let cq1 = Interaction::Query(Query::Create(create_query.clone())); let cq2 = Interaction::Query(Query::Create(create_query.clone())); @@ -361,12 +362,10 @@ fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) .to_string(), func: Box::new(move |stack: &Vec| { let last = stack.last().unwrap(); - println!("last: {:?}", last); match last { Ok(_) => false, Err(e) => { - let re = regex::Regex::new("Table .* already exists").unwrap(); - re.is_match(&e.to_string()) + e.to_string().contains(&format!("Table {table_name} already exists")) } } }), From ad0288b39ca55cb7eae9d6189ec64b5f3db1b205 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Thu, 26 Dec 2024 15:23:10 -0500 Subject: [PATCH 3/7] fix formatting --- simulator/generation/plan.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 7be3ad420..1857b4a00 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -364,9 +364,9 @@ fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) let last = stack.last().unwrap(); match last { Ok(_) => false, - Err(e) => { - e.to_string().contains(&format!("Table {table_name} already exists")) - } + Err(e) => e + .to_string() + .contains(&format!("Table {table_name} already exists")), } }), }); From d3fee3b33192bc572f8c33ea61cebcd684fb283e Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sun, 29 Dec 2024 14:00:57 -0500 Subject: [PATCH 4/7] add empty line at the end of cargo.toml, add create counts to the interaction stats, turn the percentages into f64 --- simulator/Cargo.toml | 2 +- simulator/generation/mod.rs | 20 +++++++++++++++----- simulator/generation/plan.rs | 35 +++++++++++++++++++++-------------- simulator/main.rs | 18 +++++++++++------- simulator/runner/env.rs | 7 ++++--- 5 files changed, 52 insertions(+), 30 deletions(-) diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index 462ceaec7..31a54f1e6 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -22,4 +22,4 @@ log = "0.4.20" tempfile = "3.0.7" env_logger = "0.10.1" anarchist-readable-name-generator-lib = "0.1.2" -clap = { version = "4.5", features = ["derive"] } \ No newline at end of file +clap = { version = "4.5", features = ["derive"] } diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index 07a93492b..68f9c84fe 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -1,5 +1,10 @@ +use std::{ + iter::Sum, + ops::{Add, Sub, SubAssign}, +}; + use anarchist_readable_name_generator_lib::readable_name_custom; -use rand::Rng; +use rand::{distributions::uniform::SampleUniform, Rng}; pub mod plan; pub mod query; @@ -13,12 +18,17 @@ pub trait ArbitraryFrom { fn arbitrary_from(rng: &mut R, t: &T) -> Self; } -pub(crate) fn frequency<'a, T, R: rand::Rng>( - choices: Vec<(usize, Box T + 'a>)>, +pub(crate) fn frequency< + 'a, + T, + R: rand::Rng, + N: Sum + PartialOrd + Copy + Default + SampleUniform + SubAssign, +>( + choices: Vec<(N, Box T + 'a>)>, rng: &mut R, ) -> T { - let total = choices.iter().map(|(weight, _)| weight).sum::(); - let mut choice = rng.gen_range(0..total); + let total = choices.iter().map(|(weight, _)| *weight).sum::(); + let mut choice = rng.gen_range(N::default()..total); for (weight, f) in choices { if choice < weight { diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 1857b4a00..759ff5e2b 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -9,7 +9,7 @@ use crate::{ query::{Create, Insert, Predicate, Query, Select}, table::Value, }, - SimConnection, SimulatorEnv, SimulatorOpts, + SimConnection, SimulatorEnv, }; use crate::generation::{frequency, Arbitrary, ArbitraryFrom}; @@ -45,14 +45,15 @@ pub(crate) struct InteractionStats { pub(crate) read_count: usize, pub(crate) write_count: usize, pub(crate) delete_count: usize, + pub(crate) create_count: usize, } impl Display for InteractionStats { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, - "Read: {}, Write: {}, Delete: {}", - self.read_count, self.write_count, self.delete_count + "Read: {}, Write: {}, Delete: {}, Create: {}", + self.read_count, self.write_count, self.delete_count, self.create_count ) } } @@ -135,6 +136,7 @@ impl InteractionPlan { let mut read = 0; let mut write = 0; let mut delete = 0; + let mut create = 0; for interaction in &self.plan { match interaction { @@ -142,7 +144,7 @@ impl InteractionPlan { Query::Select(_) => read += 1, Query::Insert(_) => write += 1, Query::Delete(_) => delete += 1, - Query::Create(_) => {} + Query::Create(_) => create += 1, }, Interaction::Assertion(_) => {} Interaction::Fault(_) => {} @@ -153,6 +155,7 @@ impl InteractionPlan { read_count: read, write_count: write, delete_count: delete, + create_count: create, } } } @@ -400,17 +403,21 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { rng: &mut R, (env, stats): &(&SimulatorEnv, InteractionStats), ) -> Self { - let remaining_read = - ((((env.opts.max_interactions * env.opts.read_percent) as f64) / 100.0) as usize) - .saturating_sub(stats.read_count); - let remaining_write = ((((env.opts.max_interactions * env.opts.write_percent) as f64) - / 100.0) as usize) - .saturating_sub(stats.write_count); + let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) + - (stats.read_count as f64)) + .max(0.0); + let remaining_write = ((env.opts.max_interactions as f64 * env.opts.write_percent / 100.0) + - (stats.write_count as f64)) + .max(0.0); + let remaining_create = ((env.opts.max_interactions as f64 * env.opts.create_percent + / 100.0) + - (stats.create_count as f64)) + .max(0.0); frequency( vec![ ( - usize::min(remaining_read, remaining_write), + f64::min(remaining_read, remaining_write), Box::new(|rng: &mut R| property_insert_select(rng, env)), ), ( @@ -422,12 +429,12 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { Box::new(|rng: &mut R| random_write(rng, env)), ), ( - remaining_write / 10, + remaining_create, Box::new(|rng: &mut R| create_table(rng, env)), ), - (1, Box::new(|rng: &mut R| random_fault(rng, env))), + (1.0, Box::new(|rng: &mut R| random_fault(rng, env))), ( - 1, + remaining_create / 2.0, Box::new(|rng: &mut R| property_double_create_failure(rng, env)), ), ], diff --git a/simulator/main.rs b/simulator/main.rs index bda55d909..8dc6d85bc 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -1,7 +1,7 @@ use clap::Parser; use generation::plan::{Interaction, InteractionPlan, ResultSet}; use generation::{pick_index, ArbitraryFrom}; -use limbo_core::{Connection, Database, Result, RowResult, IO}; +use limbo_core::{Database, Result}; use model::table::Value; use rand::prelude::*; use rand_chacha::ChaCha8Rng; @@ -11,7 +11,6 @@ use runner::io::SimulatorIO; use std::backtrace::Backtrace; use std::io::Write; use std::path::Path; -use std::rc::Rc; use std::sync::Arc; use tempfile::TempDir; @@ -137,14 +136,18 @@ fn run_simulation( ) -> Result<()> { let mut rng = ChaCha8Rng::seed_from_u64(seed); - let (read_percent, write_percent, delete_percent) = { - let mut remaining = 100; - let read_percent = rng.gen_range(0..=remaining); + let (create_percent, read_percent, write_percent, delete_percent) = { + let mut remaining = 100.0; + let read_percent = rng.gen_range(0.0..=remaining); remaining -= read_percent; - let write_percent = rng.gen_range(0..=remaining); + let write_percent = rng.gen_range(0.0..=remaining); remaining -= write_percent; let delete_percent = remaining; - (read_percent, write_percent, delete_percent) + + let create_percent = write_percent / 10.0; + let write_percent = write_percent - create_percent; + + (create_percent, read_percent, write_percent, delete_percent) }; if cli_opts.maximum_size < 1 { @@ -156,6 +159,7 @@ fn run_simulation( max_connections: 1, // TODO: for now let's use one connection as we didn't implement // correct transactions procesing max_tables: rng.gen_range(0..128), + create_percent, read_percent, write_percent, delete_percent, diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs index 0624b94b4..53d99b2f0 100644 --- a/simulator/runner/env.rs +++ b/simulator/runner/env.rs @@ -30,9 +30,10 @@ pub(crate) struct SimulatorOpts { pub(crate) max_tables: usize, // this next options are the distribution of workload where read_percent + write_percent + // delete_percent == 100% - pub(crate) read_percent: usize, - pub(crate) write_percent: usize, - pub(crate) delete_percent: usize, + pub(crate) create_percent: f64, + pub(crate) read_percent: f64, + pub(crate) write_percent: f64, + pub(crate) delete_percent: f64, pub(crate) max_interactions: usize, pub(crate) page_size: usize, } From c01f2d4ac22bccf7d5691d424391b1f44c639abf Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sun, 29 Dec 2024 16:09:49 -0500 Subject: [PATCH 5/7] fix formatting --- simulator/generation/plan.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index f393809bd..bb9efbe0f 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -379,7 +379,6 @@ fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv Interactions(vec![cq1, cq2, assertion]) } - fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Interactions { let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng))); Interactions(vec![create_query]) From 58f23983e1eefd896ae2c55c7a11e044047aaa0c Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 30 Dec 2024 00:36:43 -0500 Subject: [PATCH 6/7] minor changes, add maximum time bound to the simulator, fix bug in the table create shadowing --- simulator/generation/mod.rs | 7 ++----- simulator/generation/plan.rs | 6 ++++-- simulator/main.rs | 37 +++++++++++++++++++++++++++++++++--- simulator/runner/cli.rs | 16 +++++++++++++++- simulator/runner/env.rs | 1 + 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index 68f9c84fe..8158b2d17 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -1,7 +1,4 @@ -use std::{ - iter::Sum, - ops::{Add, Sub, SubAssign}, -}; +use std::{iter::Sum, ops::SubAssign}; use anarchist_readable_name_generator_lib::readable_name_custom; use rand::{distributions::uniform::SampleUniform, Rng}; @@ -48,7 +45,7 @@ pub(crate) fn one_of<'a, T, R: rand::Rng>( choices[index](rng) } -pub(crate) fn pick<'a, T, R: rand::Rng>(choices: &'a Vec, rng: &mut R) -> &'a T { +pub(crate) fn pick<'a, T, R: rand::Rng>(choices: &'a [T], rng: &mut R) -> &'a T { let index = rng.gen_range(0..choices.len()); &choices[index] } diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index bb9efbe0f..dbf6dd7f6 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -101,7 +101,9 @@ impl Interactions { match interaction { Interaction::Query(query) => match query { Query::Create(create) => { - env.tables.push(create.table.clone()); + if !env.tables.iter().any(|t| t.name == create.table.name) { + env.tables.push(create.table.clone()); + } } Query::Insert(insert) => { let table = env @@ -175,7 +177,7 @@ impl ArbitraryFrom for InteractionPlan { rng: ChaCha8Rng::seed_from_u64(rng.next_u64()), }; - let num_interactions = rng.gen_range(0..env.opts.max_interactions); + let num_interactions = env.opts.max_interactions; // First create at least one table let create_query = Create::arbitrary(rng); diff --git a/simulator/main.rs b/simulator/main.rs index 992c5f3d5..52c33d5ec 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -121,6 +121,15 @@ fn main() { // Move the old database and plan file back std::fs::rename(&old_db_path, &db_path).unwrap(); std::fs::rename(&old_plan_path, &plan_path).unwrap(); + } else if let Ok(result) = result { + match result { + Ok(_) => { + log::info!("simulation completed successfully"); + } + Err(e) => { + log::error!("simulation failed: {:?}", e); + } + } } // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. println!("database path: {:?}", db_path); @@ -150,12 +159,26 @@ fn run_simulation( (create_percent, read_percent, write_percent, delete_percent) }; + if cli_opts.minimum_size < 1 { + return Err(limbo_core::LimboError::InternalError( + "minimum size must be at least 1".to_string(), + )); + } + if cli_opts.maximum_size < 1 { - panic!("maximum size must be at least 1"); + return Err(limbo_core::LimboError::InternalError( + "maximum size must be at least 1".to_string(), + )); + } + + if cli_opts.maximum_size < cli_opts.minimum_size { + return Err(limbo_core::LimboError::InternalError( + "maximum size must be greater than or equal to minimum size".to_string(), + )); } let opts = SimulatorOpts { - ticks: rng.gen_range(1..=cli_opts.maximum_size), + ticks: rng.gen_range(cli_opts.minimum_size..=cli_opts.maximum_size), max_connections: 1, // TODO: for now let's use one connection as we didn't implement // correct transactions procesing max_tables: rng.gen_range(0..128), @@ -164,7 +187,8 @@ fn run_simulation( write_percent, delete_percent, page_size: 4096, // TODO: randomize this too - max_interactions: rng.gen_range(1..=cli_opts.maximum_size), + max_interactions: rng.gen_range(cli_opts.minimum_size..=cli_opts.maximum_size), + max_time_simulation: cli_opts.maximum_time, }; let io = Arc::new(SimulatorIO::new(seed, opts.page_size).unwrap()); @@ -212,12 +236,19 @@ fn run_simulation( } fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> Result<()> { + let now = std::time::Instant::now(); // todo: add history here by recording which interaction was executed at which tick for _tick in 0..env.opts.ticks { // Pick the connection to interact with let connection_index = pick_index(env.connections.len(), &mut env.rng); // Execute the interaction for the selected connection execute_plan(env, connection_index, plans)?; + // Check if the maximum time for the simulation has been reached + if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { + return Err(limbo_core::LimboError::InternalError( + "maximum time for simulation reached".into(), + )); + } } Ok(()) diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index f977937bb..8ad42c8b3 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -15,10 +15,24 @@ pub struct SimulatorCLI { )] pub doublecheck: bool, #[clap( - short, + short = 'n', long, help = "change the maximum size of the randomly generated sequence of interactions", default_value_t = 1024 )] pub maximum_size: usize, + #[clap( + short = 'k', + long, + help = "change the minimum size of the randomly generated sequence of interactions", + default_value_t = 1 + )] + pub minimum_size: usize, + #[clap( + short = 't', + long, + help = "change the maximum time of the simulation(in seconds)", + default_value_t = 60 * 60 // default to 1 hour + )] + pub maximum_time: usize, } diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs index 53d99b2f0..7edad025f 100644 --- a/simulator/runner/env.rs +++ b/simulator/runner/env.rs @@ -36,4 +36,5 @@ pub(crate) struct SimulatorOpts { pub(crate) delete_percent: f64, pub(crate) max_interactions: usize, pub(crate) page_size: usize, + pub(crate) max_time_simulation: usize, } From d8ce88c057ce2fa9c76eb1839d18e8d734b027ec Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 30 Dec 2024 00:41:21 -0500 Subject: [PATCH 7/7] fix clippy warning --- simulator/generation/table.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/simulator/generation/table.rs b/simulator/generation/table.rs index 332aeb1f3..179c53436 100644 --- a/simulator/generation/table.rs +++ b/simulator/generation/table.rs @@ -41,11 +41,7 @@ impl Arbitrary for Column { impl Arbitrary for ColumnType { fn arbitrary(rng: &mut R) -> Self { - pick( - &vec![Self::Integer, Self::Float, Self::Text, Self::Blob], - rng, - ) - .to_owned() + pick(&[Self::Integer, Self::Float, Self::Text, Self::Blob], rng).to_owned() } }