diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d2c6f8edd..d2a201aba 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -74,6 +74,8 @@ jobs: run: ./scripts/run-sim --maximum-tests 1000 --min-tick 10 --max-tick 50 --profile write_heavy loop -n 10 -s - name: Simulator Faultless run: ./scripts/run-sim --maximum-tests 1000 --min-tick 10 --max-tick 50 --profile faultless loop -n 10 -s + - name: Simulator Differential + run: ./scripts/run-sim --maximum-tests 1000 --differential loop -n 10 -s test-limbo: runs-on: blacksmith-4vcpu-ubuntu-2404 diff --git a/Cargo.lock b/Cargo.lock index a37a562e5..9e9739689 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -621,6 +621,18 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "console_error_panic_hook" version = "0.1.7" @@ -1100,6 +1112,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encryption-throughput" version = "0.1.0" @@ -2171,6 +2189,8 @@ dependencies = [ "schemars 1.0.4", "serde", "serde_json", + "similar", + "similar-asserts", "sql_generation", "strum", "tracing", @@ -3540,6 +3560,26 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +dependencies = [ + "bstr", + "unicode-segmentation", +] + +[[package]] +name = "similar-asserts" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b441962c817e33508847a22bd82f03a30cff43642dc2fae8b050566121eb9a" +dependencies = [ + "console", + "similar", +] + [[package]] name = "slab" version = "0.4.9" diff --git a/Cargo.toml b/Cargo.toml index e8a596ed1..6f3c64d9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,6 +96,11 @@ tracing-appender = "0.2.3" env_logger = { version = "0.11.6", default-features = false } regex = "1.11.1" regex-syntax = { version = "0.8.5", default-features = false } +similar = { version = "2.7.0" } +similar-asserts = { version = "1.7.0" } + +[profile.dev.package.similar] +opt-level = 3 [profile.release] debug = "line-tables-only" diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index 8c37dd8f0..84d67304e 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -45,3 +45,5 @@ strum = { workspace = true } parking_lot = { workspace = true } indexmap = { workspace = true } either = "1.15.0" +similar = { workspace = true } +similar-asserts = { workspace = true } diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 847c30593..7e42444cd 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -1528,107 +1528,103 @@ impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property { env.profile.experimental_mvcc, ); - frequency( - vec![ - ( - if !env.opts.disable_insert_values_select { - u32::min(remaining_.select, remaining_.insert) - } else { - 0 - }, - Box::new(|rng: &mut R| { - property_insert_values_select(rng, &remaining_, conn_ctx) - }), - ), - ( - remaining_.select, - Box::new(|rng: &mut R| property_table_has_expected_content(rng, conn_ctx)), - ), - ( - u32::min(remaining_.select, remaining_.insert), - Box::new(|rng: &mut R| property_read_your_updates_back(rng, conn_ctx)), - ), - ( - if !env.opts.disable_double_create_failure { - remaining_.create / 2 - } else { - 0 - }, - Box::new(|rng: &mut R| { - property_double_create_failure(rng, &remaining_, conn_ctx) - }), - ), - ( - if !env.opts.disable_select_limit { - remaining_.select - } else { - 0 - }, - Box::new(|rng: &mut R| property_select_limit(rng, conn_ctx)), - ), - ( - if !env.opts.disable_delete_select { - u32::min(remaining_.select, remaining_.insert).min(remaining_.delete) - } else { - 0 - }, - Box::new(|rng: &mut R| property_delete_select(rng, &remaining_, conn_ctx)), - ), - ( - if !env.opts.disable_drop_select { - // remaining_.drop - 0 - } else { - 0 - }, - Box::new(|rng: &mut R| property_drop_select(rng, &remaining_, conn_ctx)), - ), - ( - if !env.opts.disable_select_optimizer { - remaining_.select / 2 - } else { - 0 - }, - Box::new(|rng: &mut R| property_select_select_optimizer(rng, conn_ctx)), - ), - ( - if opts.indexes && !env.opts.disable_where_true_false_null { - remaining_.select / 2 - } else { - 0 - }, - Box::new(|rng: &mut R| property_where_true_false_null(rng, conn_ctx)), - ), - ( - if opts.indexes && !env.opts.disable_union_all_preserves_cardinality { - remaining_.select / 3 - } else { - 0 - }, - Box::new(|rng: &mut R| property_union_all_preserves_cardinality(rng, conn_ctx)), - ), - ( - if env.profile.io.enable && !env.opts.disable_fsync_no_wait { - 50 // Freestyle number - } else { - 0 - }, - Box::new(|rng: &mut R| property_fsync_no_wait(rng, &remaining_, conn_ctx)), - ), - ( - if env.profile.io.enable - && env.profile.io.fault.enable - && !env.opts.disable_faulty_query - { - 20 - } else { - 0 - }, - Box::new(|rng: &mut R| property_faulty_query(rng, &remaining_, conn_ctx)), - ), - ], - rng, - ) + #[allow(clippy::type_complexity)] + let choices: Vec<(_, Box Property>)> = vec![ + ( + if !env.opts.disable_insert_values_select { + u32::min(remaining_.select, remaining_.insert).max(1) + } else { + 0 + }, + Box::new(|rng: &mut R| property_insert_values_select(rng, &remaining_, conn_ctx)), + ), + ( + remaining_.select.max(1), + Box::new(|rng: &mut R| property_table_has_expected_content(rng, conn_ctx)), + ), + ( + u32::min(remaining_.select, remaining_.insert).max(1), + Box::new(|rng: &mut R| property_read_your_updates_back(rng, conn_ctx)), + ), + ( + if !env.opts.disable_double_create_failure { + remaining_.create / 2 + } else { + 0 + }, + Box::new(|rng: &mut R| property_double_create_failure(rng, &remaining_, conn_ctx)), + ), + ( + if !env.opts.disable_select_limit { + remaining_.select + } else { + 0 + }, + Box::new(|rng: &mut R| property_select_limit(rng, conn_ctx)), + ), + ( + if !env.opts.disable_delete_select { + u32::min(remaining_.select, remaining_.insert).min(remaining_.delete) + } else { + 0 + }, + Box::new(|rng: &mut R| property_delete_select(rng, &remaining_, conn_ctx)), + ), + ( + if !env.opts.disable_drop_select { + // remaining_.drop + 0 + } else { + 0 + }, + Box::new(|rng: &mut R| property_drop_select(rng, &remaining_, conn_ctx)), + ), + ( + if !env.opts.disable_select_optimizer { + remaining_.select / 2 + } else { + 0 + }, + Box::new(|rng: &mut R| property_select_select_optimizer(rng, conn_ctx)), + ), + ( + if opts.indexes && !env.opts.disable_where_true_false_null { + remaining_.select / 2 + } else { + 0 + }, + Box::new(|rng: &mut R| property_where_true_false_null(rng, conn_ctx)), + ), + ( + if opts.indexes && !env.opts.disable_union_all_preserves_cardinality { + remaining_.select / 3 + } else { + 0 + }, + Box::new(|rng: &mut R| property_union_all_preserves_cardinality(rng, conn_ctx)), + ), + ( + if env.profile.io.enable && !env.opts.disable_fsync_no_wait { + 50 // Freestyle number + } else { + 0 + }, + Box::new(|rng: &mut R| property_fsync_no_wait(rng, &remaining_, conn_ctx)), + ), + ( + if env.profile.io.enable + && env.profile.io.fault.enable + && !env.opts.disable_faulty_query + { + 20 + } else { + 0 + }, + Box::new(|rng: &mut R| property_faulty_query(rng, &remaining_, conn_ctx)), + ), + ]; + + frequency(choices, rng) } } diff --git a/simulator/runner/differential.rs b/simulator/runner/differential.rs index c2be34b38..791b54a13 100644 --- a/simulator/runner/differential.rs +++ b/simulator/runner/differential.rs @@ -1,4 +1,11 @@ -use std::sync::{Arc, Mutex}; +use std::{ + collections::{BTreeMap, btree_map::Entry}, + sync::{Arc, Mutex}, +}; + +use itertools::Itertools; +use similar_asserts::SimpleDiff; +use sql_generation::model::table::SimValue; use crate::generation::plan::{ConnectionState, Interaction, InteractionPlanState}; @@ -83,6 +90,7 @@ pub(crate) fn execute_interactions( last_execution.interaction_index = state.interaction_pointer; let mut turso_state = state.clone(); + let mut rusqlite_state = state.clone(); // first execute turso let turso_res = super::execution::execute_plan( @@ -92,8 +100,6 @@ pub(crate) fn execute_interactions( &mut turso_state, ); - let mut rusqlite_state = state.clone(); - // second execute rusqlite let rusqlite_res = super::execution::execute_plan( &mut rusqlite_env, @@ -112,7 +118,9 @@ pub(crate) fn execute_interactions( return ExecutionResult::new(history, Some(err)); } - state.interaction_pointer += 1; + assert_eq!(turso_state, rusqlite_state); + + *state = turso_state; // Check if the maximum time for the simulation has been reached if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { @@ -136,79 +144,79 @@ fn compare_results( ) -> turso_core::Result<()> { match (turso_res, rusqlite_res) { (Ok(..), Ok(..)) => { - let limbo_values = turso_conn_state.stack.last(); + let turso_values = turso_conn_state.stack.last(); let rusqlite_values = rusqlite_conn_state.stack.last(); - match (limbo_values, rusqlite_values) { - (Some(limbo_values), Some(rusqlite_values)) => { - match (limbo_values, rusqlite_values) { - (Ok(limbo_values), Ok(rusqlite_values)) => { - if limbo_values != rusqlite_values { + match (turso_values, rusqlite_values) { + (Some(turso_values), Some(rusqlite_values)) => { + match (turso_values, rusqlite_values) { + (Ok(turso_values), Ok(rusqlite_values)) => { + if !compare_order_insensitive(turso_values, rusqlite_values) { tracing::error!( "returned values from limbo and rusqlite results do not match" ); - let diff = limbo_values - .iter() - .zip(rusqlite_values.iter()) - .enumerate() - .filter(|(_, (l, r))| l != r) - .collect::>(); - let diff = diff - .iter() - .flat_map(|(i, (l, r))| { - let mut diffs = vec![]; - for (j, (l, r)) in l.iter().zip(r.iter()).enumerate() { - if l != r { - tracing::debug!( - "difference at index {}, {}: {} != {}", - i, - j, - l.to_string(), - r.to_string() - ); - diffs.push(((i, j), (l.clone(), r.clone()))); - } + fn val_to_string(sim_val: &SimValue) -> String { + match &sim_val.0 { + turso_core::Value::Blob(blob) => { + let convert_blob = || -> anyhow::Result { + let val = String::from_utf8(blob.clone())?; + Ok(val) + }; + + convert_blob().unwrap_or_else(|_| sim_val.to_string()) } - diffs - }) - .collect::>(); - tracing::debug!("limbo values {:?}", limbo_values); - tracing::debug!("rusqlite values {:?}", rusqlite_values); - tracing::debug!( - "differences: {}", - diff.iter() - .map(|((i, j), (l, r))| format!( - "\t({i}, {j}): ({l}) != ({r})" - )) - .collect::>() - .join("\n") + _ => sim_val.to_string(), + } + } + + let turso_string_values: Vec> = turso_values + .iter() + .map(|rows| rows.iter().map(val_to_string).collect()) + .sorted() + .collect(); + + let rusqlite_string_values: Vec> = rusqlite_values + .iter() + .map(|rows| rows.iter().map(val_to_string).collect()) + .sorted() + .collect(); + + let turso_string = format!("{turso_string_values:#?}"); + let rusqlite_string = format!("{rusqlite_string_values:#?}"); + let diff = SimpleDiff::from_str( + &turso_string, + &rusqlite_string, + "turso", + "rusqlite", ); + tracing::error!(%diff); + return Err(turso_core::LimboError::InternalError( "returned values from limbo and rusqlite results do not match" .into(), )); } } - (Err(limbo_err), Err(rusqlite_err)) => { + (Err(turso_err), Err(rusqlite_err)) => { tracing::warn!("limbo and rusqlite both fail, requires manual check"); - tracing::warn!("limbo error {}", limbo_err); + tracing::warn!("limbo error {}", turso_err); tracing::warn!("rusqlite error {}", rusqlite_err); } - (Ok(limbo_result), Err(rusqlite_err)) => { + (Ok(turso_err), Err(rusqlite_err)) => { tracing::error!( "limbo and rusqlite results do not match, limbo returned values but rusqlite failed" ); - tracing::error!("limbo values {:?}", limbo_result); + tracing::error!("limbo values {:?}", turso_err); tracing::error!("rusqlite error {}", rusqlite_err); return Err(turso_core::LimboError::InternalError( "limbo and rusqlite results do not match".into(), )); } - (Err(limbo_err), Ok(_)) => { + (Err(turso_err), Ok(_)) => { tracing::error!( "limbo and rusqlite results do not match, limbo failed but rusqlite returned values" ); - tracing::error!("limbo error {}", limbo_err); + tracing::error!("limbo error {}", turso_err); return Err(turso_core::LimboError::InternalError( "limbo and rusqlite results do not match".into(), )); @@ -247,3 +255,34 @@ fn compare_results( } Ok(()) } + +fn count_rows(values: &[Vec]) -> BTreeMap<&Vec, i32> { + let mut counter = BTreeMap::new(); + for row in values.iter() { + match counter.entry(row) { + Entry::Vacant(entry) => { + entry.insert(1); + } + Entry::Occupied(mut entry) => { + let counter = entry.get_mut(); + + *counter += 1; + } + } + } + counter +} + +fn compare_order_insensitive( + turso_values: &[Vec], + rusqlite_values: &[Vec], +) -> bool { + if turso_values.len() != rusqlite_values.len() { + return false; + } + + let turso_counter = count_rows(turso_values); + let rusqlite_counter = count_rows(rusqlite_values); + + turso_counter == rusqlite_counter +} diff --git a/simulator/runner/doublecheck.rs b/simulator/runner/doublecheck.rs index d90408686..1b4bb8a5e 100644 --- a/simulator/runner/doublecheck.rs +++ b/simulator/runner/doublecheck.rs @@ -142,7 +142,9 @@ pub(crate) fn execute_plans( return ExecutionResult::new(history, Some(err)); } - state.interaction_pointer += 1; + assert_eq!(turso_state, doublecheck_state); + + *state = turso_state; // Check if the maximum time for the simulation has been reached if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs index 9ef9b612a..e1e233b50 100644 --- a/simulator/runner/env.rs +++ b/simulator/runner/env.rs @@ -269,7 +269,7 @@ impl SimulatorEnv { ) -> Self { let mut rng = ChaCha8Rng::seed_from_u64(seed); - let opts = SimulatorOpts { + let mut opts = SimulatorOpts { seed, ticks: rng.random_range(cli_opts.minimum_tests..=cli_opts.maximum_tests), max_tables: rng.random_range(0..128), @@ -320,6 +320,12 @@ impl SimulatorEnv { if let Some(min_tick) = cli_opts.min_tick { profile.io.latency.min_tick = min_tick; } + if cli_opts.differential { + // Disable faults when running against sqlite as we cannot control faults on it + profile.io.enable = false; + // Disable limits due to differences in return order from turso and rusqlite + opts.disable_select_limit = true; + } profile.validate().unwrap();