From 6569d27bdc971e3e50a1c92530bfe679ff837c2a Mon Sep 17 00:00:00 2001 From: pedrocarlo Date: Mon, 22 Sep 2025 15:00:50 -0300 Subject: [PATCH] compare rows order insensitively for differential testing --- simulator/generation/property.rs | 1 + simulator/runner/differential.rs | 71 ++++++++++++++++++++++++-------- simulator/runner/env.rs | 4 +- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 736a8136a..7e42444cd 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -1528,6 +1528,7 @@ impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property { env.profile.experimental_mvcc, ); + #[allow(clippy::type_complexity)] let choices: Vec<(_, Box Property>)> = vec![ ( if !env.opts.disable_insert_values_select { diff --git a/simulator/runner/differential.rs b/simulator/runner/differential.rs index 1a519228c..791b54a13 100644 --- a/simulator/runner/differential.rs +++ b/simulator/runner/differential.rs @@ -1,5 +1,9 @@ -use std::sync::{Arc, Mutex}; +use std::{ + collections::{BTreeMap, btree_map::Entry}, + sync::{Arc, Mutex}, +}; +use itertools::Itertools; use similar_asserts::SimpleDiff; use sql_generation::model::table::SimValue; @@ -140,13 +144,13 @@ fn compare_results( ) -> turso_core::Result<()> { match (turso_res, rusqlite_res) { (Ok(..), Ok(..)) => { - let limbo_values = turso_conn_state.stack.last(); + let turso_values = turso_conn_state.stack.last(); let rusqlite_values = rusqlite_conn_state.stack.last(); - match (limbo_values, rusqlite_values) { - (Some(limbo_values), Some(rusqlite_values)) => { - match (limbo_values, rusqlite_values) { - (Ok(limbo_values), Ok(rusqlite_values)) => { - if limbo_values != rusqlite_values { + match (turso_values, rusqlite_values) { + (Some(turso_values), Some(rusqlite_values)) => { + match (turso_values, rusqlite_values) { + (Ok(turso_values), Ok(rusqlite_values)) => { + if !compare_order_insensitive(turso_values, rusqlite_values) { tracing::error!( "returned values from limbo and rusqlite results do not match" ); @@ -165,17 +169,19 @@ fn compare_results( } } - let limbo_string_values: Vec> = limbo_values + let turso_string_values: Vec> = turso_values .iter() - .map(|rows| rows.iter().map(|row| val_to_string(row)).collect()) + .map(|rows| rows.iter().map(val_to_string).collect()) + .sorted() .collect(); let rusqlite_string_values: Vec> = rusqlite_values .iter() - .map(|rows| rows.iter().map(|row| val_to_string(row)).collect()) + .map(|rows| rows.iter().map(val_to_string).collect()) + .sorted() .collect(); - let turso_string = format!("{limbo_string_values:#?}"); + let turso_string = format!("{turso_string_values:#?}"); let rusqlite_string = format!("{rusqlite_string_values:#?}"); let diff = SimpleDiff::from_str( &turso_string, @@ -191,26 +197,26 @@ fn compare_results( )); } } - (Err(limbo_err), Err(rusqlite_err)) => { + (Err(turso_err), Err(rusqlite_err)) => { tracing::warn!("limbo and rusqlite both fail, requires manual check"); - tracing::warn!("limbo error {}", limbo_err); + tracing::warn!("limbo error {}", turso_err); tracing::warn!("rusqlite error {}", rusqlite_err); } - (Ok(limbo_result), Err(rusqlite_err)) => { + (Ok(turso_err), Err(rusqlite_err)) => { tracing::error!( "limbo and rusqlite results do not match, limbo returned values but rusqlite failed" ); - tracing::error!("limbo values {:?}", limbo_result); + tracing::error!("limbo values {:?}", turso_err); tracing::error!("rusqlite error {}", rusqlite_err); return Err(turso_core::LimboError::InternalError( "limbo and rusqlite results do not match".into(), )); } - (Err(limbo_err), Ok(_)) => { + (Err(turso_err), Ok(_)) => { tracing::error!( "limbo and rusqlite results do not match, limbo failed but rusqlite returned values" ); - tracing::error!("limbo error {}", limbo_err); + tracing::error!("limbo error {}", turso_err); return Err(turso_core::LimboError::InternalError( "limbo and rusqlite results do not match".into(), )); @@ -249,3 +255,34 @@ fn compare_results( } Ok(()) } + +fn count_rows(values: &[Vec]) -> BTreeMap<&Vec, i32> { + let mut counter = BTreeMap::new(); + for row in values.iter() { + match counter.entry(row) { + Entry::Vacant(entry) => { + entry.insert(1); + } + Entry::Occupied(mut entry) => { + let counter = entry.get_mut(); + + *counter += 1; + } + } + } + counter +} + +fn compare_order_insensitive( + turso_values: &[Vec], + rusqlite_values: &[Vec], +) -> bool { + if turso_values.len() != rusqlite_values.len() { + return false; + } + + let turso_counter = count_rows(turso_values); + let rusqlite_counter = count_rows(rusqlite_values); + + turso_counter == rusqlite_counter +} diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs index 5ec963e71..e1e233b50 100644 --- a/simulator/runner/env.rs +++ b/simulator/runner/env.rs @@ -269,7 +269,7 @@ impl SimulatorEnv { ) -> Self { let mut rng = ChaCha8Rng::seed_from_u64(seed); - let opts = SimulatorOpts { + let mut opts = SimulatorOpts { seed, ticks: rng.random_range(cli_opts.minimum_tests..=cli_opts.maximum_tests), max_tables: rng.random_range(0..128), @@ -323,6 +323,8 @@ impl SimulatorEnv { if cli_opts.differential { // Disable faults when running against sqlite as we cannot control faults on it profile.io.enable = false; + // Disable limits due to differences in return order from turso and rusqlite + opts.disable_select_limit = true; } profile.validate().unwrap();