aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-01-01 15:34:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'Differential testing fixes' from Pedro Muniz

Browse Source 
- Fixed some incorrect code when running interactions in differential
testing. Instead of replacing the state that was used for running the
interaction, I naively just incremented the interaction pointer.
- adjusted the comparison to check returned values without considering
the order of the rows returned
- added differential testing to run in CI
Closes #3235

Reviewed-by: Jussi Saurio <jussi.saurio@gmail.com>

Closes #3255
This commit is contained in:
Jussi Saurio
2025-09-22 22:58:03 +03:00
parent f6e3beda45 4568e76d0c
commit 04d364d2d0
8 changed files with 245 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/rust.yml vendored
View File

@@ -74,6 +74,8 @@ jobs:
run: ./scripts/run-sim --maximum-tests 1000 --min-tick 10 --max-tick 50 --profile write_heavy loop -n 10 -s
- name: Simulator Faultless
run: ./scripts/run-sim --maximum-tests 1000 --min-tick 10 --max-tick 50 --profile faultless loop -n 10 -s
- name: Simulator Differential
run: ./scripts/run-sim --maximum-tests 1000 --differential loop -n 10 -s
test-limbo:
runs-on: blacksmith-4vcpu-ubuntu-2404

40
Cargo.lock generated
View File

@@ -621,6 +621,18 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "console"
version = "0.15.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
dependencies = [
"encode_unicode",
"libc",
"once_cell",
"windows-sys 0.59.0",
]
[[package]]
name = "console_error_panic_hook"
version = "0.1.7"
@@ -1100,6 +1112,12 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "encode_unicode"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "encryption-throughput"
version = "0.1.0"
@@ -2171,6 +2189,8 @@ dependencies = [
"schemars 1.0.4",
"serde",
"serde_json",
"similar",
"similar-asserts",
"sql_generation",
"strum",
"tracing",
@@ -3540,6 +3560,26 @@ dependencies = [
"libc",
]
[[package]]
name = "similar"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
dependencies = [
"bstr",
"unicode-segmentation",
]
[[package]]
name = "similar-asserts"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5b441962c817e33508847a22bd82f03a30cff43642dc2fae8b050566121eb9a"
dependencies = [
"console",
"similar",
]
[[package]]
name = "slab"
version = "0.4.9"

5
Cargo.toml
View File

@@ -96,6 +96,11 @@ tracing-appender = "0.2.3"
env_logger = { version = "0.11.6", default-features = false }
regex = "1.11.1"
regex-syntax = { version = "0.8.5", default-features = false }
similar = { version = "2.7.0" }
similar-asserts = { version = "1.7.0" }
[profile.dev.package.similar]
opt-level = 3
[profile.release]
debug = "line-tables-only"

2
simulator/Cargo.toml
View File

@@ -45,3 +45,5 @@ strum = { workspace = true }
parking_lot = { workspace = true }
indexmap = { workspace = true }
either = "1.15.0"
similar = { workspace = true }
similar-asserts = { workspace = true }

198
simulator/generation/property.rs
View File

@@ -1528,107 +1528,103 @@ impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property {
env.profile.experimental_mvcc,
);
frequency(
vec![
(
if !env.opts.disable_insert_values_select {
u32::min(remaining_.select, remaining_.insert)
} else {
0
},
Box::new(|rng: &mut R| {
property_insert_values_select(rng, &remaining_, conn_ctx)
}),
),
(
remaining_.select,
Box::new(|rng: &mut R| property_table_has_expected_content(rng, conn_ctx)),
),
(
u32::min(remaining_.select, remaining_.insert),
Box::new(|rng: &mut R| property_read_your_updates_back(rng, conn_ctx)),
),
(
if !env.opts.disable_double_create_failure {
remaining_.create / 2
} else {
0
},
Box::new(|rng: &mut R| {
property_double_create_failure(rng, &remaining_, conn_ctx)
}),
),
(
if !env.opts.disable_select_limit {
remaining_.select
} else {
0
},
Box::new(|rng: &mut R| property_select_limit(rng, conn_ctx)),
),
(
if !env.opts.disable_delete_select {
u32::min(remaining_.select, remaining_.insert).min(remaining_.delete)
} else {
0
},
Box::new(|rng: &mut R| property_delete_select(rng, &remaining_, conn_ctx)),
),
(
if !env.opts.disable_drop_select {
// remaining_.drop
0
} else {
0
},
Box::new(|rng: &mut R| property_drop_select(rng, &remaining_, conn_ctx)),
),
(
if !env.opts.disable_select_optimizer {
remaining_.select / 2
} else {
0
},
Box::new(|rng: &mut R| property_select_select_optimizer(rng, conn_ctx)),
),
(
if opts.indexes && !env.opts.disable_where_true_false_null {
remaining_.select / 2
} else {
0
},
Box::new(|rng: &mut R| property_where_true_false_null(rng, conn_ctx)),
),
(
if opts.indexes && !env.opts.disable_union_all_preserves_cardinality {
remaining_.select / 3
} else {
0
},
Box::new(|rng: &mut R| property_union_all_preserves_cardinality(rng, conn_ctx)),
),
(
if env.profile.io.enable && !env.opts.disable_fsync_no_wait {
50 // Freestyle number
} else {
0
},
Box::new(|rng: &mut R| property_fsync_no_wait(rng, &remaining_, conn_ctx)),
),
(
if env.profile.io.enable
&& env.profile.io.fault.enable
&& !env.opts.disable_faulty_query
{
20
} else {
0
},
Box::new(|rng: &mut R| property_faulty_query(rng, &remaining_, conn_ctx)),
),
],
rng,
)
#[allow(clippy::type_complexity)]
let choices: Vec<(_, Box<dyn Fn(&mut R) -> Property>)> = vec![
(
if !env.opts.disable_insert_values_select {
u32::min(remaining_.select, remaining_.insert).max(1)
} else {
0
},
Box::new(|rng: &mut R| property_insert_values_select(rng, &remaining_, conn_ctx)),
),
(
remaining_.select.max(1),
Box::new(|rng: &mut R| property_table_has_expected_content(rng, conn_ctx)),
),
(
u32::min(remaining_.select, remaining_.insert).max(1),
Box::new(|rng: &mut R| property_read_your_updates_back(rng, conn_ctx)),
),
(
if !env.opts.disable_double_create_failure {
remaining_.create / 2
} else {
0
},
Box::new(|rng: &mut R| property_double_create_failure(rng, &remaining_, conn_ctx)),
),
(
if !env.opts.disable_select_limit {
remaining_.select
} else {
0
},
Box::new(|rng: &mut R| property_select_limit(rng, conn_ctx)),
),
(
if !env.opts.disable_delete_select {
u32::min(remaining_.select, remaining_.insert).min(remaining_.delete)
} else {
0
},
Box::new(|rng: &mut R| property_delete_select(rng, &remaining_, conn_ctx)),
),
(
if !env.opts.disable_drop_select {
// remaining_.drop
0
} else {
0
},
Box::new(|rng: &mut R| property_drop_select(rng, &remaining_, conn_ctx)),
),
(
if !env.opts.disable_select_optimizer {
remaining_.select / 2
} else {
0
},
Box::new(|rng: &mut R| property_select_select_optimizer(rng, conn_ctx)),
),
(
if opts.indexes && !env.opts.disable_where_true_false_null {
remaining_.select / 2
} else {
0
},
Box::new(|rng: &mut R| property_where_true_false_null(rng, conn_ctx)),
),
(
if opts.indexes && !env.opts.disable_union_all_preserves_cardinality {
remaining_.select / 3
} else {
0
},
Box::new(|rng: &mut R| property_union_all_preserves_cardinality(rng, conn_ctx)),
),
(
if env.profile.io.enable && !env.opts.disable_fsync_no_wait {
50 // Freestyle number
} else {
0
},
Box::new(|rng: &mut R| property_fsync_no_wait(rng, &remaining_, conn_ctx)),
),
(
if env.profile.io.enable
&& env.profile.io.fault.enable
&& !env.opts.disable_faulty_query
{
20
} else {
0
},
Box::new(|rng: &mut R| property_faulty_query(rng, &remaining_, conn_ctx)),
),
];
frequency(choices, rng)
}
}

139
simulator/runner/differential.rs
View File

@@ -1,4 +1,11 @@
use std::sync::{Arc, Mutex};
use std::{
collections::{BTreeMap, btree_map::Entry},
sync::{Arc, Mutex},
};
use itertools::Itertools;
use similar_asserts::SimpleDiff;
use sql_generation::model::table::SimValue;
use crate::generation::plan::{ConnectionState, Interaction, InteractionPlanState};
@@ -83,6 +90,7 @@ pub(crate) fn execute_interactions(
last_execution.interaction_index = state.interaction_pointer;
let mut turso_state = state.clone();
let mut rusqlite_state = state.clone();
// first execute turso
let turso_res = super::execution::execute_plan(
@@ -92,8 +100,6 @@ pub(crate) fn execute_interactions(
&mut turso_state,
);
let mut rusqlite_state = state.clone();
// second execute rusqlite
let rusqlite_res = super::execution::execute_plan(
&mut rusqlite_env,
@@ -112,7 +118,9 @@ pub(crate) fn execute_interactions(
return ExecutionResult::new(history, Some(err));
}
state.interaction_pointer += 1;
assert_eq!(turso_state, rusqlite_state);
*state = turso_state;
// Check if the maximum time for the simulation has been reached
if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 {
@@ -136,79 +144,79 @@ fn compare_results(
) -> turso_core::Result<()> {
match (turso_res, rusqlite_res) {
(Ok(..), Ok(..)) => {
let limbo_values = turso_conn_state.stack.last();
let turso_values = turso_conn_state.stack.last();
let rusqlite_values = rusqlite_conn_state.stack.last();
match (limbo_values, rusqlite_values) {
(Some(limbo_values), Some(rusqlite_values)) => {
match (limbo_values, rusqlite_values) {
(Ok(limbo_values), Ok(rusqlite_values)) => {
if limbo_values != rusqlite_values {
match (turso_values, rusqlite_values) {
(Some(turso_values), Some(rusqlite_values)) => {
match (turso_values, rusqlite_values) {
(Ok(turso_values), Ok(rusqlite_values)) => {
if !compare_order_insensitive(turso_values, rusqlite_values) {
tracing::error!(
"returned values from limbo and rusqlite results do not match"
);
let diff = limbo_values
.iter()
.zip(rusqlite_values.iter())
.enumerate()
.filter(|(_, (l, r))| l != r)
.collect::<Vec<_>>();
let diff = diff
.iter()
.flat_map(|(i, (l, r))| {
let mut diffs = vec![];
for (j, (l, r)) in l.iter().zip(r.iter()).enumerate() {
if l != r {
tracing::debug!(
"difference at index {}, {}: {} != {}",
i,
j,
l.to_string(),
r.to_string()
);
diffs.push(((i, j), (l.clone(), r.clone())));
}
fn val_to_string(sim_val: &SimValue) -> String {
match &sim_val.0 {
turso_core::Value::Blob(blob) => {
let convert_blob = || -> anyhow::Result<String> {
let val = String::from_utf8(blob.clone())?;
Ok(val)
};
convert_blob().unwrap_or_else(|_| sim_val.to_string())
}
diffs
})
.collect::<Vec<_>>();
tracing::debug!("limbo values {:?}", limbo_values);
tracing::debug!("rusqlite values {:?}", rusqlite_values);
tracing::debug!(
"differences: {}",
diff.iter()
.map(|((i, j), (l, r))| format!(
"\t({i}, {j}): ({l}) != ({r})"
))
.collect::<Vec<_>>()
.join("\n")
_ => sim_val.to_string(),
}
}
let turso_string_values: Vec<Vec<_>> = turso_values
.iter()
.map(|rows| rows.iter().map(val_to_string).collect())
.sorted()
.collect();
let rusqlite_string_values: Vec<Vec<_>> = rusqlite_values
.iter()
.map(|rows| rows.iter().map(val_to_string).collect())
.sorted()
.collect();
let turso_string = format!("{turso_string_values:#?}");
let rusqlite_string = format!("{rusqlite_string_values:#?}");
let diff = SimpleDiff::from_str(
&turso_string,
&rusqlite_string,
"turso",
"rusqlite",
);
tracing::error!(%diff);
return Err(turso_core::LimboError::InternalError(
"returned values from limbo and rusqlite results do not match"
.into(),
));
}
}
(Err(limbo_err), Err(rusqlite_err)) => {
(Err(turso_err), Err(rusqlite_err)) => {
tracing::warn!("limbo and rusqlite both fail, requires manual check");
tracing::warn!("limbo error {}", limbo_err);
tracing::warn!("limbo error {}", turso_err);
tracing::warn!("rusqlite error {}", rusqlite_err);
}
(Ok(limbo_result), Err(rusqlite_err)) => {
(Ok(turso_err), Err(rusqlite_err)) => {
tracing::error!(
"limbo and rusqlite results do not match, limbo returned values but rusqlite failed"
);
tracing::error!("limbo values {:?}", limbo_result);
tracing::error!("limbo values {:?}", turso_err);
tracing::error!("rusqlite error {}", rusqlite_err);
return Err(turso_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
(Err(limbo_err), Ok(_)) => {
(Err(turso_err), Ok(_)) => {
tracing::error!(
"limbo and rusqlite results do not match, limbo failed but rusqlite returned values"
);
tracing::error!("limbo error {}", limbo_err);
tracing::error!("limbo error {}", turso_err);
return Err(turso_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
@@ -247,3 +255,34 @@ fn compare_results(
}
Ok(())
}
fn count_rows(values: &[Vec<SimValue>]) -> BTreeMap<&Vec<SimValue>, i32> {
let mut counter = BTreeMap::new();
for row in values.iter() {
match counter.entry(row) {
Entry::Vacant(entry) => {
entry.insert(1);
}
Entry::Occupied(mut entry) => {
let counter = entry.get_mut();
*counter += 1;
}
}
}
counter
}
fn compare_order_insensitive(
turso_values: &[Vec<SimValue>],
rusqlite_values: &[Vec<SimValue>],
) -> bool {
if turso_values.len() != rusqlite_values.len() {
return false;
}
let turso_counter = count_rows(turso_values);
let rusqlite_counter = count_rows(rusqlite_values);
turso_counter == rusqlite_counter
}

4
simulator/runner/doublecheck.rs
View File

@@ -142,7 +142,9 @@ pub(crate) fn execute_plans(
return ExecutionResult::new(history, Some(err));
}
state.interaction_pointer += 1;
assert_eq!(turso_state, doublecheck_state);
*state = turso_state;
// Check if the maximum time for the simulation has been reached
if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 {

8
simulator/runner/env.rs
View File

@@ -269,7 +269,7 @@ impl SimulatorEnv {
) -> Self {
let mut rng = ChaCha8Rng::seed_from_u64(seed);
let opts = SimulatorOpts {
let mut opts = SimulatorOpts {
seed,
ticks: rng.random_range(cli_opts.minimum_tests..=cli_opts.maximum_tests),
max_tables: rng.random_range(0..128),
@@ -320,6 +320,12 @@ impl SimulatorEnv {
if let Some(min_tick) = cli_opts.min_tick {
profile.io.latency.min_tick = min_tick;
}
if cli_opts.differential {
// Disable faults when running against sqlite as we cannot control faults on it
profile.io.enable = false;
// Disable limits due to differences in return order from turso and rusqlite
opts.disable_select_limit = true;
}
profile.validate().unwrap();