From e7c501a3bede40b32509fb263921cc4aa4f6aa93 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 15 Feb 2025 16:27:11 +0400 Subject: [PATCH 1/5] add simple fuzz test with table data --- tests/integration/fuzz/mod.rs | 244 +++++++++++++++++++++++++++------- 1 file changed, 199 insertions(+), 45 deletions(-) diff --git a/tests/integration/fuzz/mod.rs b/tests/integration/fuzz/mod.rs index a9bc88360..7467c2c30 100644 --- a/tests/integration/fuzz/mod.rs +++ b/tests/integration/fuzz/mod.rs @@ -4,7 +4,7 @@ pub mod grammar_generator; mod tests { use std::rc::Rc; - use rand::SeedableRng; + use rand::{RngCore, SeedableRng}; use rand_chacha::ChaCha8Rng; use rusqlite::params; @@ -22,50 +22,62 @@ mod tests { (rng, seed) } - fn sqlite_exec_row(conn: &rusqlite::Connection, query: &str) -> Vec { + fn sqlite_exec_rows( + conn: &rusqlite::Connection, + query: &str, + ) -> Vec> { let mut stmt = conn.prepare(&query).unwrap(); let mut rows = stmt.query(params![]).unwrap(); - let mut columns = Vec::new(); - let row = rows.next().unwrap().unwrap(); - for i in 0.. { - let column: rusqlite::types::Value = match row.get(i) { - Ok(column) => column, - Err(rusqlite::Error::InvalidColumnIndex(_)) => break, - Err(err) => panic!("unexpected rusqlite error: {}", err), - }; - columns.push(column); + let mut results = Vec::new(); + while let Some(row) = rows.next().unwrap() { + let mut result = Vec::new(); + for i in 0.. { + let column: rusqlite::types::Value = match row.get(i) { + Ok(column) => column, + Err(rusqlite::Error::InvalidColumnIndex(_)) => break, + Err(err) => panic!("unexpected rusqlite error: {}", err), + }; + result.push(column); + } + results.push(result) } - assert!(rows.next().unwrap().is_none()); - columns + results } - fn limbo_exec_row( + fn limbo_exec_rows( conn: &Rc, query: &str, - ) -> Vec { + ) -> Vec> { let mut stmt = conn.prepare(query).unwrap(); - let result = stmt.step().unwrap(); - let row = loop { - match result { - limbo_core::StepResult::Row => { - let row = stmt.row().unwrap(); - break row; + let mut rows = Vec::new(); + 'outer: loop { + let row = loop { + let result = stmt.step().unwrap(); + match result { + limbo_core::StepResult::Row => { + let row = stmt.row().unwrap(); + break row; + } + limbo_core::StepResult::IO => continue, + limbo_core::StepResult::Done => break 'outer, + r => panic!("unexpected result {:?}: expecting single row", r), } - limbo_core::StepResult::IO => continue, - r => panic!("unexpected result {:?}: expecting single row", r), - } - }; - row.get_values() - .iter() - .map(|x| match x.to_value() { - limbo_core::Value::Null => rusqlite::types::Value::Null, - limbo_core::Value::Integer(x) => rusqlite::types::Value::Integer(x), - limbo_core::Value::Float(x) => rusqlite::types::Value::Real(x), - limbo_core::Value::Text(x) => rusqlite::types::Value::Text(x.to_string()), - limbo_core::Value::Blob(x) => rusqlite::types::Value::Blob(x.to_vec()), - }) - .collect() + }; + let row = row + .get_values() + .iter() + .map(|x| match x.to_value() { + limbo_core::Value::Null => rusqlite::types::Value::Null, + limbo_core::Value::Integer(x) => rusqlite::types::Value::Integer(x), + limbo_core::Value::Float(x) => rusqlite::types::Value::Real(x), + limbo_core::Value::Text(x) => rusqlite::types::Value::Text(x.to_string()), + limbo_core::Value::Blob(x) => rusqlite::types::Value::Blob(x.to_vec()), + }) + .collect(); + rows.push(row); + } + rows } #[test] @@ -78,8 +90,8 @@ mod tests { "SELECT ~1 >> 1536", "SELECT ~ + 3 << - ~ (~ (8)) - + -1 - 3 >> 3 + -6 * (-7 * 9 >> - 2)", ] { - let limbo = limbo_exec_row(&limbo_conn, query); - let sqlite = sqlite_exec_row(&sqlite_conn, query); + let limbo = limbo_exec_rows(&limbo_conn, query); + let sqlite = sqlite_exec_rows(&sqlite_conn, query); assert_eq!( limbo, sqlite, "query: {}, limbo: {:?}, sqlite: {:?}", @@ -140,8 +152,8 @@ mod tests { log::info!("seed: {}", seed); for _ in 0..1024 { let query = g.generate(&mut rng, sql, 50); - let limbo = limbo_exec_row(&limbo_conn, &query); - let sqlite = sqlite_exec_row(&sqlite_conn, &query); + let limbo = limbo_exec_rows(&limbo_conn, &query); + let sqlite = sqlite_exec_rows(&sqlite_conn, &query); assert_eq!( limbo, sqlite, "query: {}, limbo: {:?}, sqlite: {:?}", @@ -167,8 +179,8 @@ mod tests { "SELECT (COALESCE(0, COALESCE(0, 0)));", "SELECT CAST((1 > 0) AS INTEGER);", ] { - let limbo = limbo_exec_row(&limbo_conn, query); - let sqlite = sqlite_exec_row(&sqlite_conn, query); + let limbo = limbo_exec_rows(&limbo_conn, query); + let sqlite = sqlite_exec_rows(&sqlite_conn, query); assert_eq!( limbo, sqlite, "query: {}, limbo: {:?}, sqlite: {:?}", @@ -274,8 +286,8 @@ mod tests { for _ in 0..1024 { let query = g.generate(&mut rng, sql, 50); log::info!("query: {}", query); - let limbo = limbo_exec_row(&limbo_conn, &query); - let sqlite = sqlite_exec_row(&sqlite_conn, &query); + let limbo = limbo_exec_rows(&limbo_conn, &query); + let sqlite = sqlite_exec_rows(&sqlite_conn, &query); assert_eq!( limbo, sqlite, "query: {}, limbo: {:?}, sqlite: {:?}", @@ -451,8 +463,150 @@ mod tests { for _ in 0..1024 { let query = g.generate(&mut rng, sql, 50); log::info!("query: {}", query); - let limbo = limbo_exec_row(&limbo_conn, &query); - let sqlite = sqlite_exec_row(&sqlite_conn, &query); + let limbo = limbo_exec_rows(&limbo_conn, &query); + let sqlite = sqlite_exec_rows(&sqlite_conn, &query); + assert_eq!( + limbo, sqlite, + "query: {}, limbo: {:?}, sqlite: {:?}", + query, limbo, sqlite + ); + } + } + + #[test] + pub fn table_logical_expression_fuzz_ex1() { + let _ = env_logger::try_init(); + + for queries in [[ + "CREATE TABLE t(x)", + "INSERT INTO t VALUES (10)", + "SELECT * FROM t WHERE x = 1 AND 1 OR 0", + ]] { + let db = TempDatabase::new_empty(); + let limbo_conn = db.connect_limbo(); + let sqlite_conn = rusqlite::Connection::open_in_memory().unwrap(); + for query in queries.iter() { + let limbo = limbo_exec_rows(&limbo_conn, query); + let sqlite = sqlite_exec_rows(&sqlite_conn, query); + assert_eq!( + limbo, sqlite, + "queries: {:?}, query: {}, limbo: {:?}, sqlite: {:?}", + queries, query, limbo, sqlite + ); + } + } + } + + #[test] + pub fn table_logical_expression_fuzz_run() { + let _ = env_logger::try_init(); + let g = GrammarGenerator::new(); + let (expr, expr_builder) = g.create_handle(); + let (value, value_builder) = g.create_handle(); + let (cmp_op, cmp_op_builder) = g.create_handle(); + let (bin_op, bin_op_builder) = g.create_handle(); + let (in_op, in_op_builder) = g.create_handle(); + let (paren, paren_builder) = g.create_handle(); + + value_builder + .choice() + .option( + g.create() + .concat(" ") + .push_str("(") + .push(value) + .push_str(")") + .build(), + ) + .option_symbol(rand_int(0..i32::MAX)) + .options_str(["x", "y", "z"]) + .option( + g.create() + .concat(" ") + .push_str("(") + .push(value) + .push(g.create().choice().options_str(["+", "-"]).build()) + .push(value) + .push_str(")") + .build(), + ) + .build(); + + paren_builder + .concat("") + .push_str("(") + .push(expr) + .push_str(")") + .build(); + + cmp_op_builder + .concat(" ") + .push(value) + .push( + g.create() + .choice() + .options_str(["=", "<>", ">", "<", ">=", "<=", "IS", "IS NOT"]) + .build(), + ) + .push(value) + .build(); + + bin_op_builder + .concat(" ") + .push(expr) + .push(g.create().choice().options_str(["AND", "OR"]).build()) + .push(expr) + .build(); + + in_op_builder + .concat(" ") + .push(value) + .push(g.create().choice().options_str(["IN", "NOT IN"]).build()) + .push_str("(") + .push(g.create().concat("").push(value).repeat(1..5, ", ").build()) + .push_str(")") + .build(); + + expr_builder + .choice() + .options_str(["1", "0"]) + .option_w(paren, 10.0) + .option_w(cmp_op, 10.0) + .option_w(bin_op, 10.0) + .option_w(in_op, 10.0) + .build(); + + let db = TempDatabase::new_empty(); + let limbo_conn = db.connect_limbo(); + let sqlite_conn = rusqlite::Connection::open_in_memory().unwrap(); + assert_eq!( + limbo_exec_rows(&limbo_conn, "CREATE TABLE t(x, y, z)"), + sqlite_exec_rows(&sqlite_conn, "CREATE TABLE t(x, y, z)") + ); + let (mut rng, seed) = rng_from_time(); + log::info!("seed: {}", seed); + + for _ in 0..100 { + let (x, y, z) = (rng.next_u32(), rng.next_u32(), rng.next_u32()); + let query = format!("INSERT INTO t VALUES ({}, {}, {})", x, y, z); + assert_eq!( + limbo_exec_rows(&limbo_conn, &query), + sqlite_exec_rows(&sqlite_conn, &query) + ); + } + + let sql = g + .create() + .concat(" ") + .push_str("SELECT * FROM t WHERE ") + .push(expr) + .build(); + + for _ in 0..128 { + let query = g.generate(&mut rng, sql, 50); + log::info!("query: {}", query); + let limbo = limbo_exec_rows(&limbo_conn, &query); + let sqlite = sqlite_exec_rows(&sqlite_conn, &query); assert_eq!( limbo, sqlite, "query: {}, limbo: {:?}, sqlite: {:?}", From ee8b03528d8d73da9d2f793112489cb033c884da Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 15 Feb 2025 16:27:27 +0400 Subject: [PATCH 2/5] fix codegen for and predicate - as jump_if_condition_is_true can be overwritten higher in the stack --- core/translate/expr.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 9bfcc4a8f..cff8f0c85 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -197,6 +197,7 @@ pub fn translate_condition_expr( referenced_tables, lhs, ConditionMetadata { + jump_if_condition_is_true: false, jump_target_when_true, ..condition_metadata }, From 3233d60beaf4ba45c11f8d337ef4e5322bc2aa99 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 15 Feb 2025 18:00:21 +0400 Subject: [PATCH 3/5] trying to debug hungs in CI --- tests/integration/fuzz/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/fuzz/mod.rs b/tests/integration/fuzz/mod.rs index 7467c2c30..ad0805bd3 100644 --- a/tests/integration/fuzz/mod.rs +++ b/tests/integration/fuzz/mod.rs @@ -479,8 +479,8 @@ mod tests { for queries in [[ "CREATE TABLE t(x)", - "INSERT INTO t VALUES (10)", - "SELECT * FROM t WHERE x = 1 AND 1 OR 0", + // "INSERT INTO t VALUES (10)", + // "SELECT * FROM t WHERE x = 1 AND 1 OR 0", ]] { let db = TempDatabase::new_empty(); let limbo_conn = db.connect_limbo(); @@ -497,7 +497,7 @@ mod tests { } } - #[test] + // #[test] pub fn table_logical_expression_fuzz_run() { let _ = env_logger::try_init(); let g = GrammarGenerator::new(); From becc58565dbe46329c1a20a479473e1a234b69ba Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 15 Feb 2025 22:48:01 +0400 Subject: [PATCH 4/5] run IO in fuzz tests --- tests/integration/common.rs | 3 ++- tests/integration/fuzz/mod.rs | 30 +++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/tests/integration/common.rs b/tests/integration/common.rs index 521ef8632..06501769a 100644 --- a/tests/integration/common.rs +++ b/tests/integration/common.rs @@ -1,4 +1,5 @@ use limbo_core::{CheckpointStatus, Connection, Database, IO}; +use rand::{rng, RngCore}; use std::path::PathBuf; use std::rc::Rc; use std::sync::Arc; @@ -13,7 +14,7 @@ pub struct TempDatabase { #[allow(dead_code, clippy::arc_with_non_send_sync)] impl TempDatabase { pub fn new_empty() -> Self { - Self::new("test.db") + Self::new(&format!("test-{}.db", rng().next_u32())) } pub fn new(db_name: &str) -> Self { diff --git a/tests/integration/fuzz/mod.rs b/tests/integration/fuzz/mod.rs index ad0805bd3..b6d363af4 100644 --- a/tests/integration/fuzz/mod.rs +++ b/tests/integration/fuzz/mod.rs @@ -46,6 +46,7 @@ mod tests { } fn limbo_exec_rows( + db: &TempDatabase, conn: &Rc, query: &str, ) -> Vec> { @@ -59,7 +60,10 @@ mod tests { let row = stmt.row().unwrap(); break row; } - limbo_core::StepResult::IO => continue, + limbo_core::StepResult::IO => { + db.io.run_once().unwrap(); + continue; + } limbo_core::StepResult::Done => break 'outer, r => panic!("unexpected result {:?}: expecting single row", r), } @@ -90,7 +94,7 @@ mod tests { "SELECT ~1 >> 1536", "SELECT ~ + 3 << - ~ (~ (8)) - + -1 - 3 >> 3 + -6 * (-7 * 9 >> - 2)", ] { - let limbo = limbo_exec_rows(&limbo_conn, query); + let limbo = limbo_exec_rows(&db, &limbo_conn, query); let sqlite = sqlite_exec_rows(&sqlite_conn, query); assert_eq!( limbo, sqlite, @@ -152,7 +156,7 @@ mod tests { log::info!("seed: {}", seed); for _ in 0..1024 { let query = g.generate(&mut rng, sql, 50); - let limbo = limbo_exec_rows(&limbo_conn, &query); + let limbo = limbo_exec_rows(&db, &limbo_conn, &query); let sqlite = sqlite_exec_rows(&sqlite_conn, &query); assert_eq!( limbo, sqlite, @@ -179,7 +183,7 @@ mod tests { "SELECT (COALESCE(0, COALESCE(0, 0)));", "SELECT CAST((1 > 0) AS INTEGER);", ] { - let limbo = limbo_exec_rows(&limbo_conn, query); + let limbo = limbo_exec_rows(&db, &limbo_conn, query); let sqlite = sqlite_exec_rows(&sqlite_conn, query); assert_eq!( limbo, sqlite, @@ -286,7 +290,7 @@ mod tests { for _ in 0..1024 { let query = g.generate(&mut rng, sql, 50); log::info!("query: {}", query); - let limbo = limbo_exec_rows(&limbo_conn, &query); + let limbo = limbo_exec_rows(&db, &limbo_conn, &query); let sqlite = sqlite_exec_rows(&sqlite_conn, &query); assert_eq!( limbo, sqlite, @@ -463,7 +467,7 @@ mod tests { for _ in 0..1024 { let query = g.generate(&mut rng, sql, 50); log::info!("query: {}", query); - let limbo = limbo_exec_rows(&limbo_conn, &query); + let limbo = limbo_exec_rows(&db, &limbo_conn, &query); let sqlite = sqlite_exec_rows(&sqlite_conn, &query); assert_eq!( limbo, sqlite, @@ -479,14 +483,14 @@ mod tests { for queries in [[ "CREATE TABLE t(x)", - // "INSERT INTO t VALUES (10)", - // "SELECT * FROM t WHERE x = 1 AND 1 OR 0", + "INSERT INTO t VALUES (10)", + "SELECT * FROM t WHERE x = 1 AND 1 OR 0", ]] { let db = TempDatabase::new_empty(); let limbo_conn = db.connect_limbo(); let sqlite_conn = rusqlite::Connection::open_in_memory().unwrap(); for query in queries.iter() { - let limbo = limbo_exec_rows(&limbo_conn, query); + let limbo = limbo_exec_rows(&db, &limbo_conn, query); let sqlite = sqlite_exec_rows(&sqlite_conn, query); assert_eq!( limbo, sqlite, @@ -497,7 +501,7 @@ mod tests { } } - // #[test] + #[test] pub fn table_logical_expression_fuzz_run() { let _ = env_logger::try_init(); let g = GrammarGenerator::new(); @@ -580,7 +584,7 @@ mod tests { let limbo_conn = db.connect_limbo(); let sqlite_conn = rusqlite::Connection::open_in_memory().unwrap(); assert_eq!( - limbo_exec_rows(&limbo_conn, "CREATE TABLE t(x, y, z)"), + limbo_exec_rows(&db, &limbo_conn, "CREATE TABLE t(x, y, z)"), sqlite_exec_rows(&sqlite_conn, "CREATE TABLE t(x, y, z)") ); let (mut rng, seed) = rng_from_time(); @@ -590,7 +594,7 @@ mod tests { let (x, y, z) = (rng.next_u32(), rng.next_u32(), rng.next_u32()); let query = format!("INSERT INTO t VALUES ({}, {}, {})", x, y, z); assert_eq!( - limbo_exec_rows(&limbo_conn, &query), + limbo_exec_rows(&db, &limbo_conn, &query), sqlite_exec_rows(&sqlite_conn, &query) ); } @@ -605,7 +609,7 @@ mod tests { for _ in 0..128 { let query = g.generate(&mut rng, sql, 50); log::info!("query: {}", query); - let limbo = limbo_exec_rows(&limbo_conn, &query); + let limbo = limbo_exec_rows(&db, &limbo_conn, &query); let sqlite = sqlite_exec_rows(&sqlite_conn, &query); assert_eq!( limbo, sqlite, From 77d4bb6e0e6e1d19440eae7bab3f1b7aaee3bc5e Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 15 Feb 2025 23:00:30 +0400 Subject: [PATCH 5/5] fix after merge --- tests/integration/fuzz/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/fuzz/mod.rs b/tests/integration/fuzz/mod.rs index 010d9cb4e..be6fe5e06 100644 --- a/tests/integration/fuzz/mod.rs +++ b/tests/integration/fuzz/mod.rs @@ -278,9 +278,9 @@ mod tests { for _ in 0..1024 { let query = g.generate(&mut rng, sql, 50); log::info!("query: {}", query); - let limbo = limbo_exec_row(&limbo_conn, &query); - let sqlite = sqlite_exec_row(&sqlite_conn, &query); - match (&limbo[0], &sqlite[0]) { + let limbo = limbo_exec_rows(&db, &limbo_conn, &query); + let sqlite = sqlite_exec_rows(&sqlite_conn, &query); + match (&limbo[0][0], &sqlite[0][0]) { // compare only finite results because some evaluations are not so stable around infinity (rusqlite::types::Value::Real(limbo), rusqlite::types::Value::Real(sqlite)) if limbo.is_finite() && sqlite.is_finite() =>