From 2a4d461627e966d73736984de4f852baadaf41a9 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 6 Jan 2025 18:13:21 +0300 Subject: [PATCH 01/97] previously, interactions plans were comprised of a flat sequence of operations that did not reflect the internal structure, in which they were actually concatenations of properties, which are a coherent set of interactions that are meaningful by themselves. this commit introduces this semantic layer into the data model by turning interaction plans into a sequence of properties, which are a sequence of interactions --- simulator/generation/plan.rs | 124 +++++++++++++++++++++++------------ simulator/main.rs | 11 +++- 2 files changed, 91 insertions(+), 44 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index dbf6dd7f6..fafd3741b 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -19,20 +19,53 @@ use super::{pick, pick_index}; pub(crate) type ResultSet = Result>>; pub(crate) struct InteractionPlan { - pub(crate) plan: Vec, + pub(crate) plan: Vec, pub(crate) stack: Vec, pub(crate) interaction_pointer: usize, + pub(crate) secondary_pointer: usize, +} + +pub(crate) struct Property { + pub(crate) name: Option, + pub(crate) interactions: Vec, +} + +impl Property { + pub(crate) fn new(name: Option, interactions: Vec) -> Self { + Self { name, interactions } + } + + pub(crate) fn anonymous(interactions: Vec) -> Self { + Self { + name: None, + interactions, + } + } } impl Display for InteractionPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for interaction in &self.plan { - match interaction { - Interaction::Query(query) => writeln!(f, "{};", query)?, - Interaction::Assertion(assertion) => { - writeln!(f, "-- ASSERT: {};", assertion.message)? + for property in &self.plan { + if let Some(name) = &property.name { + writeln!(f, "-- begin testing '{}'", name)?; + } + + for interaction in &property.interactions { + if property.name.is_some() { + write!(f, "\t")?; } - Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + + match interaction { + Interaction::Query(query) => writeln!(f, "{};", query)?, + Interaction::Assertion(assertion) => { + writeln!(f, "-- ASSERT: {};", assertion.message)? + } + Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + } + } + + if let Some(name) = &property.name { + writeln!(f, "-- end testing '{}'", name)?; } } @@ -93,11 +126,9 @@ impl Display for Fault { } } -pub(crate) struct Interactions(Vec); - -impl Interactions { +impl Property { pub(crate) fn shadow(&self, env: &mut SimulatorEnv) { - for interaction in &self.0 { + for interaction in &self.interactions { match interaction { Interaction::Query(query) => match query { Query::Create(create) => { @@ -129,29 +160,28 @@ impl InteractionPlan { plan: Vec::new(), stack: Vec::new(), interaction_pointer: 0, + secondary_pointer: 0, } } - pub(crate) fn push(&mut self, interaction: Interaction) { - self.plan.push(interaction); - } - pub(crate) fn stats(&self) -> InteractionStats { let mut read = 0; let mut write = 0; let mut delete = 0; let mut create = 0; - for interaction in &self.plan { - match interaction { - Interaction::Query(query) => match query { - Query::Select(_) => read += 1, - Query::Insert(_) => write += 1, - Query::Delete(_) => delete += 1, - Query::Create(_) => create += 1, - }, - Interaction::Assertion(_) => {} - Interaction::Fault(_) => {} + for property in &self.plan { + for interaction in &property.interactions { + match interaction { + Interaction::Query(query) => match query { + Query::Select(_) => read += 1, + Query::Insert(_) => write += 1, + Query::Delete(_) => delete += 1, + Query::Create(_) => create += 1, + }, + Interaction::Assertion(_) => {} + Interaction::Fault(_) => {} + } } } @@ -182,7 +212,11 @@ impl ArbitraryFrom for InteractionPlan { // First create at least one table let create_query = Create::arbitrary(rng); env.tables.push(create_query.table.clone()); - plan.push(Interaction::Query(Query::Create(create_query))); + + plan.plan.push(Property { + name: Some("initial table creation".to_string()), + interactions: vec![Interaction::Query(Query::Create(create_query))], + }); while plan.plan.len() < num_interactions { log::debug!( @@ -190,10 +224,10 @@ impl ArbitraryFrom for InteractionPlan { plan.plan.len(), num_interactions ); - let interactions = Interactions::arbitrary_from(rng, &(&env, plan.stats())); - interactions.shadow(&mut env); + let property = Property::arbitrary_from(rng, &(&env, plan.stats())); + property.shadow(&mut env); - plan.plan.extend(interactions.0.into_iter()); + plan.plan.push(property); } log::info!("Generated plan with {} interactions", plan.plan.len()); @@ -306,7 +340,7 @@ impl Interaction { } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { // Get a random table let table = pick(&env.tables, rng); // Pick a random column @@ -354,10 +388,13 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Inte }), }); - Interactions(vec![insert_query, select_query, assertion]) + Property::new( + Some("select contains inserted value".to_string()), + vec![insert_query, select_query, assertion], + ) } -fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv) -> Property { let create_query = Create::arbitrary(rng); let table_name = create_query.table.name.clone(); let cq1 = Interaction::Query(Query::Create(create_query.clone())); @@ -378,31 +415,34 @@ fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv }), }); - Interactions(vec![cq1, cq2, assertion]) + Property::new( + Some("creating the same table twice fails".to_string()), + vec![cq1, cq2, assertion], + ) } -fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Property { let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng))); - Interactions(vec![create_query]) + Property::anonymous(vec![create_query]) } -fn random_read(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn random_read(rng: &mut R, env: &SimulatorEnv) -> Property { let select_query = Interaction::Query(Query::Select(Select::arbitrary_from(rng, &env.tables))); - Interactions(vec![select_query]) + Property::anonymous(vec![select_query]) } -fn random_write(rng: &mut R, env: &SimulatorEnv) -> Interactions { +fn random_write(rng: &mut R, env: &SimulatorEnv) -> Property { let table = pick(&env.tables, rng); let insert_query = Interaction::Query(Query::Insert(Insert::arbitrary_from(rng, table))); - Interactions(vec![insert_query]) + Property::anonymous(vec![insert_query]) } -fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Interactions { +fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Property { let fault = Interaction::Fault(Fault::Disconnect); - Interactions(vec![fault]) + Property::anonymous(vec![fault]) } -impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { +impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { fn arbitrary_from( rng: &mut R, (env, stats): &(&SimulatorEnv, InteractionStats), diff --git a/simulator/main.rs b/simulator/main.rs index 52c33d5ec..794c9c041 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -266,7 +266,7 @@ fn execute_plan( return Ok(()); } - let interaction = &plan.plan[plan.interaction_pointer]; + let interaction = &plan.plan[plan.interaction_pointer].interactions[plan.secondary_pointer]; if let SimConnection::Disconnected = connection { log::info!("connecting {}", connection_index); @@ -275,7 +275,14 @@ fn execute_plan( match execute_interaction(env, connection_index, interaction, &mut plan.stack) { Ok(_) => { log::debug!("connection {} processed", connection_index); - plan.interaction_pointer += 1; + if plan.secondary_pointer + 1 + >= plan.plan[plan.interaction_pointer].interactions.len() + { + plan.interaction_pointer += 1; + plan.secondary_pointer = 0; + } else { + plan.secondary_pointer += 1; + } } Err(err) => { log::error!("error {}", err); From daa77feea13feaf2c48622d4650da9f6af8b18d7 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 6 Jan 2025 19:04:29 +0300 Subject: [PATCH 02/97] add assumptions to the interactions, where a failing assumption stops the execution of the current property and switches to the next one. --- simulator/generation/plan.rs | 72 ++++++++++++++++++++++++++++++++---- simulator/main.rs | 54 +++++++++++++++++++++------ 2 files changed, 108 insertions(+), 18 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index fafd3741b..ca3a6d503 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -57,6 +57,9 @@ impl Display for InteractionPlan { match interaction { Interaction::Query(query) => writeln!(f, "{};", query)?, + Interaction::Assumption(assumption) => { + writeln!(f, "-- ASSUME: {};", assumption.message)? + } Interaction::Assertion(assertion) => { writeln!(f, "-- ASSERT: {};", assertion.message)? } @@ -93,6 +96,7 @@ impl Display for InteractionStats { pub(crate) enum Interaction { Query(Query), + Assumption(Assertion), Assertion(Assertion), Fault(Fault), } @@ -101,13 +105,14 @@ impl Display for Interaction { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Query(query) => write!(f, "{}", query), + Self::Assumption(assumption) => write!(f, "ASSUME: {}", assumption.message), Self::Assertion(assertion) => write!(f, "ASSERT: {}", assertion.message), Self::Fault(fault) => write!(f, "FAULT: {}", fault), } } } -type AssertionFunc = dyn Fn(&Vec) -> bool; +type AssertionFunc = dyn Fn(&Vec, &SimulatorEnv) -> bool; pub(crate) struct Assertion { pub(crate) func: Box, @@ -148,6 +153,7 @@ impl Property { Query::Select(_) => {} }, Interaction::Assertion(_) => {} + Interaction::Assumption(_) => {} Interaction::Fault(_) => {} } } @@ -180,6 +186,7 @@ impl InteractionPlan { Query::Create(_) => create += 1, }, Interaction::Assertion(_) => {} + Interaction::Assumption(_) => {} Interaction::Fault(_) => {} } } @@ -285,31 +292,67 @@ impl Interaction { Self::Assertion(_) => { unreachable!("unexpected: this function should only be called on queries") } - Interaction::Fault(_) => { + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on queries") + } + Self::Fault(_) => { unreachable!("unexpected: this function should only be called on queries") } } } - pub(crate) fn execute_assertion(&self, stack: &Vec) -> Result<()> { + pub(crate) fn execute_assertion( + &self, + stack: &Vec, + env: &SimulatorEnv, + ) -> Result<()> { match self { Self::Query(_) => { unreachable!("unexpected: this function should only be called on assertions") } Self::Assertion(assertion) => { - if !assertion.func.as_ref()(stack) { + if !assertion.func.as_ref()(stack, env) { return Err(limbo_core::LimboError::InternalError( assertion.message.clone(), )); } Ok(()) } + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on assertions") + } Self::Fault(_) => { unreachable!("unexpected: this function should only be called on assertions") } } } + pub(crate) fn execute_assumption( + &self, + stack: &Vec, + env: &SimulatorEnv, + ) -> Result<()> { + match self { + Self::Query(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + Self::Assertion(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + Self::Assumption(assumption) => { + if !assumption.func.as_ref()(stack, env) { + return Err(limbo_core::LimboError::InternalError( + assumption.message.clone(), + )); + } + Ok(()) + } + Self::Fault(_) => { + unreachable!("unexpected: this function should only be called on assumptions") + } + } + } + pub(crate) fn execute_fault(&self, env: &mut SimulatorEnv, conn_index: usize) -> Result<()> { match self { Self::Query(_) => { @@ -318,6 +361,9 @@ impl Interaction { Self::Assertion(_) => { unreachable!("unexpected: this function should only be called on faults") } + Self::Assumption(_) => { + unreachable!("unexpected: this function should only be called on faults") + } Self::Fault(fault) => { match fault { Fault::Disconnect => { @@ -358,6 +404,18 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop row.push(value); } } + + // Check that the table exists + let assumption = Interaction::Assumption(Assertion { + message: format!("table {} exists", table.name), + func: Box::new({ + let table_name = table.name.clone(); + move |_: &Vec, env: &SimulatorEnv| { + env.tables.iter().any(|t| t.name == table_name) + } + }), + }); + // Insert the row let insert_query = Interaction::Query(Query::Insert(Insert { table: table.name.clone(), @@ -379,7 +437,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop column.name, value, ), - func: Box::new(move |stack: &Vec| { + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { let rows = stack.last().unwrap(); match rows { Ok(rows) => rows.iter().any(|r| r == &row), @@ -390,7 +448,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop Property::new( Some("select contains inserted value".to_string()), - vec![insert_query, select_query, assertion], + vec![assumption, insert_query, select_query, assertion], ) } @@ -404,7 +462,7 @@ fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv message: "creating two tables with the name should result in a failure for the second query" .to_string(), - func: Box::new(move |stack: &Vec| { + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { let last = stack.last().unwrap(); match last { Ok(_) => false, diff --git a/simulator/main.rs b/simulator/main.rs index 794c9c041..ec65e228f 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -273,15 +273,27 @@ fn execute_plan( env.connections[connection_index] = SimConnection::Connected(env.db.connect()); } else { match execute_interaction(env, connection_index, interaction, &mut plan.stack) { - Ok(_) => { + Ok(next_execution) => { log::debug!("connection {} processed", connection_index); - if plan.secondary_pointer + 1 - >= plan.plan[plan.interaction_pointer].interactions.len() - { - plan.interaction_pointer += 1; - plan.secondary_pointer = 0; - } else { - plan.secondary_pointer += 1; + // Move to the next interaction or property + match next_execution { + ExecutionContinuation::NextInteraction => { + if plan.secondary_pointer + 1 + >= plan.plan[plan.interaction_pointer].interactions.len() + { + // If we have reached the end of the interactions for this property, move to the next property + plan.interaction_pointer += 1; + plan.secondary_pointer = 0; + } else { + // Otherwise, move to the next interaction + plan.secondary_pointer += 1; + } + } + ExecutionContinuation::NextProperty => { + // Skip to the next property + plan.interaction_pointer += 1; + plan.secondary_pointer = 0; + } } } Err(err) => { @@ -294,12 +306,23 @@ fn execute_plan( Ok(()) } +/// The next point of control flow after executing an interaction. +/// `execute_interaction` uses this type in conjunction with a result, where +/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case +/// indicates the next step in the plan. +enum ExecutionContinuation { + /// Default continuation, execute the next interaction. + NextInteraction, + /// Typically used in the case of preconditions failures, skip to the next property. + NextProperty, +} + fn execute_interaction( env: &mut SimulatorEnv, connection_index: usize, interaction: &Interaction, stack: &mut Vec, -) -> Result<()> { +) -> Result { log::info!("executing: {}", interaction); match interaction { generation::plan::Interaction::Query(_) => { @@ -314,15 +337,24 @@ fn execute_interaction( stack.push(results); } generation::plan::Interaction::Assertion(_) => { - interaction.execute_assertion(stack)?; + interaction.execute_assertion(stack, env)?; stack.clear(); } + generation::plan::Interaction::Assumption(_) => { + let assumption_result = interaction.execute_assumption(stack, env); + stack.clear(); + + if assumption_result.is_err() { + log::warn!("assumption failed: {:?}", assumption_result); + return Ok(ExecutionContinuation::NextProperty); + } + } Interaction::Fault(_) => { interaction.execute_fault(env, connection_index)?; } } - Ok(()) + Ok(ExecutionContinuation::NextInteraction) } fn compare_equal_rows(a: &[Vec], b: &[Vec]) { From cc56276c3ab92aed808e4f0f05315c10f3e599d9 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Tue, 7 Jan 2025 13:40:29 +0300 Subject: [PATCH 03/97] add execution history to the simulator, the history records three indexes(connection, interaction pointer, and secondary pointer) that can uniquely identify the executed interaction at any point. we will use the history for shrinking purposes. --- simulator/generation/plan.rs | 21 ++---- simulator/main.rs | 129 ++++++++++++++++++++++++++--------- 2 files changed, 104 insertions(+), 46 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index ca3a6d503..0a18d3821 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -1,8 +1,6 @@ use std::{fmt::Display, rc::Rc}; use limbo_core::{Connection, Result, StepResult}; -use rand::SeedableRng; -use rand_chacha::ChaCha8Rng; use crate::{ model::{ @@ -201,19 +199,12 @@ impl InteractionPlan { } } -impl ArbitraryFrom for InteractionPlan { - fn arbitrary_from(rng: &mut R, env: &SimulatorEnv) -> Self { +impl InteractionPlan { + // todo: This is a hack to get around the fact that `ArbitraryFrom` can't take a mutable + // reference of T, so instead write a bespoke function without using the trait system. + pub(crate) fn arbitrary_from(rng: &mut R, env: &mut SimulatorEnv) -> Self { let mut plan = InteractionPlan::new(); - let mut env = SimulatorEnv { - opts: env.opts.clone(), - tables: vec![], - connections: vec![], - io: env.io.clone(), - db: env.db.clone(), - rng: ChaCha8Rng::seed_from_u64(rng.next_u64()), - }; - let num_interactions = env.opts.max_interactions; // First create at least one table @@ -231,8 +222,8 @@ impl ArbitraryFrom for InteractionPlan { plan.plan.len(), num_interactions ); - let property = Property::arbitrary_from(rng, &(&env, plan.stats())); - property.shadow(&mut env); + let property = Property::arbitrary_from(rng, &(env, plan.stats())); + property.shadow(env); plan.plan.push(property); } diff --git a/simulator/main.rs b/simulator/main.rs index ec65e228f..5491c5780 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -1,8 +1,9 @@ #![allow(clippy::arc_with_non_send_sync, dead_code)] use clap::Parser; +use core::panic; +use generation::pick_index; use generation::plan::{Interaction, InteractionPlan, ResultSet}; -use generation::{pick_index, ArbitraryFrom}; -use limbo_core::{Database, Result}; +use limbo_core::{Database, LimboError, Result}; use model::table::Value; use rand::prelude::*; use rand_chacha::ChaCha8Rng; @@ -36,10 +37,12 @@ fn main() { let db_path = output_dir.join("simulator.db"); let plan_path = output_dir.join("simulator.plan"); + let history_path = output_dir.join("simulator.history"); // Print the seed, the locations of the database and the plan file log::info!("database path: {:?}", db_path); log::info!("simulator plan path: {:?}", plan_path); + log::info!("simulator history path: {:?}", history_path); log::info!("seed: {}", seed); std::panic::set_hook(Box::new(move |info| { @@ -73,28 +76,34 @@ fn main() { std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path)); match (result, result2) { - (Ok(Ok(_)), Err(_)) => { + (Ok(ExecutionResult { error: None, .. }), Err(_)) => { log::error!("doublecheck failed! first run succeeded, but second run panicked."); } - (Ok(Err(_)), Err(_)) => { + (Ok(ExecutionResult { error: Some(_), .. }), Err(_)) => { log::error!( "doublecheck failed! first run failed assertion, but second run panicked." ); } - (Err(_), Ok(Ok(_))) => { + (Err(_), Ok(ExecutionResult { error: None, .. })) => { log::error!("doublecheck failed! first run panicked, but second run succeeded."); } - (Err(_), Ok(Err(_))) => { + (Err(_), Ok(ExecutionResult { error: Some(_), .. })) => { log::error!( "doublecheck failed! first run panicked, but second run failed assertion." ); } - (Ok(Ok(_)), Ok(Err(_))) => { + ( + Ok(ExecutionResult { error: None, .. }), + Ok(ExecutionResult { error: Some(_), .. }), + ) => { log::error!( "doublecheck failed! first run succeeded, but second run failed assertion." ); } - (Ok(Err(_)), Ok(Ok(_))) => { + ( + Ok(ExecutionResult { error: Some(_), .. }), + Ok(ExecutionResult { error: None, .. }), + ) => { log::error!( "doublecheck failed! first run failed assertion, but second run succeeded." ); @@ -122,18 +131,32 @@ fn main() { std::fs::rename(&old_db_path, &db_path).unwrap(); std::fs::rename(&old_plan_path, &plan_path).unwrap(); } else if let Ok(result) = result { - match result { - Ok(_) => { + // No panic occurred, so write the history to a file + let f = std::fs::File::create(&history_path).unwrap(); + let mut f = std::io::BufWriter::new(f); + for execution in result.history.history.iter() { + writeln!( + f, + "{} {} {}", + execution.connection_index, execution.interaction_index, execution.secondary_index + ) + .unwrap(); + } + + match result.error { + None => { log::info!("simulation completed successfully"); } - Err(e) => { + Some(e) => { log::error!("simulation failed: {:?}", e); } } } + // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. println!("database path: {:?}", db_path); println!("simulator plan path: {:?}", plan_path); + println!("simulator history path: {:?}", history_path); println!("seed: {}", seed); } @@ -142,7 +165,7 @@ fn run_simulation( cli_opts: &SimulatorCLI, db_path: &Path, plan_path: &Path, -) -> Result<()> { +) -> ExecutionResult { let mut rng = ChaCha8Rng::seed_from_u64(seed); let (create_percent, read_percent, write_percent, delete_percent) = { @@ -160,21 +183,15 @@ fn run_simulation( }; if cli_opts.minimum_size < 1 { - return Err(limbo_core::LimboError::InternalError( - "minimum size must be at least 1".to_string(), - )); + panic!("minimum size must be at least 1"); } if cli_opts.maximum_size < 1 { - return Err(limbo_core::LimboError::InternalError( - "maximum size must be at least 1".to_string(), - )); + panic!("maximum size must be at least 1"); } if cli_opts.maximum_size < cli_opts.minimum_size { - return Err(limbo_core::LimboError::InternalError( - "maximum size must be greater than or equal to minimum size".to_string(), - )); + panic!("maximum size must be greater than or equal to minimum size"); } let opts = SimulatorOpts { @@ -212,7 +229,7 @@ fn run_simulation( log::info!("Generating database interaction plan..."); let mut plans = (1..=env.opts.max_connections) - .map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &env)) + .map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &mut env)) .collect::>(); let mut f = std::fs::File::create(plan_path).unwrap(); @@ -224,9 +241,6 @@ fn run_simulation( log::info!("Executing database interaction plan..."); let result = execute_plans(&mut env, &mut plans); - if result.is_err() { - log::error!("error executing plans: {:?}", result.as_ref().err()); - } env.io.print_stats(); @@ -235,23 +249,76 @@ fn run_simulation( result } -fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> Result<()> { +struct Execution { + connection_index: usize, + interaction_index: usize, + secondary_index: usize, +} + +impl Execution { + fn new(connection_index: usize, interaction_index: usize, secondary_index: usize) -> Self { + Self { + connection_index, + interaction_index, + secondary_index, + } + } +} + +struct ExecutionHistory { + history: Vec, +} + +impl ExecutionHistory { + fn new() -> Self { + Self { + history: Vec::new(), + } + } +} + +struct ExecutionResult { + history: ExecutionHistory, + error: Option, +} + +impl ExecutionResult { + fn new(history: ExecutionHistory, error: Option) -> Self { + Self { history, error } + } +} + +fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> ExecutionResult { + let mut history = ExecutionHistory::new(); let now = std::time::Instant::now(); // todo: add history here by recording which interaction was executed at which tick for _tick in 0..env.opts.ticks { // Pick the connection to interact with let connection_index = pick_index(env.connections.len(), &mut env.rng); + history.history.push(Execution::new( + connection_index, + plans[connection_index].interaction_pointer, + plans[connection_index].secondary_pointer, + )); // Execute the interaction for the selected connection - execute_plan(env, connection_index, plans)?; + match execute_plan(env, connection_index, plans) { + Ok(_) => {} + Err(err) => { + return ExecutionResult::new(history, Some(err)); + } + } // Check if the maximum time for the simulation has been reached if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { - return Err(limbo_core::LimboError::InternalError( - "maximum time for simulation reached".into(), - )); + return ExecutionResult::new( + history, + Some(limbo_core::LimboError::InternalError( + "maximum time for simulation reached".into(), + )), + ); } } - Ok(()) + ExecutionResult::new(history, None) } fn execute_plan( From b796a972dc372d65f2e3fb734c06c34487c1fc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 19:50:50 +0900 Subject: [PATCH 04/97] Fix LimboDB.load to be static method --- .../github/tursodatabase/core/LimboDB.java | 26 +++++++++---------- .../tursodatabase/core/LimboDBTest.java | 6 ++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java index f3001aead..accbad76b 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java @@ -30,6 +30,19 @@ public final class LimboDB extends AbstractDB { } } + /** + * Loads the SQLite interface backend. + */ + public static void load() { + if (isLoaded) return; + + try { + System.loadLibrary("_limbo_java"); + } finally { + isLoaded = true; + } + } + /** * @param url e.g. "jdbc:sqlite:fileName * @param fileName e.g. path to file @@ -43,19 +56,6 @@ public final class LimboDB extends AbstractDB { super(url, fileName); } - /** - * Loads the SQLite interface backend. - */ - public void load() { - if (isLoaded) return; - - try { - System.loadLibrary("_limbo_java"); - } finally { - isLoaded = true; - } - } - // WRAPPER FUNCTIONS //////////////////////////////////////////// // TODO: add support for JNI diff --git a/bindings/java/src/test/java/org/github/tursodatabase/core/LimboDBTest.java b/bindings/java/src/test/java/org/github/tursodatabase/core/LimboDBTest.java index feeeff060..66e842ea4 100644 --- a/bindings/java/src/test/java/org/github/tursodatabase/core/LimboDBTest.java +++ b/bindings/java/src/test/java/org/github/tursodatabase/core/LimboDBTest.java @@ -15,16 +15,16 @@ public class LimboDBTest { @Test void db_should_open_normally() throws Exception { String dbPath = TestUtils.createTempFile(); + LimboDB.load(); LimboDB db = LimboDB.create("jdbc:sqlite" + dbPath, dbPath); - db.load(); db.open(0); } @Test void should_throw_exception_when_opened_twice() throws Exception { String dbPath = TestUtils.createTempFile(); + LimboDB.load(); LimboDB db = LimboDB.create("jdbc:sqlite:" + dbPath, dbPath); - db.load(); db.open(0); assertThatThrownBy(() -> db.open(0)).isInstanceOf(SQLException.class); @@ -33,8 +33,8 @@ public class LimboDBTest { @Test void throwJavaException_should_throw_appropriate_java_exception() throws Exception { String dbPath = TestUtils.createTempFile(); + LimboDB.load(); LimboDB db = LimboDB.create("jdbc:sqlite:" + dbPath, dbPath); - db.load(); final int limboExceptionCode = LimboErrorCode.ETC.code; try { From 12bcfc399b7cc9e3d5f4d84f2527aff056bdb64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 19:55:54 +0900 Subject: [PATCH 05/97] Add LimboConnection.java --- .../github/tursodatabase/LimboConnection.java | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java new file mode 100644 index 000000000..5bb5e973f --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java @@ -0,0 +1,64 @@ +package org.github.tursodatabase; + +import org.github.tursodatabase.core.AbstractDB; +import org.github.tursodatabase.core.LimboDB; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Properties; + +public abstract class LimboConnection implements Connection { + + private final AbstractDB database; + + public LimboConnection(AbstractDB database) { + this.database = database; + } + + public LimboConnection(String url, String fileName) throws SQLException { + this(url, fileName, new Properties()); + } + + /** + * Creates a connection to limbo database. + * + * @param url e.g. "jdbc:sqlite:fileName" + * @param fileName path to file + */ + public LimboConnection(String url, String fileName, Properties properties) throws SQLException { + AbstractDB db = null; + + try { + db = open(url, fileName, properties); + } catch (Throwable t) { + try { + if (db != null) { + db.close(); + } + } catch (Throwable t2) { + t.addSuppressed(t2); + } + + throw t; + } + + this.database = db; + } + + private static AbstractDB open(String url, String fileName, Properties properties) throws SQLException { + if (fileName.isBlank()) { + throw new IllegalArgumentException("fileName should not be empty"); + } + + final AbstractDB database; + try { + LimboDB.load(); + database = LimboDB.create(url, fileName); + } catch (Exception e) { + throw new SQLException("Error opening connection", e); + } + + database.open(0); + return database; + } +} From d88204252f545411a13c2a302162a2e4ad2dbfa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 19:57:48 +0900 Subject: [PATCH 06/97] Add JDBC4Connection.java --- .../tursodatabase/jdbc4/JDBC4Connection.java | 285 ++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java new file mode 100644 index 000000000..538e1bfb8 --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -0,0 +1,285 @@ +package org.github.tursodatabase.jdbc4; + +import org.github.tursodatabase.LimboConnection; + +import java.sql.*; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.Executor; + +public class JDBC4Connection extends LimboConnection { + + public JDBC4Connection(String url, String fileName, Properties properties) throws SQLException { + super(url, fileName, properties); + } + + @Override + public Statement createStatement() throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql) throws SQLException { + return null; + } + + @Override + public CallableStatement prepareCall(String sql) throws SQLException { + return null; + } + + @Override + public String nativeSQL(String sql) throws SQLException { + return ""; + } + + @Override + public void setAutoCommit(boolean autoCommit) throws SQLException { + + } + + @Override + public boolean getAutoCommit() throws SQLException { + return false; + } + + @Override + public void commit() throws SQLException { + + } + + @Override + public void rollback() throws SQLException { + + } + + @Override + public void close() throws SQLException { + + } + + @Override + public boolean isClosed() throws SQLException { + return false; + } + + @Override + public DatabaseMetaData getMetaData() throws SQLException { + return null; + } + + @Override + public void setReadOnly(boolean readOnly) throws SQLException { + + } + + @Override + public boolean isReadOnly() throws SQLException { + return false; + } + + @Override + public void setCatalog(String catalog) throws SQLException { + + } + + @Override + public String getCatalog() throws SQLException { + return ""; + } + + @Override + public void setTransactionIsolation(int level) throws SQLException { + + } + + @Override + public int getTransactionIsolation() throws SQLException { + return 0; + } + + @Override + public SQLWarning getWarnings() throws SQLException { + return null; + } + + @Override + public void clearWarnings() throws SQLException { + + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + return null; + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + return null; + } + + @Override + public Map> getTypeMap() throws SQLException { + return Map.of(); + } + + @Override + public void setTypeMap(Map> map) throws SQLException { + + } + + @Override + public void setHoldability(int holdability) throws SQLException { + + } + + @Override + public int getHoldability() throws SQLException { + return 0; + } + + @Override + public Savepoint setSavepoint() throws SQLException { + return null; + } + + @Override + public Savepoint setSavepoint(String name) throws SQLException { + return null; + } + + @Override + public void rollback(Savepoint savepoint) throws SQLException { + + } + + @Override + public void releaseSavepoint(Savepoint savepoint) throws SQLException { + + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + return null; + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + return null; + } + + @Override + public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + return null; + } + + @Override + public Clob createClob() throws SQLException { + return null; + } + + @Override + public Blob createBlob() throws SQLException { + return null; + } + + @Override + public NClob createNClob() throws SQLException { + return null; + } + + @Override + public SQLXML createSQLXML() throws SQLException { + return null; + } + + @Override + public boolean isValid(int timeout) throws SQLException { + return false; + } + + @Override + public void setClientInfo(String name, String value) throws SQLClientInfoException { + + } + + @Override + public void setClientInfo(Properties properties) throws SQLClientInfoException { + + } + + @Override + public String getClientInfo(String name) throws SQLException { + return ""; + } + + @Override + public Properties getClientInfo() throws SQLException { + return null; + } + + @Override + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + return null; + } + + @Override + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + return null; + } + + @Override + public void setSchema(String schema) throws SQLException { + + } + + @Override + public String getSchema() throws SQLException { + return ""; + } + + @Override + public void abort(Executor executor) throws SQLException { + + } + + @Override + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { + + } + + @Override + public int getNetworkTimeout() throws SQLException { + return 0; + } + + @Override + public T unwrap(Class iface) throws SQLException { + return null; + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return false; + } +} From e8e09cc745edd852359460254b3dc50f88841c3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 20:03:32 +0900 Subject: [PATCH 07/97] Add JDBC.java --- .../java/org/github/tursodatabase/JDBC.java | 62 +++++++++++++++++++ .../org/github/tursodatabase/JDBCTest.java | 23 +++++++ 2 files changed, 85 insertions(+) create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/JDBC.java create mode 100644 bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java new file mode 100644 index 000000000..edf938e67 --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java @@ -0,0 +1,62 @@ +package org.github.tursodatabase; + +import org.github.tursodatabase.jdbc4.JDBC4Connection; + +import java.sql.*; +import java.util.Properties; +import java.util.logging.Logger; + +public class JDBC implements Driver { + + private static final String VALID_URL_PREFIX = "jdbc:limbo:"; + + public static LimboConnection createConnection(String url, Properties properties) throws SQLException { + if (!isValidURL(url)) return null; + + url = url.trim(); + return new JDBC4Connection(url, extractAddress(url), properties); + } + + private static boolean isValidURL(String url) { + return url != null && url.toLowerCase().startsWith(VALID_URL_PREFIX); + } + + private static String extractAddress(String url) { + return url.substring(VALID_URL_PREFIX.length()); + } + + @Override + public Connection connect(String url, Properties info) throws SQLException { + return null; + } + + @Override + public boolean acceptsURL(String url) throws SQLException { + return false; + } + + @Override + public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { + return new DriverPropertyInfo[0]; + } + + @Override + public int getMajorVersion() { + return 0; + } + + @Override + public int getMinorVersion() { + return 0; + } + + @Override + public boolean jdbcCompliant() { + return false; + } + + @Override + public Logger getParentLogger() throws SQLFeatureNotSupportedException { + return null; + } +} diff --git a/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java b/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java new file mode 100644 index 000000000..137e5b0ce --- /dev/null +++ b/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java @@ -0,0 +1,23 @@ +package org.github.tursodatabase; + +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class JDBCTest { + + @Test + void null_is_returned_when_invalid_url_is_passed() throws Exception { + LimboConnection connection = JDBC.createConnection("jdbc:invalid:xxx", new Properties()); + assertThat(connection).isNull(); + } + + @Test + void non_null_connection_is_returned_when_valid_url_is_passed() throws Exception { + String fileUrl = TestUtils.createTempFile(); + LimboConnection connection = JDBC.createConnection("jdbc:limbo:" + fileUrl, new Properties()); + assertThat(connection).isNotNull(); + } +} From bc9e9714f10b2ac81464cf2ecc7be4d8bbe053cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 20:10:38 +0900 Subject: [PATCH 08/97] Add LimboConfig.java --- .../org/github/tursodatabase/LimboConfig.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/LimboConfig.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboConfig.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboConfig.java new file mode 100644 index 000000000..388419e60 --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboConfig.java @@ -0,0 +1,20 @@ +package org.github.tursodatabase; + +import java.util.Properties; + +/** + * Limbo Configuration. + */ +public class LimboConfig { + private final Properties pragma; + + public LimboConfig(Properties properties) { + this.pragma = properties; + } + + public Properties toProperties() { + Properties copy = new Properties(); + copy.putAll(pragma); + return copy; + } +} From b360f0559f421ee0d2458dc14440490120c76c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 20:10:49 +0900 Subject: [PATCH 09/97] Add LimboDataSource.java --- .../github/tursodatabase/LimboDataSource.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java new file mode 100644 index 000000000..9748c219d --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java @@ -0,0 +1,76 @@ +package org.github.tursodatabase; + +import javax.sql.DataSource; +import java.io.PrintWriter; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.Properties; +import java.util.logging.Logger; + +/** + * Provides {@link DataSource} API for configuring Limbo database connection. + */ +public class LimboDataSource implements DataSource { + + private final LimboConfig limboConfig; + private final String url; + + /** + * Creates a datasource based on the provided configuration. + * + * @param limboConfig The configuration for the datasource. + */ + public LimboDataSource(LimboConfig limboConfig, String url) { + this.limboConfig = limboConfig; + this.url = url; + } + + @Override + public Connection getConnection() throws SQLException { + return getConnection(null, null); + } + + @Override + public Connection getConnection(String username, String password) throws SQLException { + Properties properties = limboConfig.toProperties(); + if (username != null) properties.put("user", username); + if (password != null) properties.put("pass", password); + return JDBC.createConnection(url, properties); + } + + @Override + public PrintWriter getLogWriter() throws SQLException { + return null; + } + + @Override + public void setLogWriter(PrintWriter out) throws SQLException { + + } + + @Override + public void setLoginTimeout(int seconds) throws SQLException { + + } + + @Override + public int getLoginTimeout() throws SQLException { + return 0; + } + + @Override + public Logger getParentLogger() throws SQLFeatureNotSupportedException { + return null; + } + + @Override + public T unwrap(Class iface) throws SQLException { + return null; + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return false; + } +} From da787edd99ced4498d26dcb59dbbc6c197e69bb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 20:22:58 +0900 Subject: [PATCH 10/97] Implement JDBC so that DriverManager can detect limbo connection --- .../java/org/github/tursodatabase/JDBC.java | 18 ++++++++--- .../org/github/tursodatabase/LimboConfig.java | 31 +++++++++++++++++++ .../org/github/tursodatabase/JDBCTest.java | 10 ++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java index edf938e67..87200ff5c 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java @@ -7,9 +7,16 @@ import java.util.Properties; import java.util.logging.Logger; public class JDBC implements Driver { - private static final String VALID_URL_PREFIX = "jdbc:limbo:"; + static { + try { + DriverManager.registerDriver(new JDBC()); + } catch (Exception e) { + // TODO: log + } + } + public static LimboConnection createConnection(String url, Properties properties) throws SQLException { if (!isValidURL(url)) return null; @@ -27,26 +34,28 @@ public class JDBC implements Driver { @Override public Connection connect(String url, Properties info) throws SQLException { - return null; + return createConnection(url, info); } @Override public boolean acceptsURL(String url) throws SQLException { - return false; + return isValidURL(url); } @Override public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException { - return new DriverPropertyInfo[0]; + return LimboConfig.getDriverPropertyInfo(); } @Override public int getMajorVersion() { + // TODO return 0; } @Override public int getMinorVersion() { + // TODO return 0; } @@ -57,6 +66,7 @@ public class JDBC implements Driver { @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + // TODO return null; } } diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboConfig.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboConfig.java index 388419e60..7f2a2cdf0 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/LimboConfig.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboConfig.java @@ -1,5 +1,7 @@ package org.github.tursodatabase; +import java.sql.DriverPropertyInfo; +import java.util.Arrays; import java.util.Properties; /** @@ -12,9 +14,38 @@ public class LimboConfig { this.pragma = properties; } + public static DriverPropertyInfo[] getDriverPropertyInfo() { + return Arrays.stream(Pragma.values()) + .map(p -> { + DriverPropertyInfo info = new DriverPropertyInfo(p.pragmaName, null); + info.description = p.description; + info.choices = p.choices; + info.required = false; + return info; + }) + .toArray(DriverPropertyInfo[]::new); + } + public Properties toProperties() { Properties copy = new Properties(); copy.putAll(pragma); return copy; } + + public enum Pragma { + ; + private final String pragmaName; + private final String description; + private final String[] choices; + + Pragma(String pragmaName, String description, String[] choices) { + this.pragmaName = pragmaName; + this.description = description; + this.choices = choices; + } + + public String getPragmaName() { + return pragmaName; + } + } } diff --git a/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java b/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java index 137e5b0ce..ff629bacc 100644 --- a/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java +++ b/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java @@ -2,6 +2,9 @@ package org.github.tursodatabase; import org.junit.jupiter.api.Test; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; import java.util.Properties; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; @@ -20,4 +23,11 @@ class JDBCTest { LimboConnection connection = JDBC.createConnection("jdbc:limbo:" + fileUrl, new Properties()); assertThat(connection).isNotNull(); } + + @Test + void connection_can_be_retrieved_from_DriverManager() throws SQLException { + JDBC jdbc = new JDBC(); + Connection connection = DriverManager.getConnection("jdbc:limbo:sample.db"); + assertThat(connection).isNotNull(); + } } From 71c2bdf37b14626032dbf331d3e89e2d0f84a00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 20:26:41 +0900 Subject: [PATCH 11/97] Add TODO comments --- .../github/tursodatabase/LimboDataSource.java | 9 ++- .../tursodatabase/jdbc4/JDBC4Connection.java | 70 ++++++++++++++----- 2 files changed, 60 insertions(+), 19 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java index 9748c219d..12a53c303 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java @@ -41,36 +41,41 @@ public class LimboDataSource implements DataSource { @Override public PrintWriter getLogWriter() throws SQLException { + // TODO return null; } @Override public void setLogWriter(PrintWriter out) throws SQLException { - + // TODO } @Override public void setLoginTimeout(int seconds) throws SQLException { - + // TODO } @Override public int getLoginTimeout() throws SQLException { + // TODO return 0; } @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + // TODO return null; } @Override public T unwrap(Class iface) throws SQLException { + // TODO return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { + // TODO return false; } } diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index 538e1bfb8..04c83b6b9 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -15,127 +15,142 @@ public class JDBC4Connection extends LimboConnection { @Override public Statement createStatement() throws SQLException { + // TODO return null; } @Override public PreparedStatement prepareStatement(String sql) throws SQLException { + // TODO return null; } @Override public CallableStatement prepareCall(String sql) throws SQLException { + // TODO return null; } @Override public String nativeSQL(String sql) throws SQLException { + // TODO return ""; } @Override public void setAutoCommit(boolean autoCommit) throws SQLException { - + // TODO } @Override public boolean getAutoCommit() throws SQLException { + // TODO return false; } @Override public void commit() throws SQLException { - + // TODO } @Override public void rollback() throws SQLException { - + // TODO } @Override public void close() throws SQLException { - + // TODO } @Override public boolean isClosed() throws SQLException { + // TODO return false; } @Override public DatabaseMetaData getMetaData() throws SQLException { + // TODO return null; } @Override public void setReadOnly(boolean readOnly) throws SQLException { - + // TODO } @Override public boolean isReadOnly() throws SQLException { + // TODO return false; } @Override public void setCatalog(String catalog) throws SQLException { - + // TODO } @Override public String getCatalog() throws SQLException { + // TODO return ""; } @Override public void setTransactionIsolation(int level) throws SQLException { - + // TODO } @Override public int getTransactionIsolation() throws SQLException { + // TODO return 0; } @Override public SQLWarning getWarnings() throws SQLException { + // TODO return null; } @Override public void clearWarnings() throws SQLException { - + // TODO } @Override public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + // TODO return null; } @Override public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + // TODO return null; } @Override public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + // TODO return null; } @Override public Map> getTypeMap() throws SQLException { + // TODO return Map.of(); } @Override public void setTypeMap(Map> map) throws SQLException { - + // TODO } @Override public void setHoldability(int holdability) throws SQLException { - + // TODO } @Override @@ -145,141 +160,162 @@ public class JDBC4Connection extends LimboConnection { @Override public Savepoint setSavepoint() throws SQLException { + // TODO return null; } @Override public Savepoint setSavepoint(String name) throws SQLException { + // TODO return null; } @Override public void rollback(Savepoint savepoint) throws SQLException { - + // TODO } @Override public void releaseSavepoint(Savepoint savepoint) throws SQLException { - + // TODO } @Override public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + // TODO return null; } @Override public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + // TODO return null; } @Override public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + // TODO return null; } @Override public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + // TODO return null; } @Override public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + // TODO return null; } @Override public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + // TODO return null; } @Override public Clob createClob() throws SQLException { + // TODO return null; } @Override public Blob createBlob() throws SQLException { + // TODO return null; } @Override public NClob createNClob() throws SQLException { + // TODO return null; } @Override public SQLXML createSQLXML() throws SQLException { + // TODO return null; } @Override public boolean isValid(int timeout) throws SQLException { + // TODO return false; } @Override public void setClientInfo(String name, String value) throws SQLClientInfoException { - + // TODO } @Override public void setClientInfo(Properties properties) throws SQLClientInfoException { - + // TODO } @Override public String getClientInfo(String name) throws SQLException { + // TODO return ""; } @Override public Properties getClientInfo() throws SQLException { + // TODO return null; } @Override public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + // TODO return null; } @Override public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + // TODO return null; } @Override public void setSchema(String schema) throws SQLException { - + // TODO } @Override public String getSchema() throws SQLException { + // TODO return ""; } @Override public void abort(Executor executor) throws SQLException { - + // TODO } @Override public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { - + // TODO } @Override public int getNetworkTimeout() throws SQLException { + // TODO return 0; } @Override public T unwrap(Class iface) throws SQLException { + // TODO return null; } @Override public boolean isWrapperFor(Class iface) throws SQLException { + // TODO return false; } } From 59180599fd1179664cb9c6e4a046aa3ee1276498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Fri, 10 Jan 2025 20:41:49 +0900 Subject: [PATCH 12/97] Add java.sql.Driver to automatically detect JDBC --- .../src/main/resources/META-INF/services/java.sql.Driver | 1 + .../src/test/java/org/github/tursodatabase/JDBCTest.java | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 bindings/java/src/main/resources/META-INF/services/java.sql.Driver diff --git a/bindings/java/src/main/resources/META-INF/services/java.sql.Driver b/bindings/java/src/main/resources/META-INF/services/java.sql.Driver new file mode 100644 index 000000000..71922046d --- /dev/null +++ b/bindings/java/src/main/resources/META-INF/services/java.sql.Driver @@ -0,0 +1 @@ +org.github.tursodatabase.JDBC diff --git a/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java b/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java index ff629bacc..d0cdc4dc3 100644 --- a/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java +++ b/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java @@ -26,8 +26,8 @@ class JDBCTest { @Test void connection_can_be_retrieved_from_DriverManager() throws SQLException { - JDBC jdbc = new JDBC(); - Connection connection = DriverManager.getConnection("jdbc:limbo:sample.db"); - assertThat(connection).isNotNull(); + try (Connection connection = DriverManager.getConnection("jdbc:limbo:sample.db")) { + assertThat(connection).isNotNull(); + } } } From 191b586f05737b4b26b71f706255c6f7beac8d7a Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sat, 11 Jan 2025 02:20:22 +0300 Subject: [PATCH 13/97] this commit restructures the interaction generation in order to have better counterexample minimization. - it separates interaction plans from their state of execution - it removes closures from the property definitions, encoding properties as an enum variant, and deriving the closures from the variants. - it adds some naive counterexample minimization capabilities to the Limbo simulator and reduces the plan sizes considerably. - it makes small changes to various points of the simulator for better error reporting, enhancing code readability, small fixes to handle previously missed cases --- simulator/generation/mod.rs | 3 +- simulator/generation/plan.rs | 473 ++++++++++++++--------------- simulator/generation/property.rs | 234 +++++++++++++++ simulator/generation/query.rs | 22 +- simulator/main.rs | 500 +++++++++++++++---------------- simulator/model/query.rs | 20 +- simulator/runner/cli.rs | 6 + simulator/runner/execution.rs | 202 +++++++++++++ simulator/runner/mod.rs | 1 + simulator/shrink/mod.rs | 1 + simulator/shrink/plan.rs | 28 ++ 11 files changed, 984 insertions(+), 506 deletions(-) create mode 100644 simulator/generation/property.rs create mode 100644 simulator/runner/execution.rs create mode 100644 simulator/shrink/mod.rs create mode 100644 simulator/shrink/plan.rs diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index 8158b2d17..6107124f0 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -4,6 +4,7 @@ use anarchist_readable_name_generator_lib::readable_name_custom; use rand::{distributions::uniform::SampleUniform, Rng}; pub mod plan; +pub mod property; pub mod query; pub mod table; @@ -21,7 +22,7 @@ pub(crate) fn frequency< R: rand::Rng, N: Sum + PartialOrd + Copy + Default + SampleUniform + SubAssign, >( - choices: Vec<(N, Box T + 'a>)>, + choices: Vec<(N, Box T + 'a>)>, rng: &mut R, ) -> T { let total = choices.iter().map(|(weight, _)| *weight).sum::(); diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 0a18d3821..e286bb34a 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -1,10 +1,10 @@ -use std::{fmt::Display, rc::Rc}; +use std::{fmt::Display, rc::Rc, vec}; use limbo_core::{Connection, Result, StepResult}; use crate::{ model::{ - query::{Create, Insert, Predicate, Query, Select}, + query::{Create, Insert, Query, Select}, table::Value, }, SimConnection, SimulatorEnv, @@ -12,61 +12,115 @@ use crate::{ use crate::generation::{frequency, Arbitrary, ArbitraryFrom}; -use super::{pick, pick_index}; +use super::{pick, property::Property}; pub(crate) type ResultSet = Result>>; +#[derive(Clone)] pub(crate) struct InteractionPlan { - pub(crate) plan: Vec, + pub(crate) plan: Vec, +} + +pub(crate) struct InteractionPlanState { pub(crate) stack: Vec, pub(crate) interaction_pointer: usize, pub(crate) secondary_pointer: usize, } -pub(crate) struct Property { - pub(crate) name: Option, - pub(crate) interactions: Vec, +#[derive(Clone)] +pub(crate) enum Interactions { + Property(Property), + Query(Query), + Fault(Fault), } -impl Property { - pub(crate) fn new(name: Option, interactions: Vec) -> Self { - Self { name, interactions } +impl Interactions { + pub(crate) fn name(&self) -> Option { + match self { + Interactions::Property(property) => Some(property.name()), + Interactions::Query(_) => None, + Interactions::Fault(_) => None, + } } - pub(crate) fn anonymous(interactions: Vec) -> Self { - Self { - name: None, - interactions, + pub(crate) fn interactions(&self) -> Vec { + match self { + Interactions::Property(property) => property.interactions(), + Interactions::Query(query) => vec![Interaction::Query(query.clone())], + Interactions::Fault(fault) => vec![Interaction::Fault(fault.clone())], + } + } +} + +impl Interactions { + pub(crate) fn dependencies(&self) -> Vec { + match self { + Interactions::Property(property) => { + property + .interactions() + .iter() + .fold(vec![], |mut acc, i| match i { + Interaction::Query(q) => { + acc.extend(q.dependencies()); + acc + } + _ => acc, + }) + } + Interactions::Query(query) => query.dependencies(), + Interactions::Fault(_) => vec![], + } + } + + pub(crate) fn uses(&self) -> Vec { + match self { + Interactions::Property(property) => { + property + .interactions() + .iter() + .fold(vec![], |mut acc, i| match i { + Interaction::Query(q) => { + acc.extend(q.uses()); + acc + } + _ => acc, + }) + } + Interactions::Query(query) => query.uses(), + Interactions::Fault(_) => vec![], } } } impl Display for InteractionPlan { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - for property in &self.plan { - if let Some(name) = &property.name { - writeln!(f, "-- begin testing '{}'", name)?; - } + for interactions in &self.plan { + match interactions { + Interactions::Property(property) => { + let name = property.name(); + writeln!(f, "-- begin testing '{}'", name)?; + for interaction in property.interactions() { + write!(f, "\t")?; - for interaction in &property.interactions { - if property.name.is_some() { - write!(f, "\t")?; - } - - match interaction { - Interaction::Query(query) => writeln!(f, "{};", query)?, - Interaction::Assumption(assumption) => { - writeln!(f, "-- ASSUME: {};", assumption.message)? + match interaction { + Interaction::Query(query) => writeln!(f, "{};", query)?, + Interaction::Assumption(assumption) => { + writeln!(f, "-- ASSUME: {};", assumption.message)? + } + Interaction::Assertion(assertion) => { + writeln!(f, "-- ASSERT: {};", assertion.message)? + } + Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + } } - Interaction::Assertion(assertion) => { - writeln!(f, "-- ASSERT: {};", assertion.message)? - } - Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?, + writeln!(f, "-- end testing '{}'", name)?; + } + Interactions::Fault(fault) => { + writeln!(f, "-- FAULT '{}'", fault)?; + } + Interactions::Query(query) => { + writeln!(f, "{};", query)?; } - } - - if let Some(name) = &property.name { - writeln!(f, "-- end testing '{}'", name)?; } } @@ -74,7 +128,7 @@ impl Display for InteractionPlan { } } -#[derive(Debug)] +#[derive(Debug, Clone, Copy)] pub(crate) struct InteractionStats { pub(crate) read_count: usize, pub(crate) write_count: usize, @@ -112,11 +166,16 @@ impl Display for Interaction { type AssertionFunc = dyn Fn(&Vec, &SimulatorEnv) -> bool; +enum AssertionAST { + Pick(), +} + pub(crate) struct Assertion { pub(crate) func: Box, pub(crate) message: String, } +#[derive(Debug, Clone)] pub(crate) enum Fault { Disconnect, } @@ -129,43 +188,60 @@ impl Display for Fault { } } -impl Property { +impl Interactions { pub(crate) fn shadow(&self, env: &mut SimulatorEnv) { - for interaction in &self.interactions { - match interaction { - Interaction::Query(query) => match query { - Query::Create(create) => { - if !env.tables.iter().any(|t| t.name == create.table.name) { - env.tables.push(create.table.clone()); - } + match self { + Interactions::Property(property) => { + for interaction in property.interactions() { + match interaction { + Interaction::Query(query) => match query { + Query::Create(create) => { + if !env.tables.iter().any(|t| t.name == create.table.name) { + env.tables.push(create.table.clone()); + } + } + Query::Insert(insert) => { + let table = env + .tables + .iter_mut() + .find(|t| t.name == insert.table) + .unwrap(); + table.rows.extend(insert.values.clone()); + } + Query::Delete(_) => todo!(), + Query::Select(_) => {} + }, + Interaction::Assertion(_) => {} + Interaction::Assumption(_) => {} + Interaction::Fault(_) => {} } - Query::Insert(insert) => { - let table = env - .tables - .iter_mut() - .find(|t| t.name == insert.table) - .unwrap(); - table.rows.extend(insert.values.clone()); - } - Query::Delete(_) => todo!(), - Query::Select(_) => {} - }, - Interaction::Assertion(_) => {} - Interaction::Assumption(_) => {} - Interaction::Fault(_) => {} + } } + Interactions::Query(query) => match query { + Query::Create(create) => { + if !env.tables.iter().any(|t| t.name == create.table.name) { + env.tables.push(create.table.clone()); + } + } + Query::Insert(insert) => { + let table = env + .tables + .iter_mut() + .find(|t| t.name == insert.table) + .unwrap(); + table.rows.extend(insert.values.clone()); + } + Query::Delete(_) => todo!(), + Query::Select(_) => {} + }, + Interactions::Fault(_) => {} } } } impl InteractionPlan { pub(crate) fn new() -> Self { - Self { - plan: Vec::new(), - stack: Vec::new(), - interaction_pointer: 0, - secondary_pointer: 0, - } + Self { plan: Vec::new() } } pub(crate) fn stats(&self) -> InteractionStats { @@ -174,19 +250,27 @@ impl InteractionPlan { let mut delete = 0; let mut create = 0; - for property in &self.plan { - for interaction in &property.interactions { - match interaction { - Interaction::Query(query) => match query { - Query::Select(_) => read += 1, - Query::Insert(_) => write += 1, - Query::Delete(_) => delete += 1, - Query::Create(_) => create += 1, - }, - Interaction::Assertion(_) => {} - Interaction::Assumption(_) => {} - Interaction::Fault(_) => {} + for interactions in &self.plan { + match interactions { + Interactions::Property(property) => { + for interaction in &property.interactions() { + if let Interaction::Query(query) = interaction { + match query { + Query::Select(_) => read += 1, + Query::Insert(_) => write += 1, + Query::Delete(_) => delete += 1, + Query::Create(_) => create += 1, + } + } + } } + Interactions::Query(query) => match query { + Query::Select(_) => read += 1, + Query::Insert(_) => write += 1, + Query::Delete(_) => delete += 1, + Query::Create(_) => create += 1, + }, + Interactions::Fault(_) => {} } } @@ -211,10 +295,8 @@ impl InteractionPlan { let create_query = Create::arbitrary(rng); env.tables.push(create_query.table.clone()); - plan.plan.push(Property { - name: Some("initial table creation".to_string()), - interactions: vec![Interaction::Query(Query::Create(create_query))], - }); + plan.plan + .push(Interactions::Query(Query::Create(create_query))); while plan.plan.len() < num_interactions { log::debug!( @@ -222,10 +304,10 @@ impl InteractionPlan { plan.plan.len(), num_interactions ); - let property = Property::arbitrary_from(rng, &(env, plan.stats())); - property.shadow(env); + let interactions = Interactions::arbitrary_from(rng, &(env, plan.stats())); + interactions.shadow(env); - plan.plan.push(property); + plan.plan.push(interactions); } log::info!("Generated plan with {} interactions", plan.plan.len()); @@ -235,60 +317,51 @@ impl InteractionPlan { impl Interaction { pub(crate) fn execute_query(&self, conn: &mut Rc) -> ResultSet { - match self { - Self::Query(query) => { - let query_str = query.to_string(); - let rows = conn.query(&query_str); - if rows.is_err() { - let err = rows.err(); - log::error!( - "Error running query '{}': {:?}", - &query_str[0..query_str.len().min(4096)], - err - ); - return Err(err.unwrap()); - } - let rows = rows.unwrap(); - assert!(rows.is_some()); - let mut rows = rows.unwrap(); - let mut out = Vec::new(); - while let Ok(row) = rows.next_row() { - match row { - StepResult::Row(row) => { - let mut r = Vec::new(); - for el in &row.values { - let v = match el { - limbo_core::Value::Null => Value::Null, - limbo_core::Value::Integer(i) => Value::Integer(*i), - limbo_core::Value::Float(f) => Value::Float(*f), - limbo_core::Value::Text(t) => Value::Text(t.to_string()), - limbo_core::Value::Blob(b) => Value::Blob(b.to_vec()), - }; - r.push(v); - } + if let Self::Query(query) = self { + let query_str = query.to_string(); + let rows = conn.query(&query_str); + if rows.is_err() { + let err = rows.err(); + log::error!( + "Error running query '{}': {:?}", + &query_str[0..query_str.len().min(4096)], + err + ); + return Err(err.unwrap()); + } + let rows = rows.unwrap(); + assert!(rows.is_some()); + let mut rows = rows.unwrap(); + let mut out = Vec::new(); + while let Ok(row) = rows.next_row() { + match row { + StepResult::Row(row) => { + let mut r = Vec::new(); + for el in &row.values { + let v = match el { + limbo_core::Value::Null => Value::Null, + limbo_core::Value::Integer(i) => Value::Integer(*i), + limbo_core::Value::Float(f) => Value::Float(*f), + limbo_core::Value::Text(t) => Value::Text(t.to_string()), + limbo_core::Value::Blob(b) => Value::Blob(b.to_vec()), + }; + r.push(v); + } - out.push(r); - } - StepResult::IO => {} - StepResult::Interrupt => {} - StepResult::Done => { - break; - } - StepResult::Busy => {} + out.push(r); } + StepResult::IO => {} + StepResult::Interrupt => {} + StepResult::Done => { + break; + } + StepResult::Busy => {} } + } - Ok(out) - } - Self::Assertion(_) => { - unreachable!("unexpected: this function should only be called on queries") - } - Self::Assumption(_) => { - unreachable!("unexpected: this function should only be called on queries") - } - Self::Fault(_) => { - unreachable!("unexpected: this function should only be called on queries") - } + Ok(out) + } else { + unreachable!("unexpected: this function should only be called on queries") } } @@ -377,121 +450,25 @@ impl Interaction { } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { - // Get a random table +fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Interactions { + Interactions::Query(Query::Create(Create::arbitrary(rng))) +} + +fn random_read(rng: &mut R, env: &SimulatorEnv) -> Interactions { + Interactions::Query(Query::Select(Select::arbitrary_from(rng, &env.tables))) +} + +fn random_write(rng: &mut R, env: &SimulatorEnv) -> Interactions { let table = pick(&env.tables, rng); - // Pick a random column - let column_index = pick_index(table.columns.len(), rng); - let column = &table.columns[column_index].clone(); - // Generate a random value of the column type - let value = Value::arbitrary_from(rng, &column.column_type); - // Create a whole new row - let mut row = Vec::new(); - for (i, column) in table.columns.iter().enumerate() { - if i == column_index { - row.push(value.clone()); - } else { - let value = Value::arbitrary_from(rng, &column.column_type); - row.push(value); - } - } - - // Check that the table exists - let assumption = Interaction::Assumption(Assertion { - message: format!("table {} exists", table.name), - func: Box::new({ - let table_name = table.name.clone(); - move |_: &Vec, env: &SimulatorEnv| { - env.tables.iter().any(|t| t.name == table_name) - } - }), - }); - - // Insert the row - let insert_query = Interaction::Query(Query::Insert(Insert { - table: table.name.clone(), - values: vec![row.clone()], - })); - - // Select the row - let select_query = Interaction::Query(Query::Select(Select { - table: table.name.clone(), - predicate: Predicate::Eq(column.name.clone(), value.clone()), - })); - - // Check that the row is there - let assertion = Interaction::Assertion(Assertion { - message: format!( - "row [{:?}] not found in table {} after inserting ({} = {})", - row.iter().map(|v| v.to_string()).collect::>(), - table.name, - column.name, - value, - ), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { - let rows = stack.last().unwrap(); - match rows { - Ok(rows) => rows.iter().any(|r| r == &row), - Err(_) => false, - } - }), - }); - - Property::new( - Some("select contains inserted value".to_string()), - vec![assumption, insert_query, select_query, assertion], - ) + let insert_query = Query::Insert(Insert::arbitrary_from(rng, table)); + Interactions::Query(insert_query) } -fn property_double_create_failure(rng: &mut R, _env: &SimulatorEnv) -> Property { - let create_query = Create::arbitrary(rng); - let table_name = create_query.table.name.clone(); - let cq1 = Interaction::Query(Query::Create(create_query.clone())); - let cq2 = Interaction::Query(Query::Create(create_query.clone())); - - let assertion = Interaction::Assertion(Assertion { - message: - "creating two tables with the name should result in a failure for the second query" - .to_string(), - func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { - let last = stack.last().unwrap(); - match last { - Ok(_) => false, - Err(e) => e - .to_string() - .contains(&format!("Table {table_name} already exists")), - } - }), - }); - - Property::new( - Some("creating the same table twice fails".to_string()), - vec![cq1, cq2, assertion], - ) +fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Interactions { + Interactions::Fault(Fault::Disconnect) } -fn create_table(rng: &mut R, _env: &SimulatorEnv) -> Property { - let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng))); - Property::anonymous(vec![create_query]) -} - -fn random_read(rng: &mut R, env: &SimulatorEnv) -> Property { - let select_query = Interaction::Query(Query::Select(Select::arbitrary_from(rng, &env.tables))); - Property::anonymous(vec![select_query]) -} - -fn random_write(rng: &mut R, env: &SimulatorEnv) -> Property { - let table = pick(&env.tables, rng); - let insert_query = Interaction::Query(Query::Insert(Insert::arbitrary_from(rng, table))); - Property::anonymous(vec![insert_query]) -} - -fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Property { - let fault = Interaction::Fault(Fault::Disconnect); - Property::anonymous(vec![fault]) -} - -impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { +impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { fn arbitrary_from( rng: &mut R, (env, stats): &(&SimulatorEnv, InteractionStats), @@ -510,8 +487,10 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { frequency( vec![ ( - f64::min(remaining_read, remaining_write), - Box::new(|rng: &mut R| property_insert_select(rng, env)), + f64::min(remaining_read, remaining_write) + remaining_create, + Box::new(|rng: &mut R| { + Interactions::Property(Property::arbitrary_from(rng, &(env, stats))) + }), ), ( remaining_read, @@ -526,10 +505,6 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Property { Box::new(|rng: &mut R| create_table(rng, env)), ), (1.0, Box::new(|rng: &mut R| random_fault(rng, env))), - ( - remaining_create / 2.0, - Box::new(|rng: &mut R| property_double_create_failure(rng, env)), - ), ], rng, ) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs new file mode 100644 index 000000000..39956210b --- /dev/null +++ b/simulator/generation/property.rs @@ -0,0 +1,234 @@ +use crate::{ + model::{ + query::{Create, Insert, Predicate, Query, Select}, + table::Value, + }, + runner::env::SimulatorEnv, +}; + +use super::{ + frequency, pick, pick_index, + plan::{Assertion, Interaction, InteractionStats, ResultSet}, + ArbitraryFrom, +}; + +/// Properties are representations of executable specifications +/// about the database behavior. +#[derive(Clone)] +pub(crate) enum Property { + /// Insert-Select is a property in which the inserted row + /// must be in the resulting rows of a select query that has a + /// where clause that matches the inserted row. + /// The execution of the property is as follows + /// INSERT INTO VALUES (...) + /// I_0 + /// I_1 + /// ... + /// I_n + /// SELECT * FROM WHERE + /// The interactions in the middle has the following constraints; + /// - There will be no errors in the middle interactions. + /// - The inserted row will not be deleted. + /// - The inserted row will not be updated. + /// - The table `t` will not be renamed, dropped, or altered. + InsertSelect { + /// The insert query + insert: Insert, + /// Additional interactions in the middle of the property + interactions: Vec, + /// The select query + select: Select, + }, + /// Double Create Failure is a property in which creating + /// the same table twice leads to an error. + /// The execution of the property is as follows + /// CREATE TABLE (...) + /// I_0 + /// I_1 + /// ... + /// I_n + /// CREATE TABLE (...) -> Error + /// The interactions in the middle has the following constraints; + /// - There will be no errors in the middle interactions. + /// - Table `t` will not be renamed or dropped. + DoubleCreateFailure { + /// The create query + create: Create, + /// Additional interactions in the middle of the property + interactions: Vec, + }, +} + +impl Property { + pub(crate) fn name(&self) -> String { + match self { + Property::InsertSelect { .. } => "Insert-Select".to_string(), + Property::DoubleCreateFailure { .. } => "Double-Create-Failure".to_string(), + } + } + pub(crate) fn interactions(&self) -> Vec { + match self { + Property::InsertSelect { + insert, + interactions: _, // todo: add extensional interactions + select, + } => { + // Check that the row is there + let row = insert + .values + .first() // `.first` is safe, because we know we are inserting a row in the insert select property + .expect("insert query should have at least 1 value") + .clone(); + + let assumption = Interaction::Assumption(Assertion { + message: format!("table {} exists", insert.table), + func: Box::new({ + let table_name = insert.table.clone(); + move |_: &Vec, env: &SimulatorEnv| { + env.tables.iter().any(|t| t.name == table_name) + } + }), + }); + + let assertion = Interaction::Assertion(Assertion { + message: format!( + // todo: add the part inserting ({} = {})", + "row [{:?}] not found in table {}", + row.iter().map(|v| v.to_string()).collect::>(), + insert.table, + ), + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + let rows = stack.last().unwrap(); + match rows { + Ok(rows) => rows.iter().any(|r| r == &row), + Err(_) => false, + } + }), + }); + + vec![ + assumption, + Interaction::Query(Query::Insert(insert.clone())), + Interaction::Query(Query::Select(select.clone())), + assertion, + ] + } + Property::DoubleCreateFailure { + create, + interactions: _, // todo: add extensional interactions + } => { + let table_name = create.table.name.clone(); + let cq1 = Interaction::Query(Query::Create(create.clone())); + let cq2 = Interaction::Query(Query::Create(create.clone())); + + let assertion = Interaction::Assertion(Assertion { + message: + "creating two tables with the name should result in a failure for the second query" + .to_string(), + func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { + let last = stack.last().unwrap(); + match last { + Ok(_) => false, + Err(e) => e + .to_string() + .contains(&format!("Table {table_name} already exists")), + } + }), + }); + + vec![cq1, cq2, assertion] + } + } + } +} + +fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> (f64, f64, f64) { + let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) + - (stats.read_count as f64)) + .max(0.0); + let remaining_write = ((env.opts.max_interactions as f64 * env.opts.write_percent / 100.0) + - (stats.write_count as f64)) + .max(0.0); + let remaining_create = ((env.opts.max_interactions as f64 * env.opts.create_percent / 100.0) + - (stats.create_count as f64)) + .max(0.0); + + (remaining_read, remaining_write, remaining_create) +} + +fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { + // Get a random table + let table = pick(&env.tables, rng); + // Pick a random column + let column_index = pick_index(table.columns.len(), rng); + let column = &table.columns[column_index].clone(); + // Generate a random value of the column type + let value = Value::arbitrary_from(rng, &column.column_type); + // Create a whole new row + let mut row = Vec::new(); + for (i, column) in table.columns.iter().enumerate() { + if i == column_index { + row.push(value.clone()); + } else { + let value = Value::arbitrary_from(rng, &column.column_type); + row.push(value); + } + } + + // Insert the row + let insert_query = Insert { + table: table.name.clone(), + values: vec![row.clone()], + }; + + // Select the row + let select_query = Select { + table: table.name.clone(), + predicate: Predicate::arbitrary_from( + rng, + &(table, &Predicate::Eq(column.name.clone(), value.clone())), + ), + }; + + Property::InsertSelect { + insert: insert_query, + interactions: Vec::new(), + select: select_query, + } +} + +fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) -> Property { + // Get a random table + let table = pick(&env.tables, rng); + // Create the table + let create_query = Create { + table: table.clone(), + }; + + Property::DoubleCreateFailure { + create: create_query, + interactions: Vec::new(), + } +} + +impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property { + fn arbitrary_from( + rng: &mut R, + (env, stats): &(&SimulatorEnv, &InteractionStats), + ) -> Self { + let (remaining_read, remaining_write, remaining_create) = remaining(env, stats); + frequency( + vec![ + ( + f64::min(remaining_read, remaining_write), + Box::new(|rng: &mut R| property_insert_select(rng, env)), + ), + ( + remaining_create / 2.0, + Box::new(|rng: &mut R| property_double_create_failure(rng, env)), + ), + ], + rng, + ) + } +} diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index b39ef6785..c99638f6d 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -161,13 +161,13 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { // An AND for false requires at least one of its children to be false if *predicate_value { Predicate::And( - (0..rng.gen_range(1..=3)) + (0..rng.gen_range(0..=3)) .map(|_| SimplePredicate::arbitrary_from(rng, &(*table, true)).0) .collect(), ) } else { // Create a vector of random booleans - let mut booleans = (0..rng.gen_range(1..=3)) + let mut booleans = (0..rng.gen_range(0..=3)) .map(|_| rng.gen_bool(0.5)) .collect::>(); @@ -190,7 +190,7 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { // An OR for false requires each of its children to be false if *predicate_value { // Create a vector of random booleans - let mut booleans = (0..rng.gen_range(1..=3)) + let mut booleans = (0..rng.gen_range(0..=3)) .map(|_| rng.gen_bool(0.5)) .collect::>(); let len = booleans.len(); @@ -207,7 +207,7 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { ) } else { Predicate::Or( - (0..rng.gen_range(1..=3)) + (0..rng.gen_range(0..=3)) .map(|_| SimplePredicate::arbitrary_from(rng, &(*table, false)).0) .collect(), ) @@ -245,3 +245,17 @@ impl ArbitraryFrom<(&str, &Value)> for Predicate { ) } } + +impl ArbitraryFrom<(&Table, &Predicate)> for Predicate { + fn arbitrary_from(rng: &mut R, (t, p): &(&Table, &Predicate)) -> Self { + if rng.gen_bool(0.5) { + // produce a true predicate + let p_t = CompoundPredicate::arbitrary_from(rng, &(*t, true)).0; + Predicate::And(vec![p_t, (*p).clone()]) + } else { + // produce a false predicate + let p_f = CompoundPredicate::arbitrary_from(rng, &(*t, false)).0; + Predicate::Or(vec![p_f, (*p).clone()]) + } + } +} diff --git a/simulator/main.rs b/simulator/main.rs index 5491c5780..a4e99f6ea 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -1,24 +1,25 @@ #![allow(clippy::arc_with_non_send_sync, dead_code)] use clap::Parser; use core::panic; -use generation::pick_index; -use generation::plan::{Interaction, InteractionPlan, ResultSet}; -use limbo_core::{Database, LimboError, Result}; -use model::table::Value; +use generation::plan::{InteractionPlan, InteractionPlanState}; +use limbo_core::Database; use rand::prelude::*; use rand_chacha::ChaCha8Rng; use runner::cli::SimulatorCLI; use runner::env::{SimConnection, SimulatorEnv, SimulatorOpts}; +use runner::execution::{execute_plans, Execution, ExecutionHistory, ExecutionResult}; use runner::io::SimulatorIO; +use std::any::Any; use std::backtrace::Backtrace; use std::io::Write; use std::path::Path; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use tempfile::TempDir; mod generation; mod model; mod runner; +mod shrink; fn main() { let _ = env_logger::try_init(); @@ -36,15 +37,30 @@ fn main() { }; let db_path = output_dir.join("simulator.db"); + let doublecheck_db_path = db_path.with_extension("_doublecheck.db"); + let shrunk_db_path = db_path.with_extension("_shrink.db"); + let plan_path = output_dir.join("simulator.plan"); + let shrunk_plan_path = plan_path.with_extension("_shrunk.plan"); + let history_path = output_dir.join("simulator.history"); // Print the seed, the locations of the database and the plan file log::info!("database path: {:?}", db_path); + if cli_opts.doublecheck { + log::info!("doublecheck database path: {:?}", doublecheck_db_path); + } else if cli_opts.shrink { + log::info!("shrunk database path: {:?}", shrunk_db_path); + } log::info!("simulator plan path: {:?}", plan_path); + if cli_opts.shrink { + log::info!("shrunk plan path: {:?}", shrunk_plan_path); + } log::info!("simulator history path: {:?}", history_path); log::info!("seed: {}", seed); + let last_execution = Arc::new(Mutex::new(Execution::new(0, 0, 0))); + std::panic::set_hook(Box::new(move |info| { log::error!("panic occurred"); @@ -61,110 +77,258 @@ fn main() { log::error!("captured backtrace:\n{}", bt); })); - let result = std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path)); + let result = SandboxedResult::from( + std::panic::catch_unwind(|| { + run_simulation( + seed, + &cli_opts, + &db_path, + &plan_path, + last_execution.clone(), + None, + ) + }), + last_execution.clone(), + ); if cli_opts.doublecheck { - // Move the old database and plan file to a new location - let old_db_path = db_path.with_extension("_old.db"); - let old_plan_path = plan_path.with_extension("_old.plan"); - - std::fs::rename(&db_path, &old_db_path).unwrap(); - std::fs::rename(&plan_path, &old_plan_path).unwrap(); - // Run the simulation again - let result2 = - std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path)); + let result2 = SandboxedResult::from( + std::panic::catch_unwind(|| { + run_simulation( + seed, + &cli_opts, + &doublecheck_db_path, + &plan_path, + last_execution.clone(), + None, + ) + }), + last_execution.clone(), + ); match (result, result2) { - (Ok(ExecutionResult { error: None, .. }), Err(_)) => { + (SandboxedResult::Correct, SandboxedResult::Panicked { .. }) => { log::error!("doublecheck failed! first run succeeded, but second run panicked."); } - (Ok(ExecutionResult { error: Some(_), .. }), Err(_)) => { + (SandboxedResult::FoundBug { .. }, SandboxedResult::Panicked { .. }) => { log::error!( - "doublecheck failed! first run failed assertion, but second run panicked." + "doublecheck failed! first run failed an assertion, but second run panicked." ); } - (Err(_), Ok(ExecutionResult { error: None, .. })) => { + (SandboxedResult::Panicked { .. }, SandboxedResult::Correct) => { log::error!("doublecheck failed! first run panicked, but second run succeeded."); } - (Err(_), Ok(ExecutionResult { error: Some(_), .. })) => { + (SandboxedResult::Panicked { .. }, SandboxedResult::FoundBug { .. }) => { log::error!( - "doublecheck failed! first run panicked, but second run failed assertion." + "doublecheck failed! first run panicked, but second run failed an assertion." ); } - ( - Ok(ExecutionResult { error: None, .. }), - Ok(ExecutionResult { error: Some(_), .. }), - ) => { + (SandboxedResult::Correct, SandboxedResult::FoundBug { .. }) => { log::error!( - "doublecheck failed! first run succeeded, but second run failed assertion." + "doublecheck failed! first run succeeded, but second run failed an assertion." ); } - ( - Ok(ExecutionResult { error: Some(_), .. }), - Ok(ExecutionResult { error: None, .. }), - ) => { + (SandboxedResult::FoundBug { .. }, SandboxedResult::Correct) => { log::error!( - "doublecheck failed! first run failed assertion, but second run succeeded." + "doublecheck failed! first run failed an assertion, but second run succeeded." ); } - (Err(_), Err(_)) | (Ok(_), Ok(_)) => { + (SandboxedResult::Correct, SandboxedResult::Correct) + | (SandboxedResult::FoundBug { .. }, SandboxedResult::FoundBug { .. }) + | (SandboxedResult::Panicked { .. }, SandboxedResult::Panicked { .. }) => { // Compare the two database files byte by byte - let old_db = std::fs::read(&old_db_path).unwrap(); - let new_db = std::fs::read(&db_path).unwrap(); - if old_db != new_db { + let db_bytes = std::fs::read(&db_path).unwrap(); + let doublecheck_db_bytes = std::fs::read(&doublecheck_db_path).unwrap(); + if db_bytes != doublecheck_db_bytes { log::error!("doublecheck failed! database files are different."); } else { log::info!("doublecheck succeeded! database files are the same."); } } } - - // Move the new database and plan file to a new location - let new_db_path = db_path.with_extension("_double.db"); - let new_plan_path = plan_path.with_extension("_double.plan"); - - std::fs::rename(&db_path, &new_db_path).unwrap(); - std::fs::rename(&plan_path, &new_plan_path).unwrap(); - - // Move the old database and plan file back - std::fs::rename(&old_db_path, &db_path).unwrap(); - std::fs::rename(&old_plan_path, &plan_path).unwrap(); - } else if let Ok(result) = result { - // No panic occurred, so write the history to a file - let f = std::fs::File::create(&history_path).unwrap(); - let mut f = std::io::BufWriter::new(f); - for execution in result.history.history.iter() { - writeln!( - f, - "{} {} {}", - execution.connection_index, execution.interaction_index, execution.secondary_index - ) - .unwrap(); - } - - match result.error { - None => { - log::info!("simulation completed successfully"); + } else { + // No doublecheck, run shrinking if panicking or found a bug. + match &result { + SandboxedResult::Correct => { + log::info!("simulation succeeded"); } - Some(e) => { - log::error!("simulation failed: {:?}", e); + SandboxedResult::Panicked { + error, + last_execution, + } + | SandboxedResult::FoundBug { + error, + last_execution, + .. + } => { + if let SandboxedResult::FoundBug { history, .. } = &result { + // No panic occurred, so write the history to a file + let f = std::fs::File::create(&history_path).unwrap(); + let mut f = std::io::BufWriter::new(f); + for execution in history.history.iter() { + writeln!( + f, + "{} {} {}", + execution.connection_index, + execution.interaction_index, + execution.secondary_index + ) + .unwrap(); + } + } + + log::error!("simulation failed: '{}'", error); + + if cli_opts.shrink { + log::info!("Starting to shrink"); + let shrink = Some(last_execution); + let last_execution = Arc::new(Mutex::new(*last_execution)); + + let shrunk = SandboxedResult::from( + std::panic::catch_unwind(|| { + run_simulation( + seed, + &cli_opts, + &shrunk_db_path, + &shrunk_plan_path, + last_execution.clone(), + shrink, + ) + }), + last_execution, + ); + + match (shrunk, &result) { + ( + SandboxedResult::Panicked { error: e1, .. }, + SandboxedResult::Panicked { error: e2, .. }, + ) + | ( + SandboxedResult::FoundBug { error: e1, .. }, + SandboxedResult::FoundBug { error: e2, .. }, + ) => { + if &e1 != e2 { + log::error!( + "shrinking failed, the error was not properly reproduced" + ); + } else { + log::info!("shrinking succeeded"); + } + } + (_, SandboxedResult::Correct) => { + unreachable!("shrinking should never be called on a correct simulation") + } + _ => { + log::error!("shrinking failed, the error was not properly reproduced"); + } + } + + // Write the shrunk plan to a file + let shrunk_plan = std::fs::read(&shrunk_plan_path).unwrap(); + let mut f = std::fs::File::create(&shrunk_plan_path).unwrap(); + f.write_all(&shrunk_plan).unwrap(); + } } } } // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. println!("database path: {:?}", db_path); + if cli_opts.doublecheck { + println!("doublecheck database path: {:?}", doublecheck_db_path); + } else if cli_opts.shrink { + println!("shrunk database path: {:?}", shrunk_db_path); + } println!("simulator plan path: {:?}", plan_path); + if cli_opts.shrink { + println!("shrunk plan path: {:?}", shrunk_plan_path); + } println!("simulator history path: {:?}", history_path); println!("seed: {}", seed); } +fn move_db_and_plan_files(output_dir: &Path) { + let old_db_path = output_dir.join("simulator.db"); + let old_plan_path = output_dir.join("simulator.plan"); + + let new_db_path = output_dir.join("simulator_double.db"); + let new_plan_path = output_dir.join("simulator_double.plan"); + + std::fs::rename(&old_db_path, &new_db_path).unwrap(); + std::fs::rename(&old_plan_path, &new_plan_path).unwrap(); +} + +fn revert_db_and_plan_files(output_dir: &Path) { + let old_db_path = output_dir.join("simulator.db"); + let old_plan_path = output_dir.join("simulator.plan"); + + let new_db_path = output_dir.join("simulator_double.db"); + let new_plan_path = output_dir.join("simulator_double.plan"); + + std::fs::rename(&new_db_path, &old_db_path).unwrap(); + std::fs::rename(&new_plan_path, &old_plan_path).unwrap(); +} + +enum SandboxedResult { + Panicked { + error: String, + last_execution: Execution, + }, + FoundBug { + error: String, + history: ExecutionHistory, + last_execution: Execution, + }, + Correct, +} + +impl SandboxedResult { + fn from( + result: Result>, + last_execution: Arc>, + ) -> Self { + match result { + Ok(ExecutionResult { error: None, .. }) => SandboxedResult::Correct, + Ok(ExecutionResult { error: Some(e), .. }) => { + let error = format!("{:?}", e); + let last_execution = last_execution.lock().unwrap(); + SandboxedResult::Panicked { + error, + last_execution: *last_execution, + } + } + Err(payload) => { + log::error!("panic occurred"); + let err = if let Some(s) = payload.downcast_ref::<&str>() { + log::error!("{}", s); + s.to_string() + } else if let Some(s) = payload.downcast_ref::() { + log::error!("{}", s); + s.to_string() + } else { + log::error!("unknown panic payload"); + "unknown panic payload".to_string() + }; + + last_execution.clear_poison(); + + SandboxedResult::Panicked { + error: err, + last_execution: *last_execution.lock().unwrap(), + } + } + } + } +} + fn run_simulation( seed: u64, cli_opts: &SimulatorCLI, db_path: &Path, plan_path: &Path, + last_execution: Arc>, + shrink: Option<&Execution>, ) -> ExecutionResult { let mut rng = ChaCha8Rng::seed_from_u64(seed); @@ -231,16 +395,34 @@ fn run_simulation( let mut plans = (1..=env.opts.max_connections) .map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &mut env)) .collect::>(); + let mut states = plans + .iter() + .map(|_| InteractionPlanState { + stack: vec![], + interaction_pointer: 0, + secondary_pointer: 0, + }) + .collect::>(); + + let plan = if let Some(failing_execution) = shrink { + // todo: for now, we only use 1 connection, so it's safe to use the first plan. + println!("Interactions Before: {}", plans[0].plan.len()); + let shrunk = plans[0].shrink_interaction_plan(failing_execution); + println!("Interactions After: {}", shrunk.plan.len()); + shrunk + } else { + plans[0].clone() + }; let mut f = std::fs::File::create(plan_path).unwrap(); // todo: create a detailed plan file with all the plans. for now, we only use 1 connection, so it's safe to use the first plan. - f.write_all(plans[0].to_string().as_bytes()).unwrap(); + f.write_all(plan.to_string().as_bytes()).unwrap(); - log::info!("{}", plans[0].stats()); + log::info!("{}", plan.stats()); log::info!("Executing database interaction plan..."); - let result = execute_plans(&mut env, &mut plans); + let result = execute_plans(&mut env, &mut plans, &mut states, last_execution); env.io.print_stats(); @@ -248,187 +430,3 @@ fn run_simulation( result } - -struct Execution { - connection_index: usize, - interaction_index: usize, - secondary_index: usize, -} - -impl Execution { - fn new(connection_index: usize, interaction_index: usize, secondary_index: usize) -> Self { - Self { - connection_index, - interaction_index, - secondary_index, - } - } -} - -struct ExecutionHistory { - history: Vec, -} - -impl ExecutionHistory { - fn new() -> Self { - Self { - history: Vec::new(), - } - } -} - -struct ExecutionResult { - history: ExecutionHistory, - error: Option, -} - -impl ExecutionResult { - fn new(history: ExecutionHistory, error: Option) -> Self { - Self { history, error } - } -} - -fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> ExecutionResult { - let mut history = ExecutionHistory::new(); - let now = std::time::Instant::now(); - // todo: add history here by recording which interaction was executed at which tick - for _tick in 0..env.opts.ticks { - // Pick the connection to interact with - let connection_index = pick_index(env.connections.len(), &mut env.rng); - history.history.push(Execution::new( - connection_index, - plans[connection_index].interaction_pointer, - plans[connection_index].secondary_pointer, - )); - // Execute the interaction for the selected connection - match execute_plan(env, connection_index, plans) { - Ok(_) => {} - Err(err) => { - return ExecutionResult::new(history, Some(err)); - } - } - // Check if the maximum time for the simulation has been reached - if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { - return ExecutionResult::new( - history, - Some(limbo_core::LimboError::InternalError( - "maximum time for simulation reached".into(), - )), - ); - } - } - - ExecutionResult::new(history, None) -} - -fn execute_plan( - env: &mut SimulatorEnv, - connection_index: usize, - plans: &mut [InteractionPlan], -) -> Result<()> { - let connection = &env.connections[connection_index]; - let plan = &mut plans[connection_index]; - - if plan.interaction_pointer >= plan.plan.len() { - return Ok(()); - } - - let interaction = &plan.plan[plan.interaction_pointer].interactions[plan.secondary_pointer]; - - if let SimConnection::Disconnected = connection { - log::info!("connecting {}", connection_index); - env.connections[connection_index] = SimConnection::Connected(env.db.connect()); - } else { - match execute_interaction(env, connection_index, interaction, &mut plan.stack) { - Ok(next_execution) => { - log::debug!("connection {} processed", connection_index); - // Move to the next interaction or property - match next_execution { - ExecutionContinuation::NextInteraction => { - if plan.secondary_pointer + 1 - >= plan.plan[plan.interaction_pointer].interactions.len() - { - // If we have reached the end of the interactions for this property, move to the next property - plan.interaction_pointer += 1; - plan.secondary_pointer = 0; - } else { - // Otherwise, move to the next interaction - plan.secondary_pointer += 1; - } - } - ExecutionContinuation::NextProperty => { - // Skip to the next property - plan.interaction_pointer += 1; - plan.secondary_pointer = 0; - } - } - } - Err(err) => { - log::error!("error {}", err); - return Err(err); - } - } - } - - Ok(()) -} - -/// The next point of control flow after executing an interaction. -/// `execute_interaction` uses this type in conjunction with a result, where -/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case -/// indicates the next step in the plan. -enum ExecutionContinuation { - /// Default continuation, execute the next interaction. - NextInteraction, - /// Typically used in the case of preconditions failures, skip to the next property. - NextProperty, -} - -fn execute_interaction( - env: &mut SimulatorEnv, - connection_index: usize, - interaction: &Interaction, - stack: &mut Vec, -) -> Result { - log::info!("executing: {}", interaction); - match interaction { - generation::plan::Interaction::Query(_) => { - let conn = match &mut env.connections[connection_index] { - SimConnection::Connected(conn) => conn, - SimConnection::Disconnected => unreachable!(), - }; - - log::debug!("{}", interaction); - let results = interaction.execute_query(conn); - log::debug!("{:?}", results); - stack.push(results); - } - generation::plan::Interaction::Assertion(_) => { - interaction.execute_assertion(stack, env)?; - stack.clear(); - } - generation::plan::Interaction::Assumption(_) => { - let assumption_result = interaction.execute_assumption(stack, env); - stack.clear(); - - if assumption_result.is_err() { - log::warn!("assumption failed: {:?}", assumption_result); - return Ok(ExecutionContinuation::NextProperty); - } - } - Interaction::Fault(_) => { - interaction.execute_fault(env, connection_index)?; - } - } - - Ok(ExecutionContinuation::NextInteraction) -} - -fn compare_equal_rows(a: &[Vec], b: &[Vec]) { - assert_eq!(a.len(), b.len(), "lengths are different"); - for (r1, r2) in a.iter().zip(b) { - for (v1, v2) in r1.iter().zip(r2) { - assert_eq!(v1, v2, "values are different"); - } - } -} diff --git a/simulator/model/query.rs b/simulator/model/query.rs index 66297b2ad..40d7b7c89 100644 --- a/simulator/model/query.rs +++ b/simulator/model/query.rs @@ -53,7 +53,7 @@ impl Display for Predicate { } // This type represents the potential queries on the database. -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) enum Query { Create(Create), Select(Select), @@ -61,6 +61,24 @@ pub(crate) enum Query { Delete(Delete), } +impl Query { + pub(crate) fn dependencies(&self) -> Vec { + match self { + Query::Create(_) => vec![], + Query::Select(Select { table, .. }) + | Query::Insert(Insert { table, .. }) + | Query::Delete(Delete { table, .. }) => vec![table.clone()], + } + } + pub(crate) fn uses(&self) -> Vec { + match self { + Query::Create(Create { table }) => vec![table.name.clone()], + Query::Select(Select { table, .. }) + | Query::Insert(Insert { table, .. }) + | Query::Delete(Delete { table, .. }) => vec![table.clone()], + } + } +} #[derive(Debug, Clone)] pub(crate) struct Create { pub(crate) table: Table, diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index 8ad42c8b3..6b56b9b82 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -35,4 +35,10 @@ pub struct SimulatorCLI { default_value_t = 60 * 60 // default to 1 hour )] pub maximum_time: usize, + #[clap( + short = 'm', + long, + help = "minimize(shrink) the failing counterexample" + )] + pub shrink: bool, } diff --git a/simulator/runner/execution.rs b/simulator/runner/execution.rs new file mode 100644 index 000000000..064542805 --- /dev/null +++ b/simulator/runner/execution.rs @@ -0,0 +1,202 @@ +use std::sync::{Arc, Mutex}; + +use limbo_core::{LimboError, Result}; + +use crate::generation::{ + self, pick_index, + plan::{Interaction, InteractionPlan, InteractionPlanState, ResultSet}, +}; + +use super::env::{SimConnection, SimulatorEnv}; + +#[derive(Clone, Copy)] +pub(crate) struct Execution { + pub(crate) connection_index: usize, + pub(crate) interaction_index: usize, + pub(crate) secondary_index: usize, +} + +impl Execution { + pub(crate) fn new( + connection_index: usize, + interaction_index: usize, + secondary_index: usize, + ) -> Self { + Self { + connection_index, + interaction_index, + secondary_index, + } + } +} + +pub(crate) struct ExecutionHistory { + pub(crate) history: Vec, +} + +impl ExecutionHistory { + fn new() -> Self { + Self { + history: Vec::new(), + } + } +} + +pub(crate) struct ExecutionResult { + pub(crate) history: ExecutionHistory, + pub(crate) error: Option, +} + +impl ExecutionResult { + fn new(history: ExecutionHistory, error: Option) -> Self { + Self { history, error } + } +} + +pub(crate) fn execute_plans( + env: &mut SimulatorEnv, + plans: &mut [InteractionPlan], + states: &mut [InteractionPlanState], + last_execution: Arc>, +) -> ExecutionResult { + let mut history = ExecutionHistory::new(); + let now = std::time::Instant::now(); + for _tick in 0..env.opts.ticks { + // Pick the connection to interact with + let connection_index = pick_index(env.connections.len(), &mut env.rng); + let state = &mut states[connection_index]; + + history.history.push(Execution::new( + connection_index, + state.interaction_pointer, + state.secondary_pointer, + )); + let mut last_execution = last_execution.lock().unwrap(); + last_execution.connection_index = connection_index; + last_execution.interaction_index = state.interaction_pointer; + last_execution.secondary_index = state.secondary_pointer; + // Execute the interaction for the selected connection + match execute_plan(env, connection_index, plans, states) { + Ok(_) => {} + Err(err) => { + return ExecutionResult::new(history, Some(err)); + } + } + // Check if the maximum time for the simulation has been reached + if now.elapsed().as_secs() >= env.opts.max_time_simulation as u64 { + return ExecutionResult::new( + history, + Some(limbo_core::LimboError::InternalError( + "maximum time for simulation reached".into(), + )), + ); + } + } + + ExecutionResult::new(history, None) +} + +fn execute_plan( + env: &mut SimulatorEnv, + connection_index: usize, + plans: &mut [InteractionPlan], + states: &mut [InteractionPlanState], +) -> Result<()> { + let connection = &env.connections[connection_index]; + let plan = &mut plans[connection_index]; + let state = &mut states[connection_index]; + + if state.interaction_pointer >= plan.plan.len() { + return Ok(()); + } + + let interaction = &plan.plan[state.interaction_pointer].interactions()[state.secondary_pointer]; + + if let SimConnection::Disconnected = connection { + log::info!("connecting {}", connection_index); + env.connections[connection_index] = SimConnection::Connected(env.db.connect()); + } else { + match execute_interaction(env, connection_index, interaction, &mut state.stack) { + Ok(next_execution) => { + log::debug!("connection {} processed", connection_index); + // Move to the next interaction or property + match next_execution { + ExecutionContinuation::NextInteraction => { + if state.secondary_pointer + 1 + >= plan.plan[state.interaction_pointer].interactions().len() + { + // If we have reached the end of the interactions for this property, move to the next property + state.interaction_pointer += 1; + state.secondary_pointer = 0; + } else { + // Otherwise, move to the next interaction + state.secondary_pointer += 1; + } + } + ExecutionContinuation::NextProperty => { + // Skip to the next property + state.interaction_pointer += 1; + state.secondary_pointer = 0; + } + } + } + Err(err) => { + log::error!("error {}", err); + return Err(err); + } + } + } + + Ok(()) +} + +/// The next point of control flow after executing an interaction. +/// `execute_interaction` uses this type in conjunction with a result, where +/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case +/// indicates the next step in the plan. +enum ExecutionContinuation { + /// Default continuation, execute the next interaction. + NextInteraction, + /// Typically used in the case of preconditions failures, skip to the next property. + NextProperty, +} + +fn execute_interaction( + env: &mut SimulatorEnv, + connection_index: usize, + interaction: &Interaction, + stack: &mut Vec, +) -> Result { + log::info!("executing: {}", interaction); + match interaction { + generation::plan::Interaction::Query(_) => { + let conn = match &mut env.connections[connection_index] { + SimConnection::Connected(conn) => conn, + SimConnection::Disconnected => unreachable!(), + }; + + log::debug!("{}", interaction); + let results = interaction.execute_query(conn); + log::debug!("{:?}", results); + stack.push(results); + } + generation::plan::Interaction::Assertion(_) => { + interaction.execute_assertion(stack, env)?; + stack.clear(); + } + generation::plan::Interaction::Assumption(_) => { + let assumption_result = interaction.execute_assumption(stack, env); + stack.clear(); + + if assumption_result.is_err() { + log::warn!("assumption failed: {:?}", assumption_result); + return Ok(ExecutionContinuation::NextProperty); + } + } + Interaction::Fault(_) => { + interaction.execute_fault(env, connection_index)?; + } + } + + Ok(ExecutionContinuation::NextInteraction) +} diff --git a/simulator/runner/mod.rs b/simulator/runner/mod.rs index 10a777fd9..3f014bef0 100644 --- a/simulator/runner/mod.rs +++ b/simulator/runner/mod.rs @@ -1,5 +1,6 @@ pub mod cli; pub mod env; +pub mod execution; #[allow(dead_code)] pub mod file; pub mod io; diff --git a/simulator/shrink/mod.rs b/simulator/shrink/mod.rs new file mode 100644 index 000000000..7764a5c30 --- /dev/null +++ b/simulator/shrink/mod.rs @@ -0,0 +1 @@ +pub mod plan; diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs new file mode 100644 index 000000000..87eb90248 --- /dev/null +++ b/simulator/shrink/plan.rs @@ -0,0 +1,28 @@ +use crate::{generation::plan::InteractionPlan, runner::execution::Execution}; + +impl InteractionPlan { + /// Create a smaller interaction plan by deleting a property + pub(crate) fn shrink_interaction_plan(&self, failing_execution: &Execution) -> InteractionPlan { + let mut plan = self.clone(); + let failing_property = &self.plan[failing_execution.interaction_index]; + let depending_tables = failing_property.dependencies(); + + let before = self.plan.len(); + + // Remove all properties after the failing one + plan.plan.truncate(failing_execution.interaction_index + 1); + // Remove all properties that do not use the failing tables + plan.plan + .retain(|p| p.uses().iter().any(|t| depending_tables.contains(t))); + + let after = plan.plan.len(); + + log::info!( + "Shrinking interaction plan from {} to {} properties", + before, + after + ); + + plan + } +} From 3b4e2bc81f6f8ba52ac4d351a53986e903fd9f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sat, 11 Jan 2025 10:01:03 +0900 Subject: [PATCH 14/97] Add dependencies and related licenses --- NOTICE.md | 19 +++ bindings/java/build.gradle.kts | 33 ++++- licenses/LICENSE.assertj.al20.txt | 201 +++++++++++++++++++++++++++ licenses/LICENSE.errorprone.al20.txt | 201 +++++++++++++++++++++++++++ 4 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 NOTICE.md create mode 100644 licenses/LICENSE.assertj.al20.txt create mode 100644 licenses/LICENSE.errorprone.al20.txt diff --git a/NOTICE.md b/NOTICE.md new file mode 100644 index 000000000..5a08d6fed --- /dev/null +++ b/NOTICE.md @@ -0,0 +1,19 @@ +Limbo +======= + +Please visit out github for more information: + +* https://github.com/tursodatabase/limbo + +Dependencies +============ + +This product depends on Error Prone, distributed by the Error Prone project: + +* License: licenses/LICENSE.errorprone.al20.txt (Apache License v2.0) +* Homepage: https://github.com/google/error-prone + +This product depends on AssertJ, distributed by the AssertJ authors: + +* License: licenses/LICENSE.assertj.al20.txt (Apache License v2.0) +* Homepage: https://joel-costigliola.github.io/assertj/ diff --git a/bindings/java/build.gradle.kts b/bindings/java/build.gradle.kts index f1349859d..286bf5c1b 100644 --- a/bindings/java/build.gradle.kts +++ b/bindings/java/build.gradle.kts @@ -1,16 +1,28 @@ +import net.ltgt.gradle.errorprone.CheckSeverity +import net.ltgt.gradle.errorprone.errorprone + plugins { java application + id("net.ltgt.errorprone") version "4.1.0" } group = "org.github.tursodatabase" version = "0.0.1-SNAPSHOT" +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + repositories { mavenCentral() } dependencies { + errorprone("com.uber.nullaway:nullaway:0.12.3") + errorprone("com.google.errorprone:error_prone_core:2.36.0") + testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") testImplementation("org.assertj:assertj-core:3.27.0") @@ -30,5 +42,24 @@ application { tasks.test { useJUnitPlatform() // In order to find rust built file under resources, we need to set it as system path - systemProperty("java.library.path", "${System.getProperty("java.library.path")}:$projectDir/src/test/resources/limbo/debug") + systemProperty( + "java.library.path", + "${System.getProperty("java.library.path")}:$projectDir/src/test/resources/limbo/debug" + ) +} + +tasks.withType { + options.errorprone { + check("NullAway", CheckSeverity.ERROR) + option("NullAway:AnnotatedPackages", "org.github.tursodatabase") + option( + "NullAway:CustomNullableAnnotations", + "org.github.tursodatabase.annotations.Nullable,org.github.tursodatabase.annotations.SkipNullableCheck" + ) + } + if (name.lowercase().contains("test")) { + options.errorprone { + disable("NullAway") + } + } } diff --git a/licenses/LICENSE.assertj.al20.txt b/licenses/LICENSE.assertj.al20.txt new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/licenses/LICENSE.assertj.al20.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/LICENSE.errorprone.al20.txt b/licenses/LICENSE.errorprone.al20.txt new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/licenses/LICENSE.errorprone.al20.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From aa6f2b3827e376b7d9cb98b2f6c062d16e1bf896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sat, 11 Jan 2025 10:15:58 +0900 Subject: [PATCH 15/97] Add @SkipNullableCheck and @Nullable annotations --- bindings/java/build.gradle.kts | 3 ++ .../java/org/github/tursodatabase/JDBC.java | 8 +++++- .../github/tursodatabase/LimboDataSource.java | 9 +++++- .../tursodatabase/annotations/Nullable.java | 18 ++++++++++++ .../annotations/SkipNullableCheck.java | 18 ++++++++++++ .../github/tursodatabase/core/LimboDB.java | 23 ++++++++++----- .../tursodatabase/jdbc4/JDBC4Connection.java | 28 ++++++++++++++++++- 7 files changed, 97 insertions(+), 10 deletions(-) create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/annotations/Nullable.java create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/annotations/SkipNullableCheck.java diff --git a/bindings/java/build.gradle.kts b/bindings/java/build.gradle.kts index 286bf5c1b..9936bec60 100644 --- a/bindings/java/build.gradle.kts +++ b/bindings/java/build.gradle.kts @@ -50,7 +50,10 @@ tasks.test { tasks.withType { options.errorprone { + // Let's select which checks to perform. NullAway is enough for now. + disableAllChecks = true check("NullAway", CheckSeverity.ERROR) + option("NullAway:AnnotatedPackages", "org.github.tursodatabase") option( "NullAway:CustomNullableAnnotations", diff --git a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java index 87200ff5c..10eeb2687 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java @@ -1,8 +1,11 @@ package org.github.tursodatabase; +import org.github.tursodatabase.annotations.Nullable; +import org.github.tursodatabase.annotations.SkipNullableCheck; import org.github.tursodatabase.jdbc4.JDBC4Connection; import java.sql.*; +import java.util.Locale; import java.util.Properties; import java.util.logging.Logger; @@ -17,6 +20,7 @@ public class JDBC implements Driver { } } + @Nullable public static LimboConnection createConnection(String url, Properties properties) throws SQLException { if (!isValidURL(url)) return null; @@ -25,13 +29,14 @@ public class JDBC implements Driver { } private static boolean isValidURL(String url) { - return url != null && url.toLowerCase().startsWith(VALID_URL_PREFIX); + return url != null && url.toLowerCase(Locale.ROOT).startsWith(VALID_URL_PREFIX); } private static String extractAddress(String url) { return url.substring(VALID_URL_PREFIX.length()); } + @Nullable @Override public Connection connect(String url, Properties info) throws SQLException { return createConnection(url, info); @@ -65,6 +70,7 @@ public class JDBC implements Driver { } @Override + @SkipNullableCheck public Logger getParentLogger() throws SQLFeatureNotSupportedException { // TODO return null; diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java index 12a53c303..97eec44d2 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java @@ -1,5 +1,7 @@ package org.github.tursodatabase; +import org.github.tursodatabase.annotations.Nullable; + import javax.sql.DataSource; import java.io.PrintWriter; import java.sql.Connection; @@ -27,12 +29,14 @@ public class LimboDataSource implements DataSource { } @Override + @Nullable public Connection getConnection() throws SQLException { return getConnection(null, null); } @Override - public Connection getConnection(String username, String password) throws SQLException { + @Nullable + public Connection getConnection(@Nullable String username, @Nullable String password) throws SQLException { Properties properties = limboConfig.toProperties(); if (username != null) properties.put("user", username); if (password != null) properties.put("pass", password); @@ -40,6 +44,7 @@ public class LimboDataSource implements DataSource { } @Override + @Nullable public PrintWriter getLogWriter() throws SQLException { // TODO return null; @@ -62,12 +67,14 @@ public class LimboDataSource implements DataSource { } @Override + @Nullable public Logger getParentLogger() throws SQLFeatureNotSupportedException { // TODO return null; } @Override + @Nullable public T unwrap(Class iface) throws SQLException { // TODO return null; diff --git a/bindings/java/src/main/java/org/github/tursodatabase/annotations/Nullable.java b/bindings/java/src/main/java/org/github/tursodatabase/annotations/Nullable.java new file mode 100644 index 000000000..88451f8b4 --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/annotations/Nullable.java @@ -0,0 +1,18 @@ +package org.github.tursodatabase.annotations; + + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to mark nullable types. + *

+ * This annotation is used to indicate that a method, field, or parameter can be null. + * It helps in identifying potential nullability issues and improving code quality. + */ +@Retention(RetentionPolicy.SOURCE) +@Target({ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER}) +public @interface Nullable { +} diff --git a/bindings/java/src/main/java/org/github/tursodatabase/annotations/SkipNullableCheck.java b/bindings/java/src/main/java/org/github/tursodatabase/annotations/SkipNullableCheck.java new file mode 100644 index 000000000..69214e7c4 --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/annotations/SkipNullableCheck.java @@ -0,0 +1,18 @@ +package org.github.tursodatabase.annotations; + + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marker annotation to skip nullable checks. + *

+ * This annotation is used to mark methods, fields, or parameters that should be excluded from nullable checks. + * It is typically applied to code that is still under development or requires special handling. + */ +@Retention(RetentionPolicy.SOURCE) +@Target({ElementType.METHOD, ElementType.FIELD, ElementType.PARAMETER}) +public @interface SkipNullableCheck { +} diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java index accbad76b..42d2e09d3 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java @@ -4,6 +4,7 @@ package org.github.tursodatabase.core; import org.github.tursodatabase.LimboErrorCode; import org.github.tursodatabase.NativeInvocation; import org.github.tursodatabase.VisibleForTesting; +import org.github.tursodatabase.annotations.Nullable; import org.github.tursodatabase.exceptions.LimboException; import java.nio.charset.StandardCharsets; @@ -82,9 +83,15 @@ public final class LimboDB extends AbstractDB { @Override protected void open0(String fileName, int openFlags) throws SQLException { if (isOpen) { - throwLimboException(LimboErrorCode.UNKNOWN_ERROR.code, "Already opened"); + throw buildLimboException(LimboErrorCode.ETC.code, "Already opened"); } - dbPtr = openUtf8(stringToUtf8ByteArray(fileName), openFlags); + + byte[] fileNameBytes = stringToUtf8ByteArray(fileName); + if (fileNameBytes == null) { + throw buildLimboException(LimboErrorCode.ETC.code, "File name cannot be converted to byteArray. File name: " + fileName); + } + + dbPtr = openUtf8(fileNameBytes, openFlags); isOpen = true; } @@ -114,7 +121,7 @@ public final class LimboDB extends AbstractDB { @NativeInvocation private void throwLimboException(int errorCode, byte[] errorMessageBytes) throws SQLException { String errorMessage = utf8ByteBufferToString(errorMessageBytes); - throwLimboException(errorCode, errorMessage); + throw buildLimboException(errorCode, errorMessage); } /** @@ -123,7 +130,7 @@ public final class LimboDB extends AbstractDB { * @param errorCode Error code. * @param errorMessage Error message. */ - public void throwLimboException(int errorCode, String errorMessage) throws SQLException { + public LimboException buildLimboException(int errorCode, @Nullable String errorMessage) throws SQLException { LimboErrorCode code = LimboErrorCode.getErrorCode(errorCode); String msg; if (code == LimboErrorCode.UNKNOWN_ERROR) { @@ -132,10 +139,11 @@ public final class LimboDB extends AbstractDB { msg = String.format("%s (%s)", code, errorMessage); } - throw new LimboException(msg, code); + return new LimboException(msg, code); } - private static String utf8ByteBufferToString(byte[] buffer) { + @Nullable + private static String utf8ByteBufferToString(@Nullable byte[] buffer) { if (buffer == null) { return null; } @@ -143,7 +151,8 @@ public final class LimboDB extends AbstractDB { return new String(buffer, StandardCharsets.UTF_8); } - private static byte[] stringToUtf8ByteArray(String str) { + @Nullable + private static byte[] stringToUtf8ByteArray(@Nullable String str) { if (str == null) { return null; } diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index 04c83b6b9..6ffb41e3e 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -1,6 +1,7 @@ package org.github.tursodatabase.jdbc4; import org.github.tursodatabase.LimboConnection; +import org.github.tursodatabase.annotations.SkipNullableCheck; import java.sql.*; import java.util.Map; @@ -14,24 +15,28 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public Statement createStatement() throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public PreparedStatement prepareStatement(String sql) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public CallableStatement prepareCall(String sql) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public String nativeSQL(String sql) throws SQLException { // TODO return ""; @@ -70,6 +75,7 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public DatabaseMetaData getMetaData() throws SQLException { // TODO return null; @@ -109,6 +115,7 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public SQLWarning getWarnings() throws SQLException { // TODO return null; @@ -120,18 +127,21 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { // TODO return null; @@ -159,12 +169,14 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public Savepoint setSavepoint() throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public Savepoint setSavepoint(String name) throws SQLException { // TODO return null; @@ -181,60 +193,70 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public Clob createClob() throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public Blob createBlob() throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public NClob createNClob() throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public SQLXML createSQLXML() throws SQLException { // TODO return null; @@ -263,18 +285,21 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public Properties getClientInfo() throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public Array createArrayOf(String typeName, Object[] elements) throws SQLException { // TODO return null; } @Override + @SkipNullableCheck public Struct createStruct(String typeName, Object[] attributes) throws SQLException { // TODO return null; @@ -286,6 +311,7 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public String getSchema() throws SQLException { // TODO return ""; @@ -308,8 +334,8 @@ public class JDBC4Connection extends LimboConnection { } @Override + @SkipNullableCheck public T unwrap(Class iface) throws SQLException { - // TODO return null; } From 82ca627d75650188e4fea83b293974deaa254327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sat, 11 Jan 2025 10:16:19 +0900 Subject: [PATCH 16/97] Reposition NativeInvocation, VisibleForTesting annotations under annotation package --- .../tursodatabase/{ => annotations}/NativeInvocation.java | 2 +- .../tursodatabase/{ => annotations}/VisibleForTesting.java | 2 +- .../src/main/java/org/github/tursodatabase/core/LimboDB.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename bindings/java/src/main/java/org/github/tursodatabase/{ => annotations}/NativeInvocation.java (88%) rename bindings/java/src/main/java/org/github/tursodatabase/{ => annotations}/VisibleForTesting.java (88%) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/NativeInvocation.java b/bindings/java/src/main/java/org/github/tursodatabase/annotations/NativeInvocation.java similarity index 88% rename from bindings/java/src/main/java/org/github/tursodatabase/NativeInvocation.java rename to bindings/java/src/main/java/org/github/tursodatabase/annotations/NativeInvocation.java index 70fd6c100..d3a905608 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/NativeInvocation.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/annotations/NativeInvocation.java @@ -1,4 +1,4 @@ -package org.github.tursodatabase; +package org.github.tursodatabase.annotations; import java.lang.annotation.ElementType; diff --git a/bindings/java/src/main/java/org/github/tursodatabase/VisibleForTesting.java b/bindings/java/src/main/java/org/github/tursodatabase/annotations/VisibleForTesting.java similarity index 88% rename from bindings/java/src/main/java/org/github/tursodatabase/VisibleForTesting.java rename to bindings/java/src/main/java/org/github/tursodatabase/annotations/VisibleForTesting.java index 1afd119c3..5f8d30458 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/VisibleForTesting.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/annotations/VisibleForTesting.java @@ -1,4 +1,4 @@ -package org.github.tursodatabase; +package org.github.tursodatabase.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java index 42d2e09d3..da829da63 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java @@ -2,8 +2,8 @@ package org.github.tursodatabase.core; import org.github.tursodatabase.LimboErrorCode; -import org.github.tursodatabase.NativeInvocation; -import org.github.tursodatabase.VisibleForTesting; +import org.github.tursodatabase.annotations.NativeInvocation; +import org.github.tursodatabase.annotations.VisibleForTesting; import org.github.tursodatabase.annotations.Nullable; import org.github.tursodatabase.exceptions.LimboException; From a8bf61f2373291c8722077be8ce80bbdefa84ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sat, 11 Jan 2025 13:44:22 +0900 Subject: [PATCH 17/97] Change to SkipNullableCheck --- .../java/org/github/tursodatabase/LimboDataSource.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java index 97eec44d2..ff98ec651 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboDataSource.java @@ -1,6 +1,7 @@ package org.github.tursodatabase; import org.github.tursodatabase.annotations.Nullable; +import org.github.tursodatabase.annotations.SkipNullableCheck; import javax.sql.DataSource; import java.io.PrintWriter; @@ -42,9 +43,8 @@ public class LimboDataSource implements DataSource { if (password != null) properties.put("pass", password); return JDBC.createConnection(url, properties); } - @Override - @Nullable + @SkipNullableCheck public PrintWriter getLogWriter() throws SQLException { // TODO return null; @@ -67,14 +67,14 @@ public class LimboDataSource implements DataSource { } @Override - @Nullable + @SkipNullableCheck public Logger getParentLogger() throws SQLFeatureNotSupportedException { // TODO return null; } @Override - @Nullable + @SkipNullableCheck public T unwrap(Class iface) throws SQLException { // TODO return null; From 016d9d17abcb844a3c86d1400751f37cc392522d Mon Sep 17 00:00:00 2001 From: Jorge Hermo Date: Mon, 13 Jan 2025 00:05:14 +0100 Subject: [PATCH 18/97] refactor: json functions vdbe --- core/vdbe/mod.rs | 143 +++++++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 74 deletions(-) diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 64c41b3bd..208c002a6 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -1343,84 +1343,79 @@ impl Program { let arg_count = func.arg_count; match &func.func { #[cfg(feature = "json")] - crate::function::Func::Json(JsonFunc::Json) => { - let json_value = &state.registers[*start_reg]; - let json_str = get_json(json_value); - match json_str { - Ok(json) => state.registers[*dest] = json, - Err(e) => return Err(e), - } - } - #[cfg(feature = "json")] - crate::function::Func::Json(JsonFunc::JsonArray) => { - let reg_values = &state.registers[*start_reg..*start_reg + arg_count]; - - let json_array = json_array(reg_values); - - match json_array { - Ok(json) => state.registers[*dest] = json, - Err(e) => return Err(e), - } - } - #[cfg(feature = "json")] - crate::function::Func::Json(JsonFunc::JsonExtract) => { - let result = match arg_count { - 0 => json_extract(&OwnedValue::Null, &[]), - _ => { - let val = &state.registers[*start_reg]; - let reg_values = - &state.registers[*start_reg + 1..*start_reg + arg_count]; - - json_extract(val, reg_values) + crate::function::Func::Json(json_func) => match json_func { + JsonFunc::Json => { + let json_value = &state.registers[*start_reg]; + let json_str = get_json(json_value); + match json_str { + Ok(json) => state.registers[*dest] = json, + Err(e) => return Err(e), } - }; + } + JsonFunc::JsonArray => { + let reg_values = + &state.registers[*start_reg..*start_reg + arg_count]; - match result { - Ok(json) => state.registers[*dest] = json, - Err(e) => return Err(e), - } - } - #[cfg(feature = "json")] - crate::function::Func::Json( - func @ (JsonFunc::JsonArrowExtract | JsonFunc::JsonArrowShiftExtract), - ) => { - assert_eq!(arg_count, 2); - let json = &state.registers[*start_reg]; - let path = &state.registers[*start_reg + 1]; - let func = match func { - JsonFunc::JsonArrowExtract => json_arrow_extract, - JsonFunc::JsonArrowShiftExtract => json_arrow_shift_extract, - _ => unreachable!(), - }; - let json_str = func(json, path); - match json_str { - Ok(json) => state.registers[*dest] = json, - Err(e) => return Err(e), - } - } - #[cfg(feature = "json")] - crate::function::Func::Json( - func @ (JsonFunc::JsonArrayLength | JsonFunc::JsonType), - ) => { - let json_value = &state.registers[*start_reg]; - let path_value = if arg_count > 1 { - Some(&state.registers[*start_reg + 1]) - } else { - None - }; - let func_result = match func { - JsonFunc::JsonArrayLength => { - json_array_length(json_value, path_value) + let json_array = json_array(reg_values); + + match json_array { + Ok(json) => state.registers[*dest] = json, + Err(e) => return Err(e), } - JsonFunc::JsonType => json_type(json_value, path_value), - _ => unreachable!(), - }; - - match func_result { - Ok(result) => state.registers[*dest] = result, - Err(e) => return Err(e), } - } + JsonFunc::JsonExtract => { + let result = match arg_count { + 0 => json_extract(&OwnedValue::Null, &[]), + _ => { + let val = &state.registers[*start_reg]; + let reg_values = &state.registers + [*start_reg + 1..*start_reg + arg_count]; + + json_extract(val, reg_values) + } + }; + + match result { + Ok(json) => state.registers[*dest] = json, + Err(e) => return Err(e), + } + } + JsonFunc::JsonArrowExtract | JsonFunc::JsonArrowShiftExtract => { + assert_eq!(arg_count, 2); + let json = &state.registers[*start_reg]; + let path = &state.registers[*start_reg + 1]; + let json_func = match json_func { + JsonFunc::JsonArrowExtract => json_arrow_extract, + JsonFunc::JsonArrowShiftExtract => json_arrow_shift_extract, + _ => unreachable!(), + }; + let json_str = json_func(json, path); + match json_str { + Ok(json) => state.registers[*dest] = json, + Err(e) => return Err(e), + } + } + JsonFunc::JsonArrayLength | JsonFunc::JsonType => { + let json_value = &state.registers[*start_reg]; + let path_value = if arg_count > 1 { + Some(&state.registers[*start_reg + 1]) + } else { + None + }; + let func_result = match json_func { + JsonFunc::JsonArrayLength => { + json_array_length(json_value, path_value) + } + JsonFunc::JsonType => json_type(json_value, path_value), + _ => unreachable!(), + }; + + match func_result { + Ok(result) => state.registers[*dest] = result, + Err(e) => return Err(e), + } + } + }, crate::function::Func::Scalar(scalar_func) => match scalar_func { ScalarFunc::Cast => { assert!(arg_count == 2); From 82fcc27a58321c071423ef058d4f245947c19b21 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 02:31:19 +0300 Subject: [PATCH 19/97] this commit fixes query generation; - previous query generation method was faulty, producing wrong assertions - this commit adds a new arbitrary_from implementation for predicates - new implementation takes a table and a row, and produces a predicate that would evaluate to true for the row this commit makes small changes to the main for increasing readability --- simulator/generation/property.rs | 5 +- simulator/generation/query.rs | 164 ++++++++++++++++++++++++++++--- simulator/generation/table.rs | 4 +- simulator/main.rs | 114 +++++++++++---------- simulator/model/query.rs | 10 ++ simulator/runner/cli.rs | 15 +++ simulator/shrink/plan.rs | 4 + 7 files changed, 245 insertions(+), 71 deletions(-) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 39956210b..358b5be17 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -184,10 +184,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop // Select the row let select_query = Select { table: table.name.clone(), - predicate: Predicate::arbitrary_from( - rng, - &(table, &Predicate::Eq(column.name.clone(), value.clone())), - ), + predicate: Predicate::arbitrary_from(rng, &(table, &row)), }; Property::InsertSelect { diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index c99638f6d..bc71515c1 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -3,6 +3,7 @@ use crate::generation::{one_of, Arbitrary, ArbitraryFrom}; use crate::model::query::{Create, Delete, Insert, Predicate, Query, Select}; use crate::model::table::{Table, Value}; +use rand::seq::SliceRandom as _; use rand::Rng; use super::{frequency, pick}; @@ -174,7 +175,7 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { let len = booleans.len(); // Make sure at least one of them is false - if booleans.iter().all(|b| *b) { + if !booleans.is_empty() && booleans.iter().all(|b| *b) { booleans[rng.gen_range(0..len)] = false; } @@ -195,7 +196,7 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { .collect::>(); let len = booleans.len(); // Make sure at least one of them is true - if booleans.iter().all(|b| !*b) { + if !booleans.is_empty() && booleans.iter().all(|b| !*b) { booleans[rng.gen_range(0..len)] = true; } @@ -246,16 +247,155 @@ impl ArbitraryFrom<(&str, &Value)> for Predicate { } } -impl ArbitraryFrom<(&Table, &Predicate)> for Predicate { - fn arbitrary_from(rng: &mut R, (t, p): &(&Table, &Predicate)) -> Self { - if rng.gen_bool(0.5) { - // produce a true predicate - let p_t = CompoundPredicate::arbitrary_from(rng, &(*t, true)).0; - Predicate::And(vec![p_t, (*p).clone()]) - } else { - // produce a false predicate - let p_f = CompoundPredicate::arbitrary_from(rng, &(*t, false)).0; - Predicate::Or(vec![p_f, (*p).clone()]) +/// Produces a predicate that is true for the provided row in the given table +fn produce_true_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) -> Predicate { + // Pick a column + let column_index = rng.gen_range(0..t.columns.len()); + let column = &t.columns[column_index]; + let value = &row[column_index]; + one_of( + vec![ + Box::new(|_| Predicate::Eq(column.name.clone(), value.clone())), + Box::new(|rng| { + let v = loop { + let v = Value::arbitrary_from(rng, &column.column_type); + if &v != value { + break v; + } + }; + Predicate::Neq(column.name.clone(), v) + }), + Box::new(|rng| { + Predicate::Gt(column.name.clone(), LTValue::arbitrary_from(rng, value).0) + }), + Box::new(|rng| { + Predicate::Lt(column.name.clone(), GTValue::arbitrary_from(rng, value).0) + }), + ], + rng, + ) +} + +/// Produces a predicate that is false for the provided row in the given table +fn produce_false_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) -> Predicate { + // Pick a column + let column_index = rng.gen_range(0..t.columns.len()); + let column = &t.columns[column_index]; + let value = &row[column_index]; + one_of( + vec![ + Box::new(|_| Predicate::Neq(column.name.clone(), value.clone())), + Box::new(|rng| { + let v = loop { + let v = Value::arbitrary_from(rng, &column.column_type); + if &v != value { + break v; + } + }; + Predicate::Eq(column.name.clone(), v) + }), + Box::new(|rng| { + Predicate::Gt(column.name.clone(), GTValue::arbitrary_from(rng, value).0) + }), + Box::new(|rng| { + Predicate::Lt(column.name.clone(), LTValue::arbitrary_from(rng, value).0) + }), + ], + rng, + ) +} + +impl ArbitraryFrom<(&Table, &Vec)> for Predicate { + fn arbitrary_from(rng: &mut R, (t, row): &(&Table, &Vec)) -> Self { + // We want to produce a predicate that is true for the row + // We can do this by creating several predicates that + // are true, some that are false, combiend them in ways that correspond to the creation of a true predicate + + // Produce some true and false predicates + let mut true_predicates = (1..=rng.gen_range(1..=4)) + .map(|_| produce_true_predicate(rng, &(*t, row))) + .collect::>(); + + let false_predicates = (0..=rng.gen_range(0..=3)) + .map(|_| produce_false_predicate(rng, &(*t, row))) + .collect::>(); + + // Start building a top level predicate from a true predicate + let mut result = true_predicates.pop().unwrap(); + println!("True predicate: {:?}", result); + + let mut predicates = true_predicates + .iter() + .map(|p| (true, p.clone())) + .chain(false_predicates.iter().map(|p| (false, p.clone()))) + .collect::>(); + + predicates.shuffle(rng); + + while !predicates.is_empty() { + // Create a new predicate from at least 1 and at most 3 predicates + let context = + predicates[0..rng.gen_range(0..=usize::min(3, predicates.len()))].to_vec(); + // Shift `predicates` to remove the predicates in the context + predicates = predicates[context.len()..].to_vec(); + + // `result` is true, so we have the following three options to make a true predicate: + // T or F + // T or T + // T and T + + result = one_of( + vec![ + // T or (X1 or X2 or ... or Xn) + Box::new(|_| { + Predicate::Or(vec![ + result.clone(), + Predicate::Or(context.iter().map(|(_, p)| p.clone()).collect()), + ]) + }), + // T or (T1 and T2 and ... and Tn) + Box::new(|_| { + Predicate::Or(vec![ + result.clone(), + Predicate::And(context.iter().map(|(_, p)| p.clone()).collect()), + ]) + }), + // T and T + Box::new(|_| { + // Check if all the predicates in the context are true + if context.iter().all(|(b, _)| *b) { + // T and (X1 or X2 or ... or Xn) + Predicate::And(vec![ + result.clone(), + Predicate::And(context.iter().map(|(_, p)| p.clone()).collect()), + ]) + } + // Check if there is at least one true predicate + else if context.iter().any(|(b, _)| *b) { + // T and (X1 or X2 or ... or Xn) + Predicate::And(vec![ + result.clone(), + Predicate::Or(context.iter().map(|(_, p)| p.clone()).collect()), + ]) + } else { + // T and (X1 or X2 or ... or Xn or TRUE) + Predicate::And(vec![ + result.clone(), + Predicate::Or( + context + .iter() + .map(|(_, p)| p.clone()) + .chain(std::iter::once(Predicate::true_())) + .collect(), + ), + ]) + } + }), + ], + rng, + ); } + + result } } diff --git a/simulator/generation/table.rs b/simulator/generation/table.rs index 179c53436..b5b898eeb 100644 --- a/simulator/generation/table.rs +++ b/simulator/generation/table.rs @@ -15,7 +15,7 @@ impl Arbitrary for Name { impl Arbitrary for Table { fn arbitrary(rng: &mut R) -> Self { let name = Name::arbitrary(rng).0; - let columns = (1..=rng.gen_range(1..5)) + let columns = (1..=rng.gen_range(1..10)) .map(|_| Column::arbitrary(rng)) .collect(); Table { @@ -83,7 +83,7 @@ impl ArbitraryFrom for LTValue { fn arbitrary_from(rng: &mut R, value: &Value) -> Self { match value { Value::Integer(i) => Self(Value::Integer(rng.gen_range(i64::MIN..*i - 1))), - Value::Float(f) => Self(Value::Float(rng.gen_range(-1e10..*f - 1.0))), + Value::Float(f) => Self(Value::Float(f - rng.gen_range(0.0..1e10))), Value::Text(t) => { // Either shorten the string, or make at least one character smaller and mutate the rest let mut t = t.clone(); diff --git a/simulator/main.rs b/simulator/main.rs index a4e99f6ea..1728744af 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -12,7 +12,7 @@ use runner::io::SimulatorIO; use std::any::Any; use std::backtrace::Backtrace; use std::io::Write; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use tempfile::TempDir; @@ -21,10 +21,48 @@ mod model; mod runner; mod shrink; -fn main() { +struct Paths { + db: PathBuf, + plan: PathBuf, + shrunk_plan: PathBuf, + history: PathBuf, + doublecheck_db: PathBuf, + shrunk_db: PathBuf, +} + +impl Paths { + fn new(output_dir: &Path, shrink: bool, doublecheck: bool) -> Self { + let paths = Paths { + db: PathBuf::from(output_dir).join("simulator.db"), + plan: PathBuf::from(output_dir).join("simulator.plan"), + shrunk_plan: PathBuf::from(output_dir).join("simulator_shrunk.plan"), + history: PathBuf::from(output_dir).join("simulator.history"), + doublecheck_db: PathBuf::from(output_dir).join("simulator_double.db"), + shrunk_db: PathBuf::from(output_dir).join("simulator_shrunk.db"), + }; + + // Print the seed, the locations of the database and the plan file + log::info!("database path: {:?}", paths.db); + if doublecheck { + log::info!("doublecheck database path: {:?}", paths.doublecheck_db); + } else if shrink { + log::info!("shrunk database path: {:?}", paths.shrunk_db); + } + log::info!("simulator plan path: {:?}", paths.plan); + if shrink { + log::info!("shrunk plan path: {:?}", paths.shrunk_plan); + } + log::info!("simulator history path: {:?}", paths.history); + + paths + } +} + +fn main() -> Result<(), String> { let _ = env_logger::try_init(); let cli_opts = SimulatorCLI::parse(); + cli_opts.validate()?; let seed = match cli_opts.seed { Some(seed) => seed, @@ -33,30 +71,10 @@ fn main() { let output_dir = match &cli_opts.output_dir { Some(dir) => Path::new(dir).to_path_buf(), - None => TempDir::new().unwrap().into_path(), + None => TempDir::new().map_err(|e| format!("{:?}", e))?.into_path(), }; - let db_path = output_dir.join("simulator.db"); - let doublecheck_db_path = db_path.with_extension("_doublecheck.db"); - let shrunk_db_path = db_path.with_extension("_shrink.db"); - - let plan_path = output_dir.join("simulator.plan"); - let shrunk_plan_path = plan_path.with_extension("_shrunk.plan"); - - let history_path = output_dir.join("simulator.history"); - - // Print the seed, the locations of the database and the plan file - log::info!("database path: {:?}", db_path); - if cli_opts.doublecheck { - log::info!("doublecheck database path: {:?}", doublecheck_db_path); - } else if cli_opts.shrink { - log::info!("shrunk database path: {:?}", shrunk_db_path); - } - log::info!("simulator plan path: {:?}", plan_path); - if cli_opts.shrink { - log::info!("shrunk plan path: {:?}", shrunk_plan_path); - } - log::info!("simulator history path: {:?}", history_path); + let paths = Paths::new(&output_dir, cli_opts.shrink, cli_opts.doublecheck); log::info!("seed: {}", seed); let last_execution = Arc::new(Mutex::new(Execution::new(0, 0, 0))); @@ -82,8 +100,8 @@ fn main() { run_simulation( seed, &cli_opts, - &db_path, - &plan_path, + &paths.db, + &paths.plan, last_execution.clone(), None, ) @@ -98,8 +116,8 @@ fn main() { run_simulation( seed, &cli_opts, - &doublecheck_db_path, - &plan_path, + &paths.doublecheck_db, + &paths.plan, last_execution.clone(), None, ) @@ -138,8 +156,8 @@ fn main() { | (SandboxedResult::FoundBug { .. }, SandboxedResult::FoundBug { .. }) | (SandboxedResult::Panicked { .. }, SandboxedResult::Panicked { .. }) => { // Compare the two database files byte by byte - let db_bytes = std::fs::read(&db_path).unwrap(); - let doublecheck_db_bytes = std::fs::read(&doublecheck_db_path).unwrap(); + let db_bytes = std::fs::read(&paths.db).unwrap(); + let doublecheck_db_bytes = std::fs::read(&paths.doublecheck_db).unwrap(); if db_bytes != doublecheck_db_bytes { log::error!("doublecheck failed! database files are different."); } else { @@ -164,7 +182,7 @@ fn main() { } => { if let SandboxedResult::FoundBug { history, .. } = &result { // No panic occurred, so write the history to a file - let f = std::fs::File::create(&history_path).unwrap(); + let f = std::fs::File::create(&paths.history).unwrap(); let mut f = std::io::BufWriter::new(f); for execution in history.history.iter() { writeln!( @@ -190,8 +208,8 @@ fn main() { run_simulation( seed, &cli_opts, - &shrunk_db_path, - &shrunk_plan_path, + &paths.shrunk_db, + &paths.shrunk_plan, last_execution.clone(), shrink, ) @@ -225,8 +243,8 @@ fn main() { } // Write the shrunk plan to a file - let shrunk_plan = std::fs::read(&shrunk_plan_path).unwrap(); - let mut f = std::fs::File::create(&shrunk_plan_path).unwrap(); + let shrunk_plan = std::fs::read(&paths.shrunk_plan).unwrap(); + let mut f = std::fs::File::create(&paths.shrunk_plan).unwrap(); f.write_all(&shrunk_plan).unwrap(); } } @@ -234,18 +252,20 @@ fn main() { } // Print the seed, the locations of the database and the plan file at the end again for easily accessing them. - println!("database path: {:?}", db_path); + println!("database path: {:?}", paths.db); if cli_opts.doublecheck { - println!("doublecheck database path: {:?}", doublecheck_db_path); + println!("doublecheck database path: {:?}", paths.doublecheck_db); } else if cli_opts.shrink { - println!("shrunk database path: {:?}", shrunk_db_path); + println!("shrunk database path: {:?}", paths.shrunk_db); } - println!("simulator plan path: {:?}", plan_path); + println!("simulator plan path: {:?}", paths.plan); if cli_opts.shrink { - println!("shrunk plan path: {:?}", shrunk_plan_path); + println!("shrunk plan path: {:?}", paths.shrunk_plan); } - println!("simulator history path: {:?}", history_path); + println!("simulator history path: {:?}", paths.history); println!("seed: {}", seed); + + Ok(()) } fn move_db_and_plan_files(output_dir: &Path) { @@ -346,18 +366,6 @@ fn run_simulation( (create_percent, read_percent, write_percent, delete_percent) }; - if cli_opts.minimum_size < 1 { - panic!("minimum size must be at least 1"); - } - - if cli_opts.maximum_size < 1 { - panic!("maximum size must be at least 1"); - } - - if cli_opts.maximum_size < cli_opts.minimum_size { - panic!("maximum size must be greater than or equal to minimum size"); - } - let opts = SimulatorOpts { ticks: rng.gen_range(cli_opts.minimum_size..=cli_opts.maximum_size), max_connections: 1, // TODO: for now let's use one connection as we didn't implement diff --git a/simulator/model/query.rs b/simulator/model/query.rs index 40d7b7c89..a111a508f 100644 --- a/simulator/model/query.rs +++ b/simulator/model/query.rs @@ -12,6 +12,16 @@ pub(crate) enum Predicate { Lt(String, Value), // column < Value } +impl Predicate { + pub(crate) fn true_() -> Self { + Self::And(vec![]) + } + + pub(crate) fn false_() -> Self { + Self::Or(vec![]) + } +} + impl Display for Predicate { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index 6b56b9b82..365ad6a77 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -42,3 +42,18 @@ pub struct SimulatorCLI { )] pub shrink: bool, } + +impl SimulatorCLI { + pub fn validate(&self) -> Result<(), String> { + if self.minimum_size < 1 { + return Err("minimum size must be at least 1".to_string()); + } + if self.maximum_size < 1 { + return Err("maximum size must be at least 1".to_string()); + } + if self.minimum_size > self.maximum_size { + return Err("Minimum size cannot be greater than maximum size".to_string()); + } + Ok(()) + } +} diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 87eb90248..01fe18f48 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -3,6 +3,10 @@ use crate::{generation::plan::InteractionPlan, runner::execution::Execution}; impl InteractionPlan { /// Create a smaller interaction plan by deleting a property pub(crate) fn shrink_interaction_plan(&self, failing_execution: &Execution) -> InteractionPlan { + // todo: this is a very naive implementation, next steps are; + // - Shrink to multiple values by removing random interactions + // - Shrink properties by removing their extensions, or shrinking their values + let mut plan = self.clone(); let failing_property = &self.plan[failing_execution.interaction_index]; let depending_tables = failing_property.dependencies(); From 13442808dd0a77504cecf1c514f2f521e062d0b1 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 14:35:42 +0300 Subject: [PATCH 20/97] update properties to add extensional interactions between them --- simulator/generation/property.rs | 128 +++++++++++++++++++++++++------ simulator/generation/query.rs | 28 ++++++- simulator/model/query.rs | 20 +++++ simulator/model/table.rs | 16 ++++ simulator/shrink/plan.rs | 22 +++++- 5 files changed, 189 insertions(+), 25 deletions(-) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 358b5be17..7ae82de30 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -1,6 +1,6 @@ use crate::{ model::{ - query::{Create, Insert, Predicate, Query, Select}, + query::{Create, Delete, Insert, Predicate, Query, Select}, table::Value, }, runner::env::SimulatorEnv, @@ -35,7 +35,7 @@ pub(crate) enum Property { /// The insert query insert: Insert, /// Additional interactions in the middle of the property - interactions: Vec, + queries: Vec, /// The select query select: Select, }, @@ -55,7 +55,7 @@ pub(crate) enum Property { /// The create query create: Create, /// Additional interactions in the middle of the property - interactions: Vec, + queries: Vec, }, } @@ -70,7 +70,7 @@ impl Property { match self { Property::InsertSelect { insert, - interactions: _, // todo: add extensional interactions + queries, select, } => { // Check that the row is there @@ -106,21 +106,34 @@ impl Property { }), }); - vec![ - assumption, - Interaction::Query(Query::Insert(insert.clone())), - Interaction::Query(Query::Select(select.clone())), - assertion, - ] + let mut interactions = Vec::new(); + interactions.push(assumption); + interactions.push(Interaction::Query(Query::Insert(insert.clone()))); + interactions.extend(queries.clone().into_iter().map(Interaction::Query)); + interactions.push(Interaction::Query(Query::Select(select.clone()))); + interactions.push(assertion); + + interactions } Property::DoubleCreateFailure { create, - interactions: _, // todo: add extensional interactions + queries, } => { let table_name = create.table.name.clone(); + + let assumption = Interaction::Assumption(Assertion { + message: "Double-Create-Failure should not be called on an existing table" + .to_string(), + func: Box::new(move |_: &Vec, env: &SimulatorEnv| { + !env.tables.iter().any(|t| t.name == table_name) + }), + }); + let cq1 = Interaction::Query(Query::Create(create.clone())); let cq2 = Interaction::Query(Query::Create(create.clone())); + let table_name = create.table.name.clone(); + let assertion = Interaction::Assertion(Assertion { message: "creating two tables with the name should result in a failure for the second query" @@ -136,13 +149,26 @@ impl Property { }), }); - vec![cq1, cq2, assertion] + let mut interactions = Vec::new(); + interactions.push(assumption); + interactions.push(cq1); + interactions.extend(queries.clone().into_iter().map(Interaction::Query)); + interactions.push(cq2); + interactions.push(assertion); + + interactions } } } } -fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> (f64, f64, f64) { +pub(crate) struct Remaining { + pub(crate) read: f64, + pub(crate) write: f64, + pub(crate) create: f64, +} + +fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaining { let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) - (stats.read_count as f64)) .max(0.0); @@ -153,10 +179,14 @@ fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> (f64, f64, f64) { - (stats.create_count as f64)) .max(0.0); - (remaining_read, remaining_write, remaining_create) + Remaining { + read: remaining_read, + write: remaining_write, + create: remaining_create, + } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Property { +fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaining: &Remaining) -> Property { // Get a random table let table = pick(&env.tables, rng); // Pick a random column @@ -181,6 +211,36 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop values: vec![row.clone()], }; + // Create random queries respecting the constraints + let mut queries = Vec::new(); + // - [x] There will be no errors in the middle interactions. (this constraint is impossible to check, so this is just best effort) + // - [x] The inserted row will not be deleted. + // - [ ] The inserted row will not be updated. (todo: add this constraint once UPDATE is implemented) + // - [ ] The table `t` will not be renamed, dropped, or altered. (todo: add this constraint once ALTER or DROP is implemented) + for _ in 0..rng.gen_range(0..3) { + let query = Query::arbitrary_from(rng, &(table, remaining)); + match &query { + Query::Delete(Delete { + table: t, + predicate, + }) => { + // The inserted row will not be deleted. + if t == &table.name && predicate.test(&row, &table) { + continue; + } + } + Query::Create(Create { table: t }) => { + // There will be no errors in the middle interactions. + // - Creating the same table is an error + if t.name == table.name { + continue; + } + } + _ => (), + } + queries.push(query); + } + // Select the row let select_query = Select { table: table.name.clone(), @@ -189,12 +249,12 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv) -> Prop Property::InsertSelect { insert: insert_query, - interactions: Vec::new(), + queries, select: select_query, } } -fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) -> Property { +fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv, remaining: &Remaining) -> Property { // Get a random table let table = pick(&env.tables, rng); // Create the table @@ -202,27 +262,49 @@ fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv) table: table.clone(), }; + // Create random queries respecting the constraints + let mut queries = Vec::new(); + // The interactions in the middle has the following constraints; + // - [x] There will be no errors in the middle interactions.(best effort) + // - [ ] Table `t` will not be renamed or dropped.(todo: add this constraint once ALTER or DROP is implemented) + for _ in 0..rng.gen_range(0..3) { + let query = Query::arbitrary_from(rng, &(table, remaining)); + match &query { + Query::Create(Create { table: t }) => { + // There will be no errors in the middle interactions. + // - Creating the same table is an error + if t.name == table.name { + continue; + } + } + _ => (), + } + queries.push(query); + } + Property::DoubleCreateFailure { create: create_query, - interactions: Vec::new(), + queries, } } + + impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property { fn arbitrary_from( rng: &mut R, (env, stats): &(&SimulatorEnv, &InteractionStats), ) -> Self { - let (remaining_read, remaining_write, remaining_create) = remaining(env, stats); + let remaining_ = remaining(env, stats); frequency( vec![ ( - f64::min(remaining_read, remaining_write), - Box::new(|rng: &mut R| property_insert_select(rng, env)), + f64::min(remaining_.read, remaining_.write), + Box::new(|rng: &mut R| property_insert_select(rng, env, &remaining_)), ), ( - remaining_create / 2.0, - Box::new(|rng: &mut R| property_double_create_failure(rng, env)), + remaining_.create / 2.0, + Box::new(|rng: &mut R| property_double_create_failure(rng, env, &remaining_)), ), ], rng, diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index bc71515c1..0229a4750 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -6,6 +6,7 @@ use crate::model::table::{Table, Value}; use rand::seq::SliceRandom as _; use rand::Rng; +use super::property::Remaining; use super::{frequency, pick}; impl Arbitrary for Create { @@ -87,6 +88,32 @@ impl ArbitraryFrom for Query { } } +impl ArbitraryFrom<(&Table, &Remaining)> for Query { + fn arbitrary_from(rng: &mut R, (table, remaining): &(&Table, &Remaining)) -> Self { + frequency( + vec![ + ( + remaining.create, + Box::new(|rng| Self::Create(Create::arbitrary(rng))), + ), + ( + remaining.read, + Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![*table]))), + ), + ( + remaining.write, + Box::new(|rng| Self::Insert(Insert::arbitrary_from(rng, table))), + ), + ( + 0.0, + Box::new(|rng| Self::Delete(Delete::arbitrary_from(rng, table))), + ), + ], + rng, + ) + } +} + struct CompoundPredicate(Predicate); struct SimplePredicate(Predicate); @@ -322,7 +349,6 @@ impl ArbitraryFrom<(&Table, &Vec)> for Predicate { // Start building a top level predicate from a true predicate let mut result = true_predicates.pop().unwrap(); - println!("True predicate: {:?}", result); let mut predicates = true_predicates .iter() diff --git a/simulator/model/query.rs b/simulator/model/query.rs index a111a508f..9138b1988 100644 --- a/simulator/model/query.rs +++ b/simulator/model/query.rs @@ -20,6 +20,26 @@ impl Predicate { pub(crate) fn false_() -> Self { Self::Or(vec![]) } + + pub(crate) fn test(&self, row: &[Value], table: &Table) -> bool { + let get_value = |name: &str| { + table + .columns + .iter() + .zip(row.iter()) + .find(|(column, _)| column.name == name) + .map(|(_, value)| value) + }; + + match self { + Predicate::And(vec) => vec.iter().all(|p| p.test(row, table)), + Predicate::Or(vec) => vec.iter().any(|p| p.test(row, table)), + Predicate::Eq(column, value) => get_value(column) == Some(value), + Predicate::Neq(column, value) => get_value(column) != Some(value), + Predicate::Gt(column, value) => get_value(column).map(|v| v > value).unwrap_or(false), + Predicate::Lt(column, value) => get_value(column).map(|v| v < value).unwrap_or(false), + } + } } impl Display for Predicate { diff --git a/simulator/model/table.rs b/simulator/model/table.rs index 841ae0023..ab3b003af 100644 --- a/simulator/model/table.rs +++ b/simulator/model/table.rs @@ -53,6 +53,22 @@ pub(crate) enum Value { Blob(Vec), } +impl PartialOrd for Value { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (Self::Null, Self::Null) => Some(std::cmp::Ordering::Equal), + (Self::Null, _) => Some(std::cmp::Ordering::Less), + (_, Self::Null) => Some(std::cmp::Ordering::Greater), + (Self::Integer(i1), Self::Integer(i2)) => i1.partial_cmp(i2), + (Self::Float(f1), Self::Float(f2)) => f1.partial_cmp(f2), + (Self::Text(t1), Self::Text(t2)) => t1.partial_cmp(t2), + (Self::Blob(b1), Self::Blob(b2)) => b1.partial_cmp(b2), + // todo: add type coercions here + _ => None, + } + } +} + fn to_sqlite_blob(bytes: &[u8]) -> String { format!( "X'{}'", diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 01fe18f48..2f89a127f 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -1,4 +1,7 @@ -use crate::{generation::plan::InteractionPlan, runner::execution::Execution}; +use crate::{ + generation::plan::{Interaction, InteractionPlan, Interactions}, + runner::execution::Execution, +}; impl InteractionPlan { /// Create a smaller interaction plan by deleting a property @@ -19,6 +22,23 @@ impl InteractionPlan { plan.plan .retain(|p| p.uses().iter().any(|t| depending_tables.contains(t))); + // Remove the extensional parts of the properties + for interaction in plan.plan.iter_mut() { + if let Interactions::Property(p) = interaction { + match p { + crate::generation::property::Property::InsertSelect { + queries, + .. + } | + crate::generation::property::Property::DoubleCreateFailure { + queries, + .. + } => { + queries.clear(); + } + } + } + } let after = plan.plan.len(); log::info!( From 43f6c344084bd70d680b0fd6ad534e672d354f21 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 14:43:12 +0300 Subject: [PATCH 21/97] fix arbitrary_from ergonomics by removing the implicit reference in the trait signature --- simulator/generation/mod.rs | 2 +- simulator/generation/plan.rs | 12 ++++---- simulator/generation/property.rs | 27 +++++++++-------- simulator/generation/query.rs | 50 ++++++++++++++++---------------- simulator/generation/table.rs | 12 ++++---- simulator/main.rs | 1 + simulator/shrink/plan.rs | 12 +++----- 7 files changed, 57 insertions(+), 59 deletions(-) diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index 6107124f0..73eb77e80 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -13,7 +13,7 @@ pub trait Arbitrary { } pub trait ArbitraryFrom { - fn arbitrary_from(rng: &mut R, t: &T) -> Self; + fn arbitrary_from(rng: &mut R, t: T) -> Self; } pub(crate) fn frequency< diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index e286bb34a..1deeaef42 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -283,10 +283,8 @@ impl InteractionPlan { } } -impl InteractionPlan { - // todo: This is a hack to get around the fact that `ArbitraryFrom` can't take a mutable - // reference of T, so instead write a bespoke function without using the trait system. - pub(crate) fn arbitrary_from(rng: &mut R, env: &mut SimulatorEnv) -> Self { +impl ArbitraryFrom<&mut SimulatorEnv> for InteractionPlan { + fn arbitrary_from(rng: &mut R, env: &mut SimulatorEnv) -> Self { let mut plan = InteractionPlan::new(); let num_interactions = env.opts.max_interactions; @@ -304,7 +302,7 @@ impl InteractionPlan { plan.plan.len(), num_interactions ); - let interactions = Interactions::arbitrary_from(rng, &(env, plan.stats())); + let interactions = Interactions::arbitrary_from(rng, (env, plan.stats())); interactions.shadow(env); plan.plan.push(interactions); @@ -471,7 +469,7 @@ fn random_fault(_rng: &mut R, _env: &SimulatorEnv) -> Interactions impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { fn arbitrary_from( rng: &mut R, - (env, stats): &(&SimulatorEnv, InteractionStats), + (env, stats): (&SimulatorEnv, InteractionStats), ) -> Self { let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) - (stats.read_count as f64)) @@ -489,7 +487,7 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { ( f64::min(remaining_read, remaining_write) + remaining_create, Box::new(|rng: &mut R| { - Interactions::Property(Property::arbitrary_from(rng, &(env, stats))) + Interactions::Property(Property::arbitrary_from(rng, (env, &stats))) }), ), ( diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 7ae82de30..9ea1462e5 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -115,10 +115,7 @@ impl Property { interactions } - Property::DoubleCreateFailure { - create, - queries, - } => { + Property::DoubleCreateFailure { create, queries } => { let table_name = create.table.name.clone(); let assumption = Interaction::Assumption(Assertion { @@ -186,7 +183,11 @@ fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaining { } } -fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaining: &Remaining) -> Property { +fn property_insert_select( + rng: &mut R, + env: &SimulatorEnv, + remaining: &Remaining, +) -> Property { // Get a random table let table = pick(&env.tables, rng); // Pick a random column @@ -218,7 +219,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaini // - [ ] The inserted row will not be updated. (todo: add this constraint once UPDATE is implemented) // - [ ] The table `t` will not be renamed, dropped, or altered. (todo: add this constraint once ALTER or DROP is implemented) for _ in 0..rng.gen_range(0..3) { - let query = Query::arbitrary_from(rng, &(table, remaining)); + let query = Query::arbitrary_from(rng, (table, remaining)); match &query { Query::Delete(Delete { table: t, @@ -244,7 +245,7 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaini // Select the row let select_query = Select { table: table.name.clone(), - predicate: Predicate::arbitrary_from(rng, &(table, &row)), + predicate: Predicate::arbitrary_from(rng, (table, &row)), }; Property::InsertSelect { @@ -254,7 +255,11 @@ fn property_insert_select(rng: &mut R, env: &SimulatorEnv, remaini } } -fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv, remaining: &Remaining) -> Property { +fn property_double_create_failure( + rng: &mut R, + env: &SimulatorEnv, + remaining: &Remaining, +) -> Property { // Get a random table let table = pick(&env.tables, rng); // Create the table @@ -268,7 +273,7 @@ fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv, // - [x] There will be no errors in the middle interactions.(best effort) // - [ ] Table `t` will not be renamed or dropped.(todo: add this constraint once ALTER or DROP is implemented) for _ in 0..rng.gen_range(0..3) { - let query = Query::arbitrary_from(rng, &(table, remaining)); + let query = Query::arbitrary_from(rng, (table, remaining)); match &query { Query::Create(Create { table: t }) => { // There will be no errors in the middle interactions. @@ -288,12 +293,10 @@ fn property_double_create_failure(rng: &mut R, env: &SimulatorEnv, } } - - impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property { fn arbitrary_from( rng: &mut R, - (env, stats): &(&SimulatorEnv, &InteractionStats), + (env, stats): (&SimulatorEnv, &InteractionStats), ) -> Self { let remaining_ = remaining(env, stats); frequency( diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index 0229a4750..8b93fa993 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -17,7 +17,7 @@ impl Arbitrary for Create { } } -impl ArbitraryFrom> for Select { +impl ArbitraryFrom<&Vec
> for Select { fn arbitrary_from(rng: &mut R, tables: &Vec
) -> Self { let table = pick(tables, rng); Self { @@ -27,7 +27,7 @@ impl ArbitraryFrom> for Select { } } -impl ArbitraryFrom> for Select { +impl ArbitraryFrom<&Vec<&Table>> for Select { fn arbitrary_from(rng: &mut R, tables: &Vec<&Table>) -> Self { let table = pick(tables, rng); Self { @@ -37,7 +37,7 @@ impl ArbitraryFrom> for Select { } } -impl ArbitraryFrom
for Insert { +impl ArbitraryFrom<&Table> for Insert { fn arbitrary_from(rng: &mut R, table: &Table) -> Self { let num_rows = rng.gen_range(1..10); let values: Vec> = (0..num_rows) @@ -56,7 +56,7 @@ impl ArbitraryFrom
for Insert { } } -impl ArbitraryFrom
for Delete { +impl ArbitraryFrom<&Table> for Delete { fn arbitrary_from(rng: &mut R, table: &Table) -> Self { Self { table: table.name.clone(), @@ -65,7 +65,7 @@ impl ArbitraryFrom
for Delete { } } -impl ArbitraryFrom
for Query { +impl ArbitraryFrom<&Table> for Query { fn arbitrary_from(rng: &mut R, table: &Table) -> Self { frequency( vec![ @@ -89,7 +89,7 @@ impl ArbitraryFrom
for Query { } impl ArbitraryFrom<(&Table, &Remaining)> for Query { - fn arbitrary_from(rng: &mut R, (table, remaining): &(&Table, &Remaining)) -> Self { + fn arbitrary_from(rng: &mut R, (table, remaining): (&Table, &Remaining)) -> Self { frequency( vec![ ( @@ -98,7 +98,7 @@ impl ArbitraryFrom<(&Table, &Remaining)> for Query { ), ( remaining.read, - Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![*table]))), + Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![table]))), ), ( remaining.write, @@ -118,7 +118,7 @@ struct CompoundPredicate(Predicate); struct SimplePredicate(Predicate); impl ArbitraryFrom<(&Table, bool)> for SimplePredicate { - fn arbitrary_from(rng: &mut R, (table, predicate_value): &(&Table, bool)) -> Self { + fn arbitrary_from(rng: &mut R, (table, predicate_value): (&Table, bool)) -> Self { // Pick a random column let column_index = rng.gen_range(0..table.columns.len()); let column = &table.columns[column_index]; @@ -182,15 +182,15 @@ impl ArbitraryFrom<(&Table, bool)> for SimplePredicate { } impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { - fn arbitrary_from(rng: &mut R, (table, predicate_value): &(&Table, bool)) -> Self { + fn arbitrary_from(rng: &mut R, (table, predicate_value): (&Table, bool)) -> Self { // Decide if you want to create an AND or an OR Self(if rng.gen_bool(0.7) { // An AND for true requires each of its children to be true // An AND for false requires at least one of its children to be false - if *predicate_value { + if predicate_value { Predicate::And( (0..rng.gen_range(0..=3)) - .map(|_| SimplePredicate::arbitrary_from(rng, &(*table, true)).0) + .map(|_| SimplePredicate::arbitrary_from(rng, (table, true)).0) .collect(), ) } else { @@ -209,14 +209,14 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { Predicate::And( booleans .iter() - .map(|b| SimplePredicate::arbitrary_from(rng, &(*table, *b)).0) + .map(|b| SimplePredicate::arbitrary_from(rng, (table, *b)).0) .collect(), ) } } else { // An OR for true requires at least one of its children to be true // An OR for false requires each of its children to be false - if *predicate_value { + if predicate_value { // Create a vector of random booleans let mut booleans = (0..rng.gen_range(0..=3)) .map(|_| rng.gen_bool(0.5)) @@ -230,13 +230,13 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { Predicate::Or( booleans .iter() - .map(|b| SimplePredicate::arbitrary_from(rng, &(*table, *b)).0) + .map(|b| SimplePredicate::arbitrary_from(rng, (table, *b)).0) .collect(), ) } else { Predicate::Or( (0..rng.gen_range(0..=3)) - .map(|_| SimplePredicate::arbitrary_from(rng, &(*table, false)).0) + .map(|_| SimplePredicate::arbitrary_from(rng, (table, false)).0) .collect(), ) } @@ -244,28 +244,28 @@ impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate { } } -impl ArbitraryFrom
for Predicate { +impl ArbitraryFrom<&Table> for Predicate { fn arbitrary_from(rng: &mut R, table: &Table) -> Self { let predicate_value = rng.gen_bool(0.5); - CompoundPredicate::arbitrary_from(rng, &(table, predicate_value)).0 + CompoundPredicate::arbitrary_from(rng, (table, predicate_value)).0 } } impl ArbitraryFrom<(&str, &Value)> for Predicate { - fn arbitrary_from(rng: &mut R, (column_name, value): &(&str, &Value)) -> Self { + fn arbitrary_from(rng: &mut R, (column_name, value): (&str, &Value)) -> Self { one_of( vec![ Box::new(|_| Predicate::Eq(column_name.to_string(), (*value).clone())), Box::new(|rng| { Self::Gt( column_name.to_string(), - GTValue::arbitrary_from(rng, *value).0, + GTValue::arbitrary_from(rng, value).0, ) }), Box::new(|rng| { Self::Lt( column_name.to_string(), - LTValue::arbitrary_from(rng, *value).0, + LTValue::arbitrary_from(rng, value).0, ) }), ], @@ -275,7 +275,7 @@ impl ArbitraryFrom<(&str, &Value)> for Predicate { } /// Produces a predicate that is true for the provided row in the given table -fn produce_true_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) -> Predicate { +fn produce_true_predicate(rng: &mut R, (t, row): (&Table, &Vec)) -> Predicate { // Pick a column let column_index = rng.gen_range(0..t.columns.len()); let column = &t.columns[column_index]; @@ -304,7 +304,7 @@ fn produce_true_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) } /// Produces a predicate that is false for the provided row in the given table -fn produce_false_predicate(rng: &mut R, (t, row): &(&Table, &Vec)) -> Predicate { +fn produce_false_predicate(rng: &mut R, (t, row): (&Table, &Vec)) -> Predicate { // Pick a column let column_index = rng.gen_range(0..t.columns.len()); let column = &t.columns[column_index]; @@ -333,18 +333,18 @@ fn produce_false_predicate(rng: &mut R, (t, row): &(&Table, &Vec) } impl ArbitraryFrom<(&Table, &Vec)> for Predicate { - fn arbitrary_from(rng: &mut R, (t, row): &(&Table, &Vec)) -> Self { + fn arbitrary_from(rng: &mut R, (t, row): (&Table, &Vec)) -> Self { // We want to produce a predicate that is true for the row // We can do this by creating several predicates that // are true, some that are false, combiend them in ways that correspond to the creation of a true predicate // Produce some true and false predicates let mut true_predicates = (1..=rng.gen_range(1..=4)) - .map(|_| produce_true_predicate(rng, &(*t, row))) + .map(|_| produce_true_predicate(rng, (t, row))) .collect::>(); let false_predicates = (0..=rng.gen_range(0..=3)) - .map(|_| produce_false_predicate(rng, &(*t, row))) + .map(|_| produce_false_predicate(rng, (t, row))) .collect::>(); // Start building a top level predicate from a true predicate diff --git a/simulator/generation/table.rs b/simulator/generation/table.rs index b5b898eeb..af5a018f7 100644 --- a/simulator/generation/table.rs +++ b/simulator/generation/table.rs @@ -45,7 +45,7 @@ impl Arbitrary for ColumnType { } } -impl ArbitraryFrom> for Value { +impl ArbitraryFrom<&Vec<&Value>> for Value { fn arbitrary_from(rng: &mut R, values: &Vec<&Self>) -> Self { if values.is_empty() { return Self::Null; @@ -55,7 +55,7 @@ impl ArbitraryFrom> for Value { } } -impl ArbitraryFrom for Value { +impl ArbitraryFrom<&ColumnType> for Value { fn arbitrary_from(rng: &mut R, column_type: &ColumnType) -> Self { match column_type { ColumnType::Integer => Self::Integer(rng.gen_range(i64::MIN..i64::MAX)), @@ -68,7 +68,7 @@ impl ArbitraryFrom for Value { pub(crate) struct LTValue(pub(crate) Value); -impl ArbitraryFrom> for LTValue { +impl ArbitraryFrom<&Vec<&Value>> for LTValue { fn arbitrary_from(rng: &mut R, values: &Vec<&Value>) -> Self { if values.is_empty() { return Self(Value::Null); @@ -79,7 +79,7 @@ impl ArbitraryFrom> for LTValue { } } -impl ArbitraryFrom for LTValue { +impl ArbitraryFrom<&Value> for LTValue { fn arbitrary_from(rng: &mut R, value: &Value) -> Self { match value { Value::Integer(i) => Self(Value::Integer(rng.gen_range(i64::MIN..*i - 1))), @@ -128,7 +128,7 @@ impl ArbitraryFrom for LTValue { pub(crate) struct GTValue(pub(crate) Value); -impl ArbitraryFrom> for GTValue { +impl ArbitraryFrom<&Vec<&Value>> for GTValue { fn arbitrary_from(rng: &mut R, values: &Vec<&Value>) -> Self { if values.is_empty() { return Self(Value::Null); @@ -139,7 +139,7 @@ impl ArbitraryFrom> for GTValue { } } -impl ArbitraryFrom for GTValue { +impl ArbitraryFrom<&Value> for GTValue { fn arbitrary_from(rng: &mut R, value: &Value) -> Self { match value { Value::Integer(i) => Self(Value::Integer(rng.gen_range(*i..i64::MAX))), diff --git a/simulator/main.rs b/simulator/main.rs index 1728744af..db4c7955b 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -2,6 +2,7 @@ use clap::Parser; use core::panic; use generation::plan::{InteractionPlan, InteractionPlanState}; +use generation::ArbitraryFrom; use limbo_core::Database; use rand::prelude::*; use rand_chacha::ChaCha8Rng; diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index 2f89a127f..c97503f65 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -1,5 +1,5 @@ use crate::{ - generation::plan::{Interaction, InteractionPlan, Interactions}, + generation::plan::{InteractionPlan, Interactions}, runner::execution::Execution, }; @@ -26,13 +26,9 @@ impl InteractionPlan { for interaction in plan.plan.iter_mut() { if let Interactions::Property(p) = interaction { match p { - crate::generation::property::Property::InsertSelect { - queries, - .. - } | - crate::generation::property::Property::DoubleCreateFailure { - queries, - .. + crate::generation::property::Property::InsertSelect { queries, .. } + | crate::generation::property::Property::DoubleCreateFailure { + queries, .. } => { queries.clear(); } From c3ea02783d60018dd1737f40794b41359915ad5d Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 15:56:10 +0300 Subject: [PATCH 22/97] - add doc comments to generation traits and functions - remove pick_index from places where it's possible to use pick instead - allow multiple values to be inserted in the insert-select property --- simulator/generation/mod.rs | 22 +++++++++++++++ simulator/generation/property.rs | 46 +++++++++++++++----------------- simulator/generation/table.rs | 23 +++++++++++----- 3 files changed, 60 insertions(+), 31 deletions(-) diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index 73eb77e80..23775bf0d 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -8,14 +8,30 @@ pub mod property; pub mod query; pub mod table; +/// Arbitrary trait for generating random values +/// An implementation of arbitrary is assumed to be a uniform sampling of +/// the possible values of the type, with a bias towards smaller values for +/// practicality. pub trait Arbitrary { fn arbitrary(rng: &mut R) -> Self; } +/// ArbitraryFrom trait for generating random values from a given value +/// ArbitraryFrom allows for constructing relations, where the generated +/// value is dependent on the given value. These relations could be constraints +/// such as generating an integer within an interval, or a value that fits in a table, +/// or a predicate satisfying a given table row. pub trait ArbitraryFrom { fn arbitrary_from(rng: &mut R, t: T) -> Self; } +/// Frequency is a helper function for composing different generators with different frequency +/// of occurences. +/// The type signature for the `N` parameter is a bit complex, but it +/// roughly corresponds to a type that can be summed, compared, subtracted and sampled, which are +/// the operations we require for the implementation. +// todo: switch to a simpler type signature that can accomodate all integer and float types, which +// should be enough for our purposes. pub(crate) fn frequency< 'a, T, @@ -38,6 +54,7 @@ pub(crate) fn frequency< unreachable!() } +/// one_of is a helper function for composing different generators with equal probability of occurence. pub(crate) fn one_of<'a, T, R: rand::Rng>( choices: Vec T + 'a>>, rng: &mut R, @@ -46,15 +63,20 @@ pub(crate) fn one_of<'a, T, R: rand::Rng>( choices[index](rng) } +/// pick is a helper function for uniformly picking a random element from a slice pub(crate) fn pick<'a, T, R: rand::Rng>(choices: &'a [T], rng: &mut R) -> &'a T { let index = rng.gen_range(0..choices.len()); &choices[index] } +/// pick_index is typically used for picking an index from a slice to later refer to the element +/// at that index. pub(crate) fn pick_index(choices: usize, rng: &mut R) -> usize { rng.gen_range(0..choices) } +/// gen_random_text uses `anarchist_readable_name_generator_lib` to generate random +/// readable names for tables, columns, text values etc. fn gen_random_text(rng: &mut T) -> String { let big_text = rng.gen_ratio(1, 1000); if big_text { diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index 9ea1462e5..b00d36c57 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -7,7 +7,7 @@ use crate::{ }; use super::{ - frequency, pick, pick_index, + frequency, pick, plan::{Assertion, Interaction, InteractionStats, ResultSet}, ArbitraryFrom, }; @@ -66,6 +66,9 @@ impl Property { Property::DoubleCreateFailure { .. } => "Double-Create-Failure".to_string(), } } + /// interactions construct a list of interactions, which is an executable representation of the property. + /// the requirement of property -> vec conversion emerges from the need to serialize the property, + /// and `interaction` cannot be serialized directly. pub(crate) fn interactions(&self) -> Vec { match self { Property::InsertSelect { @@ -73,13 +76,16 @@ impl Property { queries, select, } => { - // Check that the row is there - let row = insert - .values - .first() // `.first` is safe, because we know we are inserting a row in the insert select property - .expect("insert query should have at least 1 value") - .clone(); + // Check that the insert query has at least 1 value + assert!( + !insert.values.is_empty(), + "insert query should have at least 1 value" + ); + // Pick a random row within the insert values + let row = pick(&insert.values, &mut rand::thread_rng()).clone(); + + // Assume that the table exists let assumption = Interaction::Assumption(Assertion { message: format!("table {} exists", insert.table), func: Box::new({ @@ -190,26 +196,18 @@ fn property_insert_select( ) -> Property { // Get a random table let table = pick(&env.tables, rng); - // Pick a random column - let column_index = pick_index(table.columns.len(), rng); - let column = &table.columns[column_index].clone(); - // Generate a random value of the column type - let value = Value::arbitrary_from(rng, &column.column_type); - // Create a whole new row - let mut row = Vec::new(); - for (i, column) in table.columns.iter().enumerate() { - if i == column_index { - row.push(value.clone()); - } else { - let value = Value::arbitrary_from(rng, &column.column_type); - row.push(value); - } - } + // Generate rows to insert + let rows = (0..rng.gen_range(1..=5)) + .map(|_| Vec::::arbitrary_from(rng, table)) + .collect::>(); - // Insert the row + // Pick a random row to select + let row = pick(&rows, rng).clone(); + + // Insert the rows let insert_query = Insert { table: table.name.clone(), - values: vec![row.clone()], + values: rows, }; // Create random queries respecting the constraints diff --git a/simulator/generation/table.rs b/simulator/generation/table.rs index af5a018f7..8e892e255 100644 --- a/simulator/generation/table.rs +++ b/simulator/generation/table.rs @@ -1,8 +1,6 @@ use rand::Rng; -use crate::generation::{ - gen_random_text, pick, pick_index, readable_name_custom, Arbitrary, ArbitraryFrom, -}; +use crate::generation::{gen_random_text, pick, readable_name_custom, Arbitrary, ArbitraryFrom}; use crate::model::table::{Column, ColumnType, Name, Table, Value}; impl Arbitrary for Name { @@ -45,6 +43,17 @@ impl Arbitrary for ColumnType { } } +impl ArbitraryFrom<&Table> for Vec { + fn arbitrary_from(rng: &mut R, table: &Table) -> Self { + let mut row = Vec::new(); + for column in table.columns.iter() { + let value = Value::arbitrary_from(rng, &column.column_type); + row.push(value); + } + row + } +} + impl ArbitraryFrom<&Vec<&Value>> for Value { fn arbitrary_from(rng: &mut R, values: &Vec<&Self>) -> Self { if values.is_empty() { @@ -74,8 +83,8 @@ impl ArbitraryFrom<&Vec<&Value>> for LTValue { return Self(Value::Null); } - let index = pick_index(values.len(), rng); - Self::arbitrary_from(rng, values[index]) + let value = pick(values, rng); + Self::arbitrary_from(rng, *value) } } @@ -134,8 +143,8 @@ impl ArbitraryFrom<&Vec<&Value>> for GTValue { return Self(Value::Null); } - let index = pick_index(values.len(), rng); - Self::arbitrary_from(rng, values[index]) + let value = pick(values, rng); + Self::arbitrary_from(rng, *value) } } From fb937eff7bad76030a99a59c6c3d40cdfe0a903c Mon Sep 17 00:00:00 2001 From: alpaylan Date: Mon, 13 Jan 2025 17:26:23 +0300 Subject: [PATCH 23/97] fix non-determinism bug arising from a call to `thread_rng` while picking which row to check existence for in the result of the select query --- simulator/generation/property.rs | 11 ++++++++--- simulator/main.rs | 5 +++-- simulator/runner/execution.rs | 3 ++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index b00d36c57..cc02e6233 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -7,7 +7,7 @@ use crate::{ }; use super::{ - frequency, pick, + frequency, pick, pick_index, plan::{Assertion, Interaction, InteractionStats, ResultSet}, ArbitraryFrom, }; @@ -34,6 +34,8 @@ pub(crate) enum Property { InsertSelect { /// The insert query insert: Insert, + /// Selected row index + row_index: usize, /// Additional interactions in the middle of the property queries: Vec, /// The select query @@ -73,6 +75,7 @@ impl Property { match self { Property::InsertSelect { insert, + row_index, queries, select, } => { @@ -83,7 +86,7 @@ impl Property { ); // Pick a random row within the insert values - let row = pick(&insert.values, &mut rand::thread_rng()).clone(); + let row = insert.values[*row_index].clone(); // Assume that the table exists let assumption = Interaction::Assumption(Assertion { @@ -202,7 +205,8 @@ fn property_insert_select( .collect::>(); // Pick a random row to select - let row = pick(&rows, rng).clone(); + let row_index = pick_index(rows.len(), rng).clone(); + let row = rows[row_index].clone(); // Insert the rows let insert_query = Insert { @@ -248,6 +252,7 @@ fn property_insert_select( Property::InsertSelect { insert: insert_query, + row_index, queries, select: select_query, } diff --git a/simulator/main.rs b/simulator/main.rs index db4c7955b..39a6096c5 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -218,7 +218,7 @@ fn main() -> Result<(), String> { last_execution, ); - match (shrunk, &result) { + match (&shrunk, &result) { ( SandboxedResult::Panicked { error: e1, .. }, SandboxedResult::Panicked { error: e2, .. }, @@ -227,7 +227,7 @@ fn main() -> Result<(), String> { SandboxedResult::FoundBug { error: e1, .. }, SandboxedResult::FoundBug { error: e2, .. }, ) => { - if &e1 != e2 { + if e1 != e2 { log::error!( "shrinking failed, the error was not properly reproduced" ); @@ -291,6 +291,7 @@ fn revert_db_and_plan_files(output_dir: &Path) { std::fs::rename(&new_plan_path, &old_plan_path).unwrap(); } +#[derive(Debug)] enum SandboxedResult { Panicked { error: String, diff --git a/simulator/runner/execution.rs b/simulator/runner/execution.rs index 064542805..3ac44e894 100644 --- a/simulator/runner/execution.rs +++ b/simulator/runner/execution.rs @@ -9,7 +9,7 @@ use crate::generation::{ use super::env::{SimConnection, SimulatorEnv}; -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy)] pub(crate) struct Execution { pub(crate) connection_index: usize, pub(crate) interaction_index: usize, @@ -30,6 +30,7 @@ impl Execution { } } +#[derive(Debug)] pub(crate) struct ExecutionHistory { pub(crate) history: Vec, } From d4de451d459f7df2d9f87ad848e4eafe7e1227f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sat, 11 Jan 2025 16:53:55 +0100 Subject: [PATCH 24/97] core: enable rustix/io_uring with io_uring feature --- core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index fc1f88fe2..c0c12a0a3 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -22,7 +22,7 @@ json = [ "dep:pest_derive", ] uuid = ["dep:uuid"] -io_uring = ["dep:io-uring"] +io_uring = ["dep:io-uring", "rustix/io_uring"] [target.'cfg(target_os = "linux")'.dependencies] io-uring = { version = "0.6.1", optional = true } From 7808665c924d1ba79247290cbded517f92c39b53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sat, 11 Jan 2025 16:56:18 +0100 Subject: [PATCH 25/97] core: make MAX_IOVECS u32 instead of usize, to match the type expected by io_uring --- core/io/io_uring.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index 3278d61b9..a499320a4 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -10,7 +10,7 @@ use std::os::unix::io::AsRawFd; use std::rc::Rc; use thiserror::Error; -const MAX_IOVECS: usize = 128; +const MAX_IOVECS: u32 = 128; const SQPOLL_IDLE: u32 = 1000; #[derive(Debug, Error)] @@ -44,7 +44,7 @@ struct WrappedIOUring { struct InnerUringIO { ring: WrappedIOUring, - iovecs: [iovec; MAX_IOVECS], + iovecs: [iovec; MAX_IOVECS as usize], next_iovec: usize, } @@ -52,10 +52,10 @@ impl UringIO { pub fn new() -> Result { let ring = match io_uring::IoUring::builder() .setup_sqpoll(SQPOLL_IDLE) - .build(MAX_IOVECS as u32) + .build(MAX_IOVECS) { Ok(ring) => ring, - Err(_) => io_uring::IoUring::new(MAX_IOVECS as u32)?, + Err(_) => io_uring::IoUring::new(MAX_IOVECS)?, }; let inner = InnerUringIO { ring: WrappedIOUring { @@ -67,7 +67,7 @@ impl UringIO { iovecs: [iovec { iov_base: std::ptr::null_mut(), iov_len: 0, - }; MAX_IOVECS], + }; MAX_IOVECS as usize], next_iovec: 0, }; debug!("Using IO backend 'io-uring'"); @@ -82,7 +82,7 @@ impl InnerUringIO { let iovec = &mut self.iovecs[self.next_iovec]; iovec.iov_base = buf as *mut std::ffi::c_void; iovec.iov_len = len; - self.next_iovec = (self.next_iovec + 1) % MAX_IOVECS; + self.next_iovec = (self.next_iovec + 1) % MAX_IOVECS as usize; iovec } } From 7b5e5efd14544abe0a81cee69cb09fa128e2f3e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sat, 11 Jan 2025 16:58:43 +0100 Subject: [PATCH 26/97] core/io/unix: replace libc calls and types with their rustix counterparts --- core/io/unix.rs | 82 +++++++++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/core/io/unix.rs b/core/io/unix.rs index db0e85ab1..e8390dec8 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -3,15 +3,17 @@ use crate::io::common; use crate::Result; use super::{Completion, File, OpenFlags, IO}; -use libc::{c_short, fcntl, flock, F_SETLK}; use log::{debug, trace}; use polling::{Event, Events, Poller}; -use rustix::fd::{AsFd, AsRawFd}; -use rustix::fs::OpenOptionsExt; -use rustix::io::Errno; +use rustix::{ + fd::{AsFd, AsRawFd}, + fs, + fs::{FlockOperation, OpenOptionsExt}, + io::Errno, +}; use std::cell::RefCell; use std::collections::HashMap; -use std::io::{Read, Seek, Write}; +use std::io::{ErrorKind, Read, Seek, Write}; use std::rc::Rc; pub struct UnixIO { @@ -136,55 +138,41 @@ pub struct UnixFile { impl File for UnixFile { fn lock_file(&self, exclusive: bool) -> Result<()> { - let fd = self.file.borrow().as_raw_fd(); - let flock = flock { - l_type: if exclusive { - libc::F_WRLCK as c_short - } else { - libc::F_RDLCK as c_short - }, - l_whence: libc::SEEK_SET as c_short, - l_start: 0, - l_len: 0, // Lock entire file - l_pid: 0, - }; - + let fd = self.file.borrow(); + let fd = fd.as_fd(); // F_SETLK is a non-blocking lock. The lock will be released when the file is closed // or the process exits or after an explicit unlock. - let lock_result = unsafe { fcntl(fd, F_SETLK, &flock) }; - if lock_result == -1 { - let err = std::io::Error::last_os_error(); - if err.kind() == std::io::ErrorKind::WouldBlock { - return Err(LimboError::LockingError( - "Failed locking file. File is locked by another process".to_string(), - )); + fs::fcntl_lock( + fd, + if exclusive { + FlockOperation::LockExclusive } else { - return Err(LimboError::LockingError(format!( - "Failed locking file, {}", - err - ))); - } - } + FlockOperation::LockShared + }, + ) + .map_err(|e| { + let io_error = std::io::Error::from(e); + let message = match io_error.kind() { + ErrorKind::WouldBlock => { + "Failed locking file. File is locked by another process".to_string() + } + _ => format!("Failed locking file, {}", io_error), + }; + LimboError::LockingError(message) + })?; + Ok(()) } fn unlock_file(&self) -> Result<()> { - let fd = self.file.borrow().as_raw_fd(); - let flock = flock { - l_type: libc::F_UNLCK as c_short, - l_whence: libc::SEEK_SET as c_short, - l_start: 0, - l_len: 0, - l_pid: 0, - }; - - let unlock_result = unsafe { fcntl(fd, F_SETLK, &flock) }; - if unlock_result == -1 { - return Err(LimboError::LockingError(format!( + let fd = self.file.borrow(); + let fd = fd.as_fd(); + fs::fcntl_lock(fd, FlockOperation::Unlock).map_err(|e| { + LimboError::LockingError(format!( "Failed to release file lock: {}", - std::io::Error::last_os_error() - ))); - } + std::io::Error::from(e) + )) + })?; Ok(()) } @@ -263,7 +251,7 @@ impl File for UnixFile { fn sync(&self, c: Rc) -> Result<()> { let file = self.file.borrow(); - let result = rustix::fs::fsync(file.as_fd()); + let result = fs::fsync(file.as_fd()); match result { std::result::Result::Ok(()) => { trace!("fsync"); From b1e8f2da735cd6addee02da169dfaef15e10e7d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sat, 11 Jan 2025 17:00:05 +0100 Subject: [PATCH 27/97] core/io/unix: minor formatting --- core/io/unix.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/core/io/unix.rs b/core/io/unix.rs index e8390dec8..035c9f29d 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -88,8 +88,8 @@ impl IO for UnixIO { } } }; - match result { - std::result::Result::Ok(n) => { + return match result { + Ok(n) => { match &cf { CompletionCallback::Read(_, ref c, _) => { c.complete(0); @@ -98,12 +98,10 @@ impl IO for UnixIO { c.complete(n as i32); } } - return Ok(()); + Ok(()) } - Err(e) => { - return Err(e.into()); - } - } + Err(e) => Err(e.into()), + }; } } Ok(()) @@ -132,7 +130,7 @@ enum CompletionCallback { pub struct UnixFile { file: Rc>, - poller: Rc>, + poller: Rc>, callbacks: Rc>>, } @@ -187,7 +185,7 @@ impl File for UnixFile { rustix::io::pread(file.as_fd(), buf.as_mut_slice(), pos as u64) }; match result { - std::result::Result::Ok(n) => { + Ok(n) => { trace!("pread n: {}", n); // Read succeeded immediately c.complete(0); @@ -224,7 +222,7 @@ impl File for UnixFile { rustix::io::pwrite(file.as_fd(), buf.as_slice(), pos as u64) }; match result { - std::result::Result::Ok(n) => { + Ok(n) => { trace!("pwrite n: {}", n); // Read succeeded immediately c.complete(n as i32); @@ -253,7 +251,7 @@ impl File for UnixFile { let file = self.file.borrow(); let result = fs::fsync(file.as_fd()); match result { - std::result::Result::Ok(()) => { + Ok(()) => { trace!("fsync"); c.complete(0); Ok(()) @@ -264,7 +262,7 @@ impl File for UnixFile { fn size(&self) -> Result { let file = self.file.borrow(); - Ok(file.metadata().unwrap().len()) + Ok(file.metadata()?.len()) } } From b146f5d4cba52648942d5b4d03b66ae649a50371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sat, 11 Jan 2025 17:06:11 +0100 Subject: [PATCH 28/97] core/io/io_uring: replace nix and libc calls with their rustix counterparts. core: remove dependency on nix. We keep depending on libc, though, because crate io_uring requires libc's iovec. --- core/io/io_uring.rs | 81 ++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 46 deletions(-) diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index a499320a4..4ff94519f 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -1,11 +1,13 @@ use super::{common, Completion, File, OpenFlags, IO}; use crate::{LimboError, Result}; -use libc::{c_short, fcntl, flock, iovec, F_SETLK}; use log::{debug, trace}; -use nix::fcntl::{FcntlArg, OFlag}; +use rustix::fs::{self, FlockOperation, OFlags}; +use rustix::io_uring::iovec; use std::cell::RefCell; use std::collections::HashMap; use std::fmt; +use std::io::ErrorKind; +use std::os::fd::AsFd; use std::os::unix::io::AsRawFd; use std::rc::Rc; use thiserror::Error; @@ -136,12 +138,12 @@ impl IO for UringIO { .open(path)?; // Let's attempt to enable direct I/O. Not all filesystems support it // so ignore any errors. - let fd = file.as_raw_fd(); + let fd = file.as_fd(); if direct { - match nix::fcntl::fcntl(fd, FcntlArg::F_SETFL(OFlag::O_DIRECT)) { - Ok(_) => {}, + match fs::fcntl_setfl(fd, OFlags::DIRECT) { + Ok(_) => {} Err(error) => debug!("Error {error:?} returned when setting O_DIRECT flag to read file. The performance of the system may be affected"), - }; + } } let uring_file = Rc::new(UringFile { io: self.inner.clone(), @@ -199,52 +201,39 @@ pub struct UringFile { impl File for UringFile { fn lock_file(&self, exclusive: bool) -> Result<()> { - let fd = self.file.as_raw_fd(); - let flock = flock { - l_type: if exclusive { - libc::F_WRLCK as c_short - } else { - libc::F_RDLCK as c_short - }, - l_whence: libc::SEEK_SET as c_short, - l_start: 0, - l_len: 0, // Lock entire file - l_pid: 0, - }; - + let fd = self.file.as_fd(); // F_SETLK is a non-blocking lock. The lock will be released when the file is closed // or the process exits or after an explicit unlock. - let lock_result = unsafe { fcntl(fd, F_SETLK, &flock) }; - if lock_result == -1 { - let err = std::io::Error::last_os_error(); - if err.kind() == std::io::ErrorKind::WouldBlock { - return Err(LimboError::LockingError( - "File is locked by another process".into(), - )); + fs::fcntl_lock( + fd, + if exclusive { + FlockOperation::LockExclusive } else { - return Err(LimboError::IOError(err)); - } - } + FlockOperation::LockShared + }, + ) + .map_err(|e| { + let io_error = std::io::Error::from(e); + let message = match io_error.kind() { + ErrorKind::WouldBlock => { + "Failed locking file. File is locked by another process".to_string() + } + _ => format!("Failed locking file, {}", io_error), + }; + LimboError::LockingError(message) + })?; + Ok(()) } fn unlock_file(&self) -> Result<()> { - let fd = self.file.as_raw_fd(); - let flock = flock { - l_type: libc::F_UNLCK as c_short, - l_whence: libc::SEEK_SET as c_short, - l_start: 0, - l_len: 0, - l_pid: 0, - }; - - let unlock_result = unsafe { fcntl(fd, F_SETLK, &flock) }; - if unlock_result == -1 { - return Err(LimboError::LockingError(format!( + let fd = self.file.as_fd(); + fs::fcntl_lock(fd, FlockOperation::Unlock).map_err(|e| { + LimboError::LockingError(format!( "Failed to release file lock: {}", - std::io::Error::last_os_error() - ))); - } + std::io::Error::from(e) + )) + })?; Ok(()) } @@ -261,7 +250,7 @@ impl File for UringFile { let len = buf.len(); let buf = buf.as_mut_ptr(); let iovec = io.get_iovec(buf, len); - io_uring::opcode::Readv::new(fd, iovec, 1) + io_uring::opcode::Readv::new(fd, iovec as *const iovec as *const libc::iovec, 1) .offset(pos as u64) .build() .user_data(io.ring.get_key()) @@ -282,7 +271,7 @@ impl File for UringFile { let buf = buffer.borrow(); trace!("pwrite(pos = {}, length = {})", pos, buf.len()); let iovec = io.get_iovec(buf.as_ptr(), buf.len()); - io_uring::opcode::Writev::new(fd, iovec, 1) + io_uring::opcode::Writev::new(fd, iovec as *const iovec as *const libc::iovec, 1) .offset(pos as u64) .build() .user_data(io.ring.get_key()) From 5e9cb58f04436d24c3bcbe5f0d0e5c3007f8d657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sun, 12 Jan 2025 14:11:37 +0100 Subject: [PATCH 29/97] core/io/io_uring: remove unnecessary path prefix for log macros, and replace one unwrap with ? --- core/io/io_uring.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index 4ff94519f..14a6bd83c 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -91,7 +91,7 @@ impl InnerUringIO { impl WrappedIOUring { fn submit_entry(&mut self, entry: &io_uring::squeue::Entry, c: Rc) { - log::trace!("submit_entry({:?})", entry); + trace!("submit_entry({:?})", entry); self.pending.insert(entry.get_user_data(), c); unsafe { self.ring @@ -111,7 +111,7 @@ impl WrappedIOUring { // NOTE: This works because CompletionQueue's next function pops the head of the queue. This is not normal behaviour of iterators let entry = self.ring.completion().next(); if entry.is_some() { - log::trace!("get_completion({:?})", entry); + trace!("get_completion({:?})", entry); // consumed an entry from completion queue, update pending_ops self.pending_ops -= 1; } @@ -292,7 +292,7 @@ impl File for UringFile { } fn size(&self) -> Result { - Ok(self.file.metadata().unwrap().len()) + Ok(self.file.metadata()?.len()) } } From 2f90a065337698c5b176f2cd97ca92620823ba19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Mon, 13 Jan 2025 21:03:05 +0100 Subject: [PATCH 30/97] core/io/unix: replace O_NONBLOCK flag from libc with equivalent from rustix --- core/io/unix.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/io/unix.rs b/core/io/unix.rs index 035c9f29d..c021c4566 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -7,8 +7,7 @@ use log::{debug, trace}; use polling::{Event, Events, Poller}; use rustix::{ fd::{AsFd, AsRawFd}, - fs, - fs::{FlockOperation, OpenOptionsExt}, + fs::{self, FlockOperation, OFlags, OpenOptionsExt}, io::Errno, }; use std::cell::RefCell; @@ -38,7 +37,7 @@ impl IO for UnixIO { trace!("open_file(path = {})", path); let file = std::fs::File::options() .read(true) - .custom_flags(libc::O_NONBLOCK) + .custom_flags(OFlags::NONBLOCK.bits() as i32) .write(true) .create(matches!(flags, OpenFlags::Create)) .open(path)?; From cca3846f950c262c6a92c28980df155d94ef658b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Mon, 13 Jan 2025 21:15:36 +0100 Subject: [PATCH 31/97] core: Previous commits didn't actually remove nix as dependency, so do that here --- Cargo.lock | 1 - core/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 915456422..621727df3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1217,7 +1217,6 @@ dependencies = [ "miette", "mimalloc", "mockall", - "nix 0.29.0", "pest", "pest_derive", "polling", diff --git a/core/Cargo.toml b/core/Cargo.toml index c0c12a0a3..0daa58c0d 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -40,7 +40,6 @@ fallible-iterator = "0.3.0" hex = "0.4.3" libc = "0.2.155" log = "0.4.20" -nix = { version = "0.29.0", features = ["fs"] } sieve-cache = "0.1.4" sqlite3-parser = { path = "../vendored/sqlite3-parser" } thiserror = "1.0.61" From eff5de50c598a5679745d1285256a0ece1d65bf9 Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Mon, 13 Jan 2025 19:53:06 -0300 Subject: [PATCH 32/97] refactor: make `translate_*` functions accept ProgramBuilder simplifies function signatures and allows attaching more context to ProgramStatus on `translate::translate`, useful for query parameters. --- core/translate/delete.rs | 13 +++--- core/translate/emitter.rs | 57 +++++++++++-------------- core/translate/insert.rs | 25 ++++------- core/translate/mod.rs | 89 ++++++++++++++++----------------------- core/translate/select.rs | 15 +++---- 5 files changed, 81 insertions(+), 118 deletions(-) diff --git a/core/translate/delete.rs b/core/translate/delete.rs index dcaed53e6..373a74024 100644 --- a/core/translate/delete.rs +++ b/core/translate/delete.rs @@ -3,26 +3,23 @@ use crate::translate::emitter::emit_program; use crate::translate::optimizer::optimize_plan; use crate::translate::plan::{DeletePlan, Plan, SourceOperator}; use crate::translate::planner::{parse_limit, parse_where}; -use crate::{schema::Schema, storage::sqlite3_ondisk::DatabaseHeader, vdbe::Program}; -use crate::{Connection, Result, SymbolTable}; +use crate::vdbe::builder::ProgramBuilder; +use crate::{schema::Schema, Result, SymbolTable}; use sqlite3_parser::ast::{Expr, Limit, QualifiedName}; -use std::rc::Weak; -use std::{cell::RefCell, rc::Rc}; use super::plan::{TableReference, TableReferenceType}; pub fn translate_delete( + program: &mut ProgramBuilder, schema: &Schema, tbl_name: &QualifiedName, where_clause: Option, limit: Option>, - database_header: Rc>, - connection: Weak, syms: &SymbolTable, -) -> Result { +) -> Result<()> { let mut delete_plan = prepare_delete_plan(schema, tbl_name, where_clause, limit)?; optimize_plan(&mut delete_plan)?; - emit_program(database_header, delete_plan, connection, syms) + emit_program(program, delete_plan, syms) } pub fn prepare_delete_plan( diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index 8fdff8cd1..cc4df6d8c 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -1,19 +1,16 @@ // This module contains code for emitting bytecode instructions for SQL query execution. // It handles translating high-level SQL operations into low-level bytecode that can be executed by the virtual machine. -use std::cell::RefCell; use std::collections::HashMap; -use std::rc::{Rc, Weak}; use sqlite3_parser::ast::{self}; use crate::function::Func; -use crate::storage::sqlite3_ondisk::DatabaseHeader; use crate::translate::plan::{DeletePlan, Plan, Search}; use crate::util::exprs_are_equivalent; use crate::vdbe::builder::ProgramBuilder; -use crate::vdbe::{insn::Insn, BranchOffset, Program}; -use crate::{Connection, Result, SymbolTable}; +use crate::vdbe::{insn::Insn, BranchOffset}; +use crate::{Result, SymbolTable}; use super::aggregation::emit_ungrouped_aggregation; use super::group_by::{emit_group_by, init_group_by, GroupByMetadata}; @@ -99,9 +96,9 @@ pub enum OperationMode { /// Initialize the program with basic setup and return initial metadata and labels fn prologue<'a>( + program: &mut ProgramBuilder, syms: &'a SymbolTable, -) -> Result<(ProgramBuilder, TranslateCtx<'a>, BranchOffset, BranchOffset)> { - let mut program = ProgramBuilder::new(); +) -> Result<(TranslateCtx<'a>, BranchOffset, BranchOffset)> { let init_label = program.allocate_label(); program.emit_insn(Insn::Init { @@ -124,7 +121,7 @@ fn prologue<'a>( resolver: Resolver::new(syms), }; - Ok((program, t_ctx, init_label, start_offset)) + Ok((t_ctx, init_label, start_offset)) } /// Clean up and finalize the program, resolving any remaining labels @@ -154,40 +151,37 @@ fn epilogue( /// Main entry point for emitting bytecode for a SQL query /// Takes a query plan and generates the corresponding bytecode program pub fn emit_program( - database_header: Rc>, + program: &mut ProgramBuilder, plan: Plan, - connection: Weak, syms: &SymbolTable, -) -> Result { +) -> Result<()> { match plan { - Plan::Select(plan) => emit_program_for_select(database_header, plan, connection, syms), - Plan::Delete(plan) => emit_program_for_delete(database_header, plan, connection, syms), + Plan::Select(plan) => emit_program_for_select(program, plan, syms), + Plan::Delete(plan) => emit_program_for_delete(program, plan, syms), } } fn emit_program_for_select( - database_header: Rc>, + program: &mut ProgramBuilder, mut plan: SelectPlan, - connection: Weak, syms: &SymbolTable, -) -> Result { - let (mut program, mut t_ctx, init_label, start_offset) = prologue(syms)?; +) -> Result<()> { + let (mut t_ctx, init_label, start_offset) = prologue(program, syms)?; // Trivial exit on LIMIT 0 if let Some(limit) = plan.limit { if limit == 0 { - epilogue(&mut program, init_label, start_offset)?; - return Ok(program.build(database_header, connection)); + epilogue(program, init_label, start_offset)?; } } // Emit main parts of query - emit_query(&mut program, &mut plan, &mut t_ctx)?; + emit_query(program, &mut plan, &mut t_ctx)?; // Finalize program - epilogue(&mut program, init_label, start_offset)?; + epilogue(program, init_label, start_offset)?; - Ok(program.build(database_header, connection)) + Ok(()) } pub fn emit_query<'a>( @@ -263,12 +257,11 @@ pub fn emit_query<'a>( } fn emit_program_for_delete( - database_header: Rc>, + program: &mut ProgramBuilder, mut plan: DeletePlan, - connection: Weak, syms: &SymbolTable, -) -> Result { - let (mut program, mut t_ctx, init_label, start_offset) = prologue(syms)?; +) -> Result<()> { + let (mut t_ctx, init_label, start_offset) = prologue(program, syms)?; // No rows will be read from source table loops if there is a constant false condition eg. WHERE 0 let after_main_loop_label = program.allocate_label(); @@ -280,7 +273,7 @@ fn emit_program_for_delete( // Initialize cursors and other resources needed for query execution init_loop( - &mut program, + program, &mut t_ctx, &plan.source, &OperationMode::DELETE, @@ -288,23 +281,23 @@ fn emit_program_for_delete( // Set up main query execution loop open_loop( - &mut program, + program, &mut t_ctx, &mut plan.source, &plan.referenced_tables, )?; - emit_delete_insns(&mut program, &mut t_ctx, &plan.source, &plan.limit)?; + emit_delete_insns(program, &mut t_ctx, &plan.source, &plan.limit)?; // Clean up and close the main execution loop - close_loop(&mut program, &mut t_ctx, &plan.source)?; + close_loop(program, &mut t_ctx, &plan.source)?; program.resolve_label(after_main_loop_label, program.offset()); // Finalize program - epilogue(&mut program, init_label, start_offset)?; + epilogue(program, init_label, start_offset)?; - Ok(program.build(database_header, connection)) + Ok(()) } fn emit_delete_insns<'a>( diff --git a/core/translate/insert.rs b/core/translate/insert.rs index 2dec74248..1b1669ddb 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -1,5 +1,4 @@ -use std::rc::Weak; -use std::{cell::RefCell, ops::Deref, rc::Rc}; +use std::ops::Deref; use sqlite3_parser::ast::{ DistinctNames, Expr, InsertBody, QualifiedName, ResolveType, ResultColumn, With, @@ -11,21 +10,17 @@ use crate::util::normalize_ident; use crate::vdbe::BranchOffset; use crate::{ schema::{Column, Schema}, - storage::sqlite3_ondisk::DatabaseHeader, translate::expr::translate_expr, - vdbe::{ - builder::{CursorType, ProgramBuilder}, - insn::Insn, - Program, - }, + vdbe::{builder::{CursorType, ProgramBuilder}, insn::Insn}, SymbolTable, }; -use crate::{Connection, Result}; +use crate::Result; use super::emitter::Resolver; #[allow(clippy::too_many_arguments)] pub fn translate_insert( + program: &mut ProgramBuilder, schema: &Schema, with: &Option, on_conflict: &Option, @@ -33,17 +28,14 @@ pub fn translate_insert( columns: &Option, body: &InsertBody, _returning: &Option>, - database_header: Rc>, - connection: Weak, syms: &SymbolTable, -) -> Result { +) -> Result<()> { if with.is_some() { crate::bail_parse_error!("WITH clause is not supported"); } if on_conflict.is_some() { crate::bail_parse_error!("ON CONFLICT clause is not supported"); } - let mut program = ProgramBuilder::new(); let resolver = Resolver::new(syms); let init_label = program.allocate_label(); program.emit_insn(Insn::Init { @@ -118,7 +110,7 @@ pub fn translate_insert( for value in values { populate_column_registers( - &mut program, + program, value, &column_mappings, column_registers_start, @@ -157,7 +149,7 @@ pub fn translate_insert( program.emit_insn(Insn::OpenWriteAwait {}); populate_column_registers( - &mut program, + program, &values[0], &column_mappings, column_registers_start, @@ -262,7 +254,8 @@ pub fn translate_insert( program.emit_insn(Insn::Goto { target_pc: start_offset, }); - Ok(program.build(database_header, connection)) + + Ok(()) } #[derive(Debug)] diff --git a/core/translate/mod.rs b/core/translate/mod.rs index fdbbc47e0..9a990eb36 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -48,6 +48,7 @@ pub fn translate( connection: Weak, syms: &SymbolTable, ) -> Result { + let mut program = ProgramBuilder::new(); match stmt { ast::Stmt::AlterTable(_, _) => bail_parse_error!("ALTER TABLE not supported yet"), ast::Stmt::Analyze(_) => bail_parse_error!("ANALYZE not supported yet"), @@ -64,14 +65,8 @@ pub fn translate( if temporary { bail_parse_error!("TEMPORARY table not supported yet"); } - translate_create_table( - tbl_name, - body, - if_not_exists, - database_header, - connection, - schema, - ) + + translate_create_table(&mut program, tbl_name, body, if_not_exists, schema)?; } ast::Stmt::CreateTrigger { .. } => bail_parse_error!("CREATE TRIGGER not supported yet"), ast::Stmt::CreateView { .. } => bail_parse_error!("CREATE VIEW not supported yet"), @@ -83,29 +78,23 @@ pub fn translate( where_clause, limit, .. - } => translate_delete( - schema, - &tbl_name, - where_clause, - limit, - database_header, - connection, - syms, - ), + } => { + translate_delete(&mut program, schema, &tbl_name, where_clause, limit, syms)?; + } ast::Stmt::Detach(_) => bail_parse_error!("DETACH not supported yet"), ast::Stmt::DropIndex { .. } => bail_parse_error!("DROP INDEX not supported yet"), ast::Stmt::DropTable { .. } => bail_parse_error!("DROP TABLE not supported yet"), ast::Stmt::DropTrigger { .. } => bail_parse_error!("DROP TRIGGER not supported yet"), ast::Stmt::DropView { .. } => bail_parse_error!("DROP VIEW not supported yet"), ast::Stmt::Pragma(name, body) => { - translate_pragma(&name, body, database_header, pager, connection) + translate_pragma(&mut program, &name, body, database_header.clone(), pager)?; } ast::Stmt::Reindex { .. } => bail_parse_error!("REINDEX not supported yet"), ast::Stmt::Release(_) => bail_parse_error!("RELEASE not supported yet"), ast::Stmt::Rollback { .. } => bail_parse_error!("ROLLBACK not supported yet"), ast::Stmt::Savepoint(_) => bail_parse_error!("SAVEPOINT not supported yet"), ast::Stmt::Select(select) => { - translate_select(schema, *select, database_header, connection, syms) + translate_select(&mut program, schema, *select, syms)?; } ast::Stmt::Update { .. } => bail_parse_error!("UPDATE not supported yet"), ast::Stmt::Vacuum(_, _) => bail_parse_error!("VACUUM not supported yet"), @@ -116,19 +105,21 @@ pub fn translate( columns, body, returning, - } => translate_insert( - schema, - &with, - &or_conflict, - &tbl_name, - &columns, - &body, - &returning, - database_header, - connection, - syms, - ), + } => { + translate_insert( + &mut program, + schema, + &with, + &or_conflict, + &tbl_name, + &columns, + &body, + &returning, + syms, + )?; + } } + Ok(program.build(database_header, connection)) } /* Example: @@ -378,14 +369,12 @@ fn check_automatic_pk_index_required( } fn translate_create_table( + program: &mut ProgramBuilder, tbl_name: ast::QualifiedName, body: ast::CreateTableBody, if_not_exists: bool, - database_header: Rc>, - connection: Weak, schema: &Schema, -) -> Result { - let mut program = ProgramBuilder::new(); +) -> Result<()> { if schema.get_table(tbl_name.name.0.as_str()).is_some() { if if_not_exists { let init_label = program.allocate_label(); @@ -403,7 +392,8 @@ fn translate_create_table( program.emit_insn(Insn::Goto { target_pc: start_offset, }); - return Ok(program.build(database_header, connection)); + + return Ok(()); } bail_parse_error!("Table {} already exists", tbl_name); } @@ -453,7 +443,7 @@ fn translate_create_table( // https://github.com/sqlite/sqlite/blob/95f6df5b8d55e67d1e34d2bff217305a2f21b1fb/src/build.c#L2856-L2871 // https://github.com/sqlite/sqlite/blob/95f6df5b8d55e67d1e34d2bff217305a2f21b1fb/src/build.c#L1334C5-L1336C65 - let index_root_reg = check_automatic_pk_index_required(&body, &mut program, &tbl_name.name.0)?; + let index_root_reg = check_automatic_pk_index_required(&body, program, &tbl_name.name.0)?; if let Some(index_root_reg) = index_root_reg { program.emit_insn(Insn::CreateBtree { db: 0, @@ -476,7 +466,7 @@ fn translate_create_table( // Add the table entry to sqlite_schema emit_schema_entry( - &mut program, + program, sqlite_schema_cursor_id, SchemaEntryType::Table, &tbl_name.name.0, @@ -492,7 +482,7 @@ fn translate_create_table( PRIMARY_KEY_AUTOMATIC_INDEX_NAME_PREFIX, tbl_name.name.0 ); emit_schema_entry( - &mut program, + program, sqlite_schema_cursor_id, SchemaEntryType::Index, &index_name, @@ -523,7 +513,8 @@ fn translate_create_table( program.emit_insn(Insn::Goto { target_pc: start_offset, }); - Ok(program.build(database_header, connection)) + + Ok(()) } enum PrimaryKeyDefinitionType<'a> { @@ -532,13 +523,12 @@ enum PrimaryKeyDefinitionType<'a> { } fn translate_pragma( + program: &mut ProgramBuilder, name: &ast::QualifiedName, body: Option, database_header: Rc>, pager: Rc, - connection: Weak, -) -> Result { - let mut program = ProgramBuilder::new(); +) -> Result<()> { let init_label = program.allocate_label(); program.emit_insn(Insn::Init { target_pc: init_label, @@ -548,17 +538,11 @@ fn translate_pragma( match body { None => { let pragma_name = &name.name.0; - query_pragma(pragma_name, database_header.clone(), &mut program)?; + query_pragma(pragma_name, database_header.clone(), program)?; } Some(ast::PragmaBody::Equals(value)) => { write = true; - update_pragma( - &name.name.0, - value, - database_header.clone(), - pager, - &mut program, - )?; + update_pragma(&name.name.0, value, database_header.clone(), pager, program)?; } Some(ast::PragmaBody::Call(_)) => { todo!() @@ -574,7 +558,8 @@ fn translate_pragma( program.emit_insn(Insn::Goto { target_pc: start_offset, }); - Ok(program.build(database_header, connection)) + + Ok(()) } fn update_pragma( diff --git a/core/translate/select.rs b/core/translate/select.rs index 44dcb5288..b1be01169 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -1,11 +1,7 @@ -use std::rc::Weak; -use std::{cell::RefCell, rc::Rc}; - use super::emitter::emit_program; use super::expr::get_name; use super::plan::SelectQueryType; use crate::function::Func; -use crate::storage::sqlite3_ondisk::DatabaseHeader; use crate::translate::optimizer::optimize_plan; use crate::translate::plan::{Aggregate, Direction, GroupBy, Plan, ResultSetColumn, SelectPlan}; use crate::translate::planner::{ @@ -13,21 +9,20 @@ use crate::translate::planner::{ parse_where, resolve_aggregates, OperatorIdCounter, }; use crate::util::normalize_ident; -use crate::{schema::Schema, vdbe::Program, Result}; -use crate::{Connection, SymbolTable}; +use crate::SymbolTable; +use crate::{schema::Schema, vdbe::builder::ProgramBuilder, Result}; use sqlite3_parser::ast; use sqlite3_parser::ast::ResultColumn; pub fn translate_select( + program: &mut ProgramBuilder, schema: &Schema, select: ast::Select, - database_header: Rc>, - connection: Weak, syms: &SymbolTable, -) -> Result { +) -> Result<()> { let mut select_plan = prepare_select_plan(schema, select)?; optimize_plan(&mut select_plan)?; - emit_program(database_header, select_plan, connection, syms) + emit_program(program, select_plan, syms) } pub fn prepare_select_plan(schema: &Schema, select: ast::Select) -> Result { From 2f2c96fa2c90ef9f6bf7cf2f2645010006f1bc98 Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Mon, 13 Jan 2025 21:31:33 -0300 Subject: [PATCH 33/97] chore: cargo fmt --- core/translate/emitter.rs | 13 ++----------- core/translate/insert.rs | 7 +++++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index cc4df6d8c..e376da160 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -150,11 +150,7 @@ fn epilogue( /// Main entry point for emitting bytecode for a SQL query /// Takes a query plan and generates the corresponding bytecode program -pub fn emit_program( - program: &mut ProgramBuilder, - plan: Plan, - syms: &SymbolTable, -) -> Result<()> { +pub fn emit_program(program: &mut ProgramBuilder, plan: Plan, syms: &SymbolTable) -> Result<()> { match plan { Plan::Select(plan) => emit_program_for_select(program, plan, syms), Plan::Delete(plan) => emit_program_for_delete(program, plan, syms), @@ -272,12 +268,7 @@ fn emit_program_for_delete( } // Initialize cursors and other resources needed for query execution - init_loop( - program, - &mut t_ctx, - &plan.source, - &OperationMode::DELETE, - )?; + init_loop(program, &mut t_ctx, &plan.source, &OperationMode::DELETE)?; // Set up main query execution loop open_loop( diff --git a/core/translate/insert.rs b/core/translate/insert.rs index 1b1669ddb..d8a6b4149 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -8,13 +8,16 @@ use crate::error::SQLITE_CONSTRAINT_PRIMARYKEY; use crate::schema::BTreeTable; use crate::util::normalize_ident; use crate::vdbe::BranchOffset; +use crate::Result; use crate::{ schema::{Column, Schema}, translate::expr::translate_expr, - vdbe::{builder::{CursorType, ProgramBuilder}, insn::Insn}, + vdbe::{ + builder::{CursorType, ProgramBuilder}, + insn::Insn, + }, SymbolTable, }; -use crate::Result; use super::emitter::Resolver; From abb3fda19ff39ce491420648cabf064bf9f1ab1c Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Mon, 13 Jan 2025 18:49:21 -0500 Subject: [PATCH 34/97] Fix all the flaky datetime tests --- core/vdbe/datetime.rs | 105 ++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/core/vdbe/datetime.rs b/core/vdbe/datetime.rs index b0497b0a8..5dd05f393 100644 --- a/core/vdbe/datetime.rs +++ b/core/vdbe/datetime.rs @@ -615,9 +615,8 @@ fn parse_modifier(modifier: &str) -> Result { #[cfg(test)] mod tests { - use std::rc::Rc; - use super::*; + use std::rc::Rc; #[test] fn test_valid_get_date_from_time_value() { @@ -1399,7 +1398,6 @@ mod tests { OwnedValue::build_text(Rc::new(value.to_string())) } - // Basic helper to format NaiveDateTime for comparison fn format(dt: NaiveDateTime) -> String { dt.format("%Y-%m-%d %H:%M:%S").to_string() } @@ -1409,18 +1407,25 @@ mod tests { #[test] fn test_single_modifier() { - let now = Utc::now().naive_utc(); - let expected = format(now - TimeDelta::days(1)); - let result = exec_datetime(&[text("now"), text("-1 day")], DateTimeOutput::DateTime); + let time = setup_datetime(); + let expected = format(time - TimeDelta::days(1)); + let result = exec_datetime( + &[text("2023-06-15 12:30:45"), text("-1 day")], + DateTimeOutput::DateTime, + ); assert_eq!(result, text(&expected)); } #[test] fn test_multiple_modifiers() { - let now = Utc::now().naive_utc(); - let expected = format(now - TimeDelta::days(1) + TimeDelta::hours(3)); + let time = setup_datetime(); + let expected = format(time - TimeDelta::days(1) + TimeDelta::hours(3)); let result = exec_datetime( - &[text("now"), text("-1 day"), text("+3 hours")], + &[ + text("2023-06-15 12:30:45"), + text("-1 day"), + text("+3 hours"), + ], DateTimeOutput::DateTime, ); assert_eq!(result, text(&expected)); @@ -1428,26 +1433,27 @@ mod tests { #[test] fn test_subsec_modifier() { - let now = Utc::now().naive_utc().time(); - let result = exec_datetime(&[text("now"), text("subsec")], DateTimeOutput::Time); - let tolerance = TimeDelta::milliseconds(1); + let time = setup_datetime(); + let result = exec_datetime( + &[text("2023-06-15 12:30:45"), text("subsec")], + DateTimeOutput::Time, + ); let result = chrono::NaiveTime::parse_from_str(&result.to_string(), "%H:%M:%S%.3f").unwrap(); - assert!( - (now - result).num_milliseconds().abs() <= tolerance.num_milliseconds(), - "Expected: {}, Actual: {}", - now, - result - ); + assert_eq!(time.time(), result); } #[test] fn test_start_of_day_modifier() { - let now = Utc::now().naive_utc(); - let start_of_day = now.date().and_hms_opt(0, 0, 0).unwrap(); + let time = setup_datetime(); + let start_of_day = time.date().and_hms_opt(0, 0, 0).unwrap(); let expected = format(start_of_day - TimeDelta::days(1)); let result = exec_datetime( - &[text("now"), text("start of day"), text("-1 day")], + &[ + text("2023-06-15 12:30:45"), + text("start of day"), + text("-1 day"), + ], DateTimeOutput::DateTime, ); assert_eq!(result, text(&expected)); @@ -1455,14 +1461,18 @@ mod tests { #[test] fn test_start_of_month_modifier() { - let now = Utc::now().naive_utc(); - let start_of_month = NaiveDate::from_ymd_opt(now.year(), now.month(), 1) + let time = setup_datetime(); + let start_of_month = NaiveDate::from_ymd_opt(time.year(), time.month(), 1) .unwrap() .and_hms_opt(0, 0, 0) .unwrap(); let expected = format(start_of_month + TimeDelta::days(1)); let result = exec_datetime( - &[text("now"), text("start of month"), text("+1 day")], + &[ + text("2023-06-15 12:30:45"), + text("start of month"), + text("+1 day"), + ], DateTimeOutput::DateTime, ); assert_eq!(result, text(&expected)); @@ -1470,15 +1480,15 @@ mod tests { #[test] fn test_start_of_year_modifier() { - let now = Utc::now().naive_utc(); - let start_of_year = NaiveDate::from_ymd_opt(now.year(), 1, 1) + let time = setup_datetime(); + let start_of_year = NaiveDate::from_ymd_opt(time.year(), 1, 1) .unwrap() .and_hms_opt(0, 0, 0) .unwrap(); let expected = format(start_of_year + TimeDelta::days(30) + TimeDelta::hours(5)); let result = exec_datetime( &[ - text("now"), + text("2023-06-15 12:30:45"), text("start of year"), text("+30 days"), text("+5 hours"), @@ -1488,33 +1498,36 @@ mod tests { assert_eq!(result, text(&expected)); } - /// Test 'localtime' and 'utc' modifiers #[test] - fn test_localtime_and_utc_modifiers() { - let local = chrono::Local::now().naive_local(); - let expected = format(local); - let result = exec_datetime(&[text("now"), text("localtime")], DateTimeOutput::DateTime); - assert_eq!(result, text(&expected)); - - let utc = Utc::now().naive_utc(); - let expected_utc = format(utc); - let result_utc = exec_datetime( - &[text(&local.to_string()), text("utc")], + fn test_timezone_modifiers() { + let dt = setup_datetime(); + let result_local = exec_datetime( + &[text("2023-06-15 12:30:45"), text("localtime")], DateTimeOutput::DateTime, ); - assert_eq!(result_utc, text(&expected_utc)); + assert_eq!( + result_local, + text( + &dt.and_utc() + .with_timezone(&chrono::Local) + .format("%Y-%m-%d %H:%M:%S") + .to_string() + ) + ); + // TODO: utc modifier assumes time given is not already utc + // add test when fixed in the future } #[test] fn test_combined_modifiers() { - let now = Utc::now().naive_utc(); - let expected = now - TimeDelta::days(1) + let time = create_datetime(2000, 1, 1, 0, 0, 0); + let expected = time - TimeDelta::days(1) + TimeDelta::hours(5) + TimeDelta::minutes(30) + TimeDelta::seconds(15); let result = exec_datetime( &[ - text("now"), + text("2000-01-01 00:00:00"), text("-1 day"), text("+5 hours"), text("+30 minutes"), @@ -1523,16 +1536,10 @@ mod tests { ], DateTimeOutput::DateTime, ); - let tolerance = TimeDelta::milliseconds(1); let result = chrono::NaiveDateTime::parse_from_str(&result.to_string(), "%Y-%m-%d %H:%M:%S%.3f") .unwrap(); - assert!( - (result - expected).num_milliseconds().abs() <= tolerance.num_milliseconds(), - "Expected: {}, Actual: {}", - expected, - result - ); + assert_eq!(expected, result); } #[test] From af020c27d6fae99ab6d1fe11b7762785f745411a Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Fri, 3 Jan 2025 10:47:22 +0200 Subject: [PATCH 35/97] Initial take on Rust bindings This implements libSQL compatible Rust API on top of Limbo's core. The purpose of this is to allow libraries and apps that build on libSQL to use Limbo. --- Cargo.lock | 80 +++++++- Cargo.toml | 1 + bindings/rust/Cargo.toml | 16 ++ bindings/rust/src/lib.rs | 128 +++++++++++++ bindings/rust/src/params.rs | 313 +++++++++++++++++++++++++++++++ bindings/rust/src/value.rs | 364 ++++++++++++++++++++++++++++++++++++ 6 files changed, 896 insertions(+), 6 deletions(-) create mode 100644 bindings/rust/Cargo.toml create mode 100644 bindings/rust/src/lib.rs create mode 100644 bindings/rust/src/params.rs create mode 100644 bindings/rust/src/value.rs diff --git a/Cargo.lock b/Cargo.lock index 915456422..fd7bec345 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,6 +1236,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "limbo_libsql" +version = "0.0.11" +dependencies = [ + "limbo_core", + "thiserror 2.0.9", + "tokio", +] + [[package]] name = "limbo_macros" version = "0.0.11" @@ -1365,6 +1374,17 @@ dependencies = [ "adler2", ] +[[package]] +name = "mio" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.52.0", +] + [[package]] name = "mockall" version = "0.13.1" @@ -1524,7 +1544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.6", + "thiserror 2.0.9", "ucd-trie", ] @@ -2129,6 +2149,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51bf3a9dccf2c079bf1465d449a485c85b36443caf765f2f127bfec28b180f75" +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -2150,6 +2179,16 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "socket2" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "sqlite3-parser" version = "0.13.0" @@ -2323,11 +2362,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.9", ] [[package]] @@ -2343,9 +2382,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", @@ -2362,6 +2401,35 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tokio" +version = "1.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "tracing" version = "0.1.41" diff --git a/Cargo.toml b/Cargo.toml index 92897cbb4..bf1eb1062 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ resolver = "2" members = [ "bindings/java", "bindings/python", + "bindings/rust", "bindings/wasm", "cli", "sqlite3", diff --git a/bindings/rust/Cargo.toml b/bindings/rust/Cargo.toml new file mode 100644 index 000000000..bdf101c56 --- /dev/null +++ b/bindings/rust/Cargo.toml @@ -0,0 +1,16 @@ +# Copyright 2025 the Limbo authors. All rights reserved. MIT license. + +[package] +name = "limbo_libsql" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +limbo_core = { path = "../../core" } +thiserror = "2.0.9" + +[dev-dependencies] +tokio = { version = "1.29.1", features = ["full"] } \ No newline at end of file diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs new file mode 100644 index 000000000..13705d08e --- /dev/null +++ b/bindings/rust/src/lib.rs @@ -0,0 +1,128 @@ +pub mod params; +mod value; + +pub use params::params_from_iter; + +use crate::params::*; +use crate::value::*; +use std::rc::Rc; +use std::sync::Arc; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("SQL conversion failure: `{0}`")] + ToSqlConversionFailure(crate::BoxError), +} + +impl From for Error { + fn from(_err: limbo_core::LimboError) -> Self { + todo!(); + } +} + +pub(crate) type BoxError = Box; + +pub type Result = std::result::Result; +pub struct Builder { + path: String, +} + +impl Builder { + pub fn new_local(path: &str) -> Self { + Self { + path: path.to_string(), + } + } + + #[allow(unused_variables, clippy::arc_with_non_send_sync)] + pub async fn build(self) -> Result { + match self.path.as_str() { + ":memory:" => { + let io: Arc = Arc::new(limbo_core::MemoryIO::new()?); + let db = limbo_core::Database::open_file(io, self.path.as_str())?; + Ok(Database { inner: db }) + } + _ => todo!(), + } + } +} + +pub struct Database { + inner: Arc, +} + +impl Database { + pub fn connect(self) -> Result { + let conn = self.inner.connect(); + Ok(Connection { inner: conn }) + } +} + +pub struct Connection { + inner: Rc, +} + +impl Connection { + pub async fn query(&self, sql: &str, params: impl IntoParams) -> Result { + let mut stmt = self.prepare(sql).await?; + stmt.query(params).await + } + + pub async fn execute(&self, sql: &str, params: impl IntoParams) -> Result { + let mut stmt = self.prepare(sql).await?; + stmt.execute(params).await + } + + pub async fn prepare(&self, sql: &str) -> Result { + let stmt = self.inner.prepare(sql)?; + Ok(Statement { + _inner: Rc::new(stmt), + }) + } +} + +pub struct Statement { + _inner: Rc, +} + +impl Statement { + pub async fn query(&mut self, params: impl IntoParams) -> Result { + let _params = params.into_params()?; + todo!(); + } + + pub async fn execute(&mut self, params: impl IntoParams) -> Result { + let _params = params.into_params()?; + todo!(); + } +} + +pub trait IntoValue { + fn into_value(self) -> Result; +} + +#[derive(Debug, Clone)] +pub enum Params { + None, + Positional(Vec), + Named(Vec<(String, Value)>), +} +pub struct Transaction {} + +pub struct Rows { + _inner: Rc, +} + +impl Rows { + pub async fn next(&mut self) -> Result> { + todo!(); + } +} + +pub struct Row {} + +impl Row { + pub fn get_value(&self, _index: usize) -> Result { + todo!(); + } +} diff --git a/bindings/rust/src/params.rs b/bindings/rust/src/params.rs new file mode 100644 index 000000000..c15b6adb5 --- /dev/null +++ b/bindings/rust/src/params.rs @@ -0,0 +1,313 @@ +//! This module contains all `Param` related utilities and traits. + +use crate::{Error, Result, Value}; + +mod sealed { + pub trait Sealed {} +} + +use sealed::Sealed; + +/// Converts some type into parameters that can be passed +/// to libsql. +/// +/// The trait is sealed and not designed to be implemented by hand +/// but instead provides a few ways to use it. +/// +/// # Passing parameters to libsql +/// +/// Many functions in this library let you pass parameters to libsql. Doing this +/// lets you avoid any risk of SQL injection, and is simpler than escaping +/// things manually. These functions generally contain some paramter that generically +/// accepts some implementation this trait. +/// +/// # Positional parameters +/// +/// These can be supplied in a few ways: +/// +/// - For heterogeneous parameter lists of 16 or less items a tuple syntax is supported +/// by doing `(1, "foo")`. +/// - For hetergeneous parameter lists of 16 or greater, the [`limbo_libsql::params!`] is supported +/// by doing `limbo_libsql::params![1, "foo"]`. +/// - For homogeneous paramter types (where they are all the same type), const arrays are +/// supported by doing `[1, 2, 3]`. +/// +/// # Example (positional) +/// +/// ```rust,no_run +/// # use limbo_libsql::{Connection, params}; +/// # async fn run(conn: Connection) -> limbo_libsql::Result<()> { +/// let mut stmt = conn.prepare("INSERT INTO test (a, b) VALUES (?1, ?2)").await?; +/// +/// // Using a tuple: +/// stmt.execute((0, "foobar")).await?; +/// +/// // Using `limbo_libsql::params!`: +/// stmt.execute(params![1i32, "blah"]).await?; +/// +/// // array literal — non-references +/// stmt.execute([2i32, 3i32]).await?; +/// +/// // array literal — references +/// stmt.execute(["foo", "bar"]).await?; +/// +/// // Slice literal, references: +/// stmt.execute([2i32, 3i32]).await?; +/// +/// # Ok(()) +/// # } +/// ``` +/// +/// # Named paramters +/// +/// - For heterogeneous parameter lists of 16 or less items a tuple syntax is supported +/// by doing `(("key1", 1), ("key2", "foo"))`. +/// - For hetergeneous parameter lists of 16 or greater, the [`limbo_libsql::params!`] is supported +/// by doing `limbo_libsql::named_params!["key1": 1, "key2": "foo"]`. +/// - For homogeneous paramter types (where they are all the same type), const arrays are +/// supported by doing `[("key1", 1), ("key2, 2), ("key3", 3)]`. +/// +/// # Example (named) +/// +/// ```rust,no_run +/// # use limbo_libsql::{Connection, named_params}; +/// # async fn run(conn: Connection) -> limbo_libsql::Result<()> { +/// let mut stmt = conn.prepare("INSERT INTO test (a, b) VALUES (:key1, :key2)").await?; +/// +/// // Using a tuple: +/// stmt.execute(((":key1", 0), (":key2", "foobar"))).await?; +/// +/// // Using `limbo_libsql::named_params!`: +/// stmt.execute(named_params! {":key1": 1i32, ":key2": "blah" }).await?; +/// +/// // const array: +/// stmt.execute([(":key1", 2i32), (":key2", 3i32)]).await?; +/// +/// # Ok(()) +/// # } +/// ``` +pub trait IntoParams: Sealed { + // Hide this because users should not be implementing this + // themselves. We should consider sealing this trait. + #[doc(hidden)] + fn into_params(self) -> Result; +} + +#[derive(Debug, Clone)] +#[doc(hidden)] +pub enum Params { + None, + Positional(Vec), + Named(Vec<(String, Value)>), +} + +/// Convert an owned iterator into Params. +/// +/// # Example +/// +/// ```rust +/// # use limbo_libsql::{Connection, params_from_iter, Rows}; +/// # async fn run(conn: &Connection) { +/// +/// let iter = vec![1, 2, 3]; +/// +/// conn.query( +/// "SELECT * FROM users WHERE id IN (?1, ?2, ?3)", +/// params_from_iter(iter) +/// ) +/// .await +/// .unwrap(); +/// # } +/// ``` +pub fn params_from_iter(iter: I) -> impl IntoParams +where + I: IntoIterator, + I::Item: IntoValue, +{ + iter.into_iter().collect::>() +} + +impl Sealed for () {} +impl IntoParams for () { + fn into_params(self) -> Result { + Ok(Params::None) + } +} + +impl Sealed for Params {} +impl IntoParams for Params { + fn into_params(self) -> Result { + Ok(self) + } +} + +impl Sealed for Vec {} +impl IntoParams for Vec { + fn into_params(self) -> Result { + let values = self + .into_iter() + .map(|i| i.into_value()) + .collect::>>()?; + + Ok(Params::Positional(values)) + } +} + +impl Sealed for Vec<(String, T)> {} +impl IntoParams for Vec<(String, T)> { + fn into_params(self) -> Result { + let values = self + .into_iter() + .map(|(k, v)| Ok((k, v.into_value()?))) + .collect::>>()?; + + Ok(Params::Named(values)) + } +} + +impl Sealed for [T; N] {} +impl IntoParams for [T; N] { + fn into_params(self) -> Result { + self.into_iter().collect::>().into_params() + } +} + +impl Sealed for [(&str, T); N] {} +impl IntoParams for [(&str, T); N] { + fn into_params(self) -> Result { + self.into_iter() + // TODO: Pretty unfortunate that we need to allocate here when we know + // the str is likely 'static. Maybe we should convert our param names + // to be `Cow<'static, str>`? + .map(|(k, v)| Ok((k.to_string(), v.into_value()?))) + .collect::>>()? + .into_params() + } +} + +impl Sealed for &[T; N] {} +impl IntoParams for &[T; N] { + fn into_params(self) -> Result { + self.iter().cloned().collect::>().into_params() + } +} + +// NOTICE: heavily inspired by rusqlite +macro_rules! tuple_into_params { + ($count:literal : $(($field:tt $ftype:ident)),* $(,)?) => { + impl<$($ftype,)*> Sealed for ($($ftype,)*) where $($ftype: IntoValue,)* {} + impl<$($ftype,)*> IntoParams for ($($ftype,)*) where $($ftype: IntoValue,)* { + fn into_params(self) -> Result { + let params = Params::Positional(vec![$(self.$field.into_value()?),*]); + Ok(params) + } + } + } +} + +macro_rules! named_tuple_into_params { + ($count:literal : $(($field:tt $ftype:ident)),* $(,)?) => { + impl<$($ftype,)*> Sealed for ($((&str, $ftype),)*) where $($ftype: IntoValue,)* {} + impl<$($ftype,)*> IntoParams for ($((&str, $ftype),)*) where $($ftype: IntoValue,)* { + fn into_params(self) -> Result { + let params = Params::Named(vec![$((self.$field.0.to_string(), self.$field.1.into_value()?)),*]); + Ok(params) + } + } + } +} + +named_tuple_into_params!(2: (0 A), (1 B)); +named_tuple_into_params!(3: (0 A), (1 B), (2 C)); +named_tuple_into_params!(4: (0 A), (1 B), (2 C), (3 D)); +named_tuple_into_params!(5: (0 A), (1 B), (2 C), (3 D), (4 E)); +named_tuple_into_params!(6: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F)); +named_tuple_into_params!(7: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G)); +named_tuple_into_params!(8: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H)); +named_tuple_into_params!(9: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I)); +named_tuple_into_params!(10: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J)); +named_tuple_into_params!(11: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K)); +named_tuple_into_params!(12: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L)); +named_tuple_into_params!(13: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L), (12 M)); +named_tuple_into_params!(14: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L), (12 M), (13 N)); +named_tuple_into_params!(15: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L), (12 M), (13 N), (14 O)); +named_tuple_into_params!(16: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L), (12 M), (13 N), (14 O), (15 P)); + +tuple_into_params!(2: (0 A), (1 B)); +tuple_into_params!(3: (0 A), (1 B), (2 C)); +tuple_into_params!(4: (0 A), (1 B), (2 C), (3 D)); +tuple_into_params!(5: (0 A), (1 B), (2 C), (3 D), (4 E)); +tuple_into_params!(6: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F)); +tuple_into_params!(7: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G)); +tuple_into_params!(8: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H)); +tuple_into_params!(9: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I)); +tuple_into_params!(10: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J)); +tuple_into_params!(11: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K)); +tuple_into_params!(12: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L)); +tuple_into_params!(13: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L), (12 M)); +tuple_into_params!(14: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L), (12 M), (13 N)); +tuple_into_params!(15: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L), (12 M), (13 N), (14 O)); +tuple_into_params!(16: (0 A), (1 B), (2 C), (3 D), (4 E), (5 F), (6 G), (7 H), (8 I), (9 J), (10 K), (11 L), (12 M), (13 N), (14 O), (15 P)); + +// TODO: Should we rename this to `ToSql` which makes less sense but +// matches the error variant we have in `Error`. Or should we change the +// error variant to match this breaking the few people that currently use +// this error variant. +pub trait IntoValue { + fn into_value(self) -> Result; +} + +impl IntoValue for T +where + T: TryInto, + T::Error: Into, +{ + fn into_value(self) -> Result { + self.try_into() + .map_err(|e| Error::ToSqlConversionFailure(e.into())) + } +} + +impl IntoValue for Result { + fn into_value(self) -> Result { + self + } +} + +/// Construct positional params from a hetergeneous set of params types. +#[macro_export] +macro_rules! params { + () => { + () + }; + ($($value:expr),* $(,)?) => {{ + use $crate::params::IntoValue; + [$($value.into_value()),*] + + }}; +} + +/// Construct named params from a hetergeneous set of params types. +#[macro_export] +macro_rules! named_params { + () => { + () + }; + ($($param_name:literal: $value:expr),* $(,)?) => {{ + use $crate::params::IntoValue; + [$(($param_name, $value.into_value())),*] + }}; +} + +#[cfg(test)] +mod tests { + use crate::Value; + + #[test] + fn test_serialize_array() { + assert_eq!( + params!([0; 16])[0].as_ref().unwrap(), + &Value::Blob(vec![0; 16]) + ); + } +} diff --git a/bindings/rust/src/value.rs b/bindings/rust/src/value.rs new file mode 100644 index 000000000..672444afc --- /dev/null +++ b/bindings/rust/src/value.rs @@ -0,0 +1,364 @@ +use std::str::FromStr; + +use crate::{Error, Result}; + +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + Null, + Integer(i64), + Real(f64), + Text(String), + Blob(Vec), +} + +/// The possible types a column can be in libsql. +#[derive(Debug, Copy, Clone)] +pub enum ValueType { + Integer = 1, + Real, + Text, + Blob, + Null, +} + +impl FromStr for ValueType { + type Err = (); + + fn from_str(s: &str) -> std::result::Result { + match s { + "TEXT" => Ok(ValueType::Text), + "INTEGER" => Ok(ValueType::Integer), + "BLOB" => Ok(ValueType::Blob), + "NULL" => Ok(ValueType::Null), + "REAL" => Ok(ValueType::Real), + _ => Err(()), + } + } +} + +impl Value { + /// Returns `true` if the value is [`Null`]. + /// + /// [`Null`]: Value::Null + #[must_use] + pub fn is_null(&self) -> bool { + matches!(self, Self::Null) + } + + /// Returns `true` if the value is [`Integer`]. + /// + /// [`Integer`]: Value::Integer + #[must_use] + pub fn is_integer(&self) -> bool { + matches!(self, Self::Integer(..)) + } + + /// Returns `true` if the value is [`Real`]. + /// + /// [`Real`]: Value::Real + #[must_use] + pub fn is_real(&self) -> bool { + matches!(self, Self::Real(..)) + } + + pub fn as_real(&self) -> Option<&f64> { + if let Self::Real(v) = self { + Some(v) + } else { + None + } + } + + /// Returns `true` if the value is [`Text`]. + /// + /// [`Text`]: Value::Text + #[must_use] + pub fn is_text(&self) -> bool { + matches!(self, Self::Text(..)) + } + + pub fn as_text(&self) -> Option<&String> { + if let Self::Text(v) = self { + Some(v) + } else { + None + } + } + + pub fn as_integer(&self) -> Option<&i64> { + if let Self::Integer(v) = self { + Some(v) + } else { + None + } + } + + /// Returns `true` if the value is [`Blob`]. + /// + /// [`Blob`]: Value::Blob + #[must_use] + pub fn is_blob(&self) -> bool { + matches!(self, Self::Blob(..)) + } + + pub fn as_blob(&self) -> Option<&Vec> { + if let Self::Blob(v) = self { + Some(v) + } else { + None + } + } +} + +impl From for Value { + fn from(value: i8) -> Value { + Value::Integer(value as i64) + } +} + +impl From for Value { + fn from(value: i16) -> Value { + Value::Integer(value as i64) + } +} + +impl From for Value { + fn from(value: i32) -> Value { + Value::Integer(value as i64) + } +} + +impl From for Value { + fn from(value: i64) -> Value { + Value::Integer(value) + } +} + +impl From for Value { + fn from(value: u8) -> Value { + Value::Integer(value as i64) + } +} + +impl From for Value { + fn from(value: u16) -> Value { + Value::Integer(value as i64) + } +} + +impl From for Value { + fn from(value: u32) -> Value { + Value::Integer(value as i64) + } +} + +impl TryFrom for Value { + type Error = crate::Error; + + fn try_from(value: u64) -> Result { + if value > i64::MAX as u64 { + Err(Error::ToSqlConversionFailure( + "u64 is too large to fit in an i64".into(), + )) + } else { + Ok(Value::Integer(value as i64)) + } + } +} + +impl From for Value { + fn from(value: f32) -> Value { + Value::Real(value as f64) + } +} + +impl From for Value { + fn from(value: f64) -> Value { + Value::Real(value) + } +} + +impl From<&str> for Value { + fn from(value: &str) -> Value { + Value::Text(value.to_owned()) + } +} + +impl From for Value { + fn from(value: String) -> Value { + Value::Text(value) + } +} + +impl From<&[u8]> for Value { + fn from(value: &[u8]) -> Value { + Value::Blob(value.to_owned()) + } +} + +impl From> for Value { + fn from(value: Vec) -> Value { + Value::Blob(value) + } +} + +impl From for Value { + fn from(value: bool) -> Value { + Value::Integer(value as i64) + } +} + +impl From> for Value +where + T: Into, +{ + fn from(value: Option) -> Self { + match value { + Some(inner) => inner.into(), + None => Value::Null, + } + } +} + +/// A borrowed version of `Value`. +#[derive(Debug)] +pub enum ValueRef<'a> { + Null, + Integer(i64), + Real(f64), + Text(&'a [u8]), + Blob(&'a [u8]), +} + +impl ValueRef<'_> { + pub fn data_type(&self) -> ValueType { + match *self { + ValueRef::Null => ValueType::Null, + ValueRef::Integer(_) => ValueType::Integer, + ValueRef::Real(_) => ValueType::Real, + ValueRef::Text(_) => ValueType::Text, + ValueRef::Blob(_) => ValueType::Blob, + } + } + + /// Returns `true` if the value ref is [`Null`]. + /// + /// [`Null`]: ValueRef::Null + #[must_use] + pub fn is_null(&self) -> bool { + matches!(self, Self::Null) + } + + /// Returns `true` if the value ref is [`Integer`]. + /// + /// [`Integer`]: ValueRef::Integer + #[must_use] + pub fn is_integer(&self) -> bool { + matches!(self, Self::Integer(..)) + } + + pub fn as_integer(&self) -> Option<&i64> { + if let Self::Integer(v) = self { + Some(v) + } else { + None + } + } + + /// Returns `true` if the value ref is [`Real`]. + /// + /// [`Real`]: ValueRef::Real + #[must_use] + pub fn is_real(&self) -> bool { + matches!(self, Self::Real(..)) + } + + pub fn as_real(&self) -> Option<&f64> { + if let Self::Real(v) = self { + Some(v) + } else { + None + } + } + + /// Returns `true` if the value ref is [`Text`]. + /// + /// [`Text`]: ValueRef::Text + #[must_use] + pub fn is_text(&self) -> bool { + matches!(self, Self::Text(..)) + } + + pub fn as_text(&self) -> Option<&[u8]> { + if let Self::Text(v) = self { + Some(v) + } else { + None + } + } + + /// Returns `true` if the value ref is [`Blob`]. + /// + /// [`Blob`]: ValueRef::Blob + #[must_use] + pub fn is_blob(&self) -> bool { + matches!(self, Self::Blob(..)) + } + + pub fn as_blob(&self) -> Option<&[u8]> { + if let Self::Blob(v) = self { + Some(v) + } else { + None + } + } +} + +impl From> for Value { + fn from(vr: ValueRef<'_>) -> Value { + match vr { + ValueRef::Null => Value::Null, + ValueRef::Integer(i) => Value::Integer(i), + ValueRef::Real(r) => Value::Real(r), + ValueRef::Text(s) => Value::Text(String::from_utf8_lossy(s).to_string()), + ValueRef::Blob(b) => Value::Blob(b.to_vec()), + } + } +} + +impl<'a> From<&'a str> for ValueRef<'a> { + fn from(s: &str) -> ValueRef<'_> { + ValueRef::Text(s.as_bytes()) + } +} + +impl<'a> From<&'a [u8]> for ValueRef<'a> { + fn from(s: &[u8]) -> ValueRef<'_> { + ValueRef::Blob(s) + } +} + +impl<'a> From<&'a Value> for ValueRef<'a> { + fn from(v: &'a Value) -> ValueRef<'a> { + match *v { + Value::Null => ValueRef::Null, + Value::Integer(i) => ValueRef::Integer(i), + Value::Real(r) => ValueRef::Real(r), + Value::Text(ref s) => ValueRef::Text(s.as_bytes()), + Value::Blob(ref b) => ValueRef::Blob(b), + } + } +} + +impl<'a, T> From> for ValueRef<'a> +where + T: Into>, +{ + #[inline] + fn from(s: Option) -> ValueRef<'a> { + match s { + Some(x) => x.into(), + None => ValueRef::Null, + } + } +} From 2186af6c8900702094f35b8209475476522b3d63 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 10:07:00 +0200 Subject: [PATCH 36/97] Bump "build-native" timeout to 10 minutes --- .github/workflows/rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index bebf36794..c7df4d187 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -37,7 +37,7 @@ jobs: env: RUST_LOG: ${{ runner.debug && 'limbo_core::storage=trace' || '' }} run: cargo test --verbose - timeout-minutes: 5 + timeout-minutes: 10 clippy: From f03b6bffdec4e522f1b92875844f1a1acf0452f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 17:12:03 +0900 Subject: [PATCH 37/97] Update license path --- NOTICE.md => bindings/java/NOTICE.md | 4 ++-- .../java/licenses/assertj-license.md | 0 .../java/licenses/errorprone-license.md | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename NOTICE.md => bindings/java/NOTICE.md (73%) rename licenses/LICENSE.assertj.al20.txt => bindings/java/licenses/assertj-license.md (100%) rename licenses/LICENSE.errorprone.al20.txt => bindings/java/licenses/errorprone-license.md (100%) diff --git a/NOTICE.md b/bindings/java/NOTICE.md similarity index 73% rename from NOTICE.md rename to bindings/java/NOTICE.md index 5a08d6fed..666b06930 100644 --- a/NOTICE.md +++ b/bindings/java/NOTICE.md @@ -10,10 +10,10 @@ Dependencies This product depends on Error Prone, distributed by the Error Prone project: -* License: licenses/LICENSE.errorprone.al20.txt (Apache License v2.0) +* License: licenses/assertj-license.md (Apache License v2.0) * Homepage: https://github.com/google/error-prone This product depends on AssertJ, distributed by the AssertJ authors: -* License: licenses/LICENSE.assertj.al20.txt (Apache License v2.0) +* License: licenses/errorprone-license.md (Apache License v2.0) * Homepage: https://joel-costigliola.github.io/assertj/ diff --git a/licenses/LICENSE.assertj.al20.txt b/bindings/java/licenses/assertj-license.md similarity index 100% rename from licenses/LICENSE.assertj.al20.txt rename to bindings/java/licenses/assertj-license.md diff --git a/licenses/LICENSE.errorprone.al20.txt b/bindings/java/licenses/errorprone-license.md similarity index 100% rename from licenses/LICENSE.errorprone.al20.txt rename to bindings/java/licenses/errorprone-license.md From f604e227b1a2f778172dad75b7fab8f0cb3d22e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 17:12:50 +0900 Subject: [PATCH 38/97] Update java.yaml workflow to use java 17 --- .github/workflows/java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 7c3c6b7ba..4a8b0310d 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -32,7 +32,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: '11' + java-version: '17' - name: Run Java tests run: make test From 04cd6555743d5ff53571bb963b871cee69460f81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 17:19:33 +0900 Subject: [PATCH 39/97] Change VALID_URL_PREFIX to use sqlite instead --- .../java/src/main/java/org/github/tursodatabase/JDBC.java | 2 +- .../java/src/test/java/org/github/tursodatabase/JDBCTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java index 87200ff5c..a341f032a 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java @@ -7,7 +7,7 @@ import java.util.Properties; import java.util.logging.Logger; public class JDBC implements Driver { - private static final String VALID_URL_PREFIX = "jdbc:limbo:"; + private static final String VALID_URL_PREFIX = "jdbc:sqlite:"; static { try { diff --git a/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java b/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java index d0cdc4dc3..45452f810 100644 --- a/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java +++ b/bindings/java/src/test/java/org/github/tursodatabase/JDBCTest.java @@ -20,13 +20,13 @@ class JDBCTest { @Test void non_null_connection_is_returned_when_valid_url_is_passed() throws Exception { String fileUrl = TestUtils.createTempFile(); - LimboConnection connection = JDBC.createConnection("jdbc:limbo:" + fileUrl, new Properties()); + LimboConnection connection = JDBC.createConnection("jdbc:sqlite:" + fileUrl, new Properties()); assertThat(connection).isNotNull(); } @Test void connection_can_be_retrieved_from_DriverManager() throws SQLException { - try (Connection connection = DriverManager.getConnection("jdbc:limbo:sample.db")) { + try (Connection connection = DriverManager.getConnection("jdbc:sqlite:sample.db")) { assertThat(connection).isNotNull(); } } From d223c72d032d6f208f403a2c68c8a9e261175323 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 10:25:11 +0200 Subject: [PATCH 40/97] Revert "core: Previous commits didn't actually remove nix as dependency, so do that here" This reverts commit cca3846f950c262c6a92c28980df155d94ef658b, we need to bring it back unfortunately. --- Cargo.lock | 1 + core/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index aa3f2d2e8..fd7bec345 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1217,6 +1217,7 @@ dependencies = [ "miette", "mimalloc", "mockall", + "nix 0.29.0", "pest", "pest_derive", "polling", diff --git a/core/Cargo.toml b/core/Cargo.toml index 0daa58c0d..c0c12a0a3 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -40,6 +40,7 @@ fallible-iterator = "0.3.0" hex = "0.4.3" libc = "0.2.155" log = "0.4.20" +nix = { version = "0.29.0", features = ["fs"] } sieve-cache = "0.1.4" sqlite3-parser = { path = "../vendored/sqlite3-parser" } thiserror = "1.0.61" From 5c9505e8f72854d359a5a3a89d6442c1ffd1a3e8 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 10:25:23 +0200 Subject: [PATCH 41/97] Revert "core/io/io_uring: replace nix and libc calls with their rustix counterparts." This reverts commit b146f5d4cba52648942d5b4d03b66ae649a50371 because it causes tests to hang. --- core/io/io_uring.rs | 81 +++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index 14a6bd83c..4f6001cdf 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -1,13 +1,11 @@ use super::{common, Completion, File, OpenFlags, IO}; use crate::{LimboError, Result}; +use libc::{c_short, fcntl, flock, iovec, F_SETLK}; use log::{debug, trace}; -use rustix::fs::{self, FlockOperation, OFlags}; -use rustix::io_uring::iovec; +use nix::fcntl::{FcntlArg, OFlag}; use std::cell::RefCell; use std::collections::HashMap; use std::fmt; -use std::io::ErrorKind; -use std::os::fd::AsFd; use std::os::unix::io::AsRawFd; use std::rc::Rc; use thiserror::Error; @@ -138,12 +136,12 @@ impl IO for UringIO { .open(path)?; // Let's attempt to enable direct I/O. Not all filesystems support it // so ignore any errors. - let fd = file.as_fd(); + let fd = file.as_raw_fd(); if direct { - match fs::fcntl_setfl(fd, OFlags::DIRECT) { - Ok(_) => {} + match nix::fcntl::fcntl(fd, FcntlArg::F_SETFL(OFlag::O_DIRECT)) { + Ok(_) => {}, Err(error) => debug!("Error {error:?} returned when setting O_DIRECT flag to read file. The performance of the system may be affected"), - } + }; } let uring_file = Rc::new(UringFile { io: self.inner.clone(), @@ -201,39 +199,52 @@ pub struct UringFile { impl File for UringFile { fn lock_file(&self, exclusive: bool) -> Result<()> { - let fd = self.file.as_fd(); + let fd = self.file.as_raw_fd(); + let flock = flock { + l_type: if exclusive { + libc::F_WRLCK as c_short + } else { + libc::F_RDLCK as c_short + }, + l_whence: libc::SEEK_SET as c_short, + l_start: 0, + l_len: 0, // Lock entire file + l_pid: 0, + }; + // F_SETLK is a non-blocking lock. The lock will be released when the file is closed // or the process exits or after an explicit unlock. - fs::fcntl_lock( - fd, - if exclusive { - FlockOperation::LockExclusive + let lock_result = unsafe { fcntl(fd, F_SETLK, &flock) }; + if lock_result == -1 { + let err = std::io::Error::last_os_error(); + if err.kind() == std::io::ErrorKind::WouldBlock { + return Err(LimboError::LockingError( + "File is locked by another process".into(), + )); } else { - FlockOperation::LockShared - }, - ) - .map_err(|e| { - let io_error = std::io::Error::from(e); - let message = match io_error.kind() { - ErrorKind::WouldBlock => { - "Failed locking file. File is locked by another process".to_string() - } - _ => format!("Failed locking file, {}", io_error), - }; - LimboError::LockingError(message) - })?; - + return Err(LimboError::IOError(err)); + } + } Ok(()) } fn unlock_file(&self) -> Result<()> { - let fd = self.file.as_fd(); - fs::fcntl_lock(fd, FlockOperation::Unlock).map_err(|e| { - LimboError::LockingError(format!( + let fd = self.file.as_raw_fd(); + let flock = flock { + l_type: libc::F_UNLCK as c_short, + l_whence: libc::SEEK_SET as c_short, + l_start: 0, + l_len: 0, + l_pid: 0, + }; + + let unlock_result = unsafe { fcntl(fd, F_SETLK, &flock) }; + if unlock_result == -1 { + return Err(LimboError::LockingError(format!( "Failed to release file lock: {}", - std::io::Error::from(e) - )) - })?; + std::io::Error::last_os_error() + ))); + } Ok(()) } @@ -250,7 +261,7 @@ impl File for UringFile { let len = buf.len(); let buf = buf.as_mut_ptr(); let iovec = io.get_iovec(buf, len); - io_uring::opcode::Readv::new(fd, iovec as *const iovec as *const libc::iovec, 1) + io_uring::opcode::Readv::new(fd, iovec, 1) .offset(pos as u64) .build() .user_data(io.ring.get_key()) @@ -271,7 +282,7 @@ impl File for UringFile { let buf = buffer.borrow(); trace!("pwrite(pos = {}, length = {})", pos, buf.len()); let iovec = io.get_iovec(buf.as_ptr(), buf.len()); - io_uring::opcode::Writev::new(fd, iovec as *const iovec as *const libc::iovec, 1) + io_uring::opcode::Writev::new(fd, iovec, 1) .offset(pos as u64) .build() .user_data(io.ring.get_key()) From cd04bf796a5a947d3becb7a2dea6bf6e354aede7 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 10:34:00 +0200 Subject: [PATCH 42/97] Update README.md --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 91d7cdad1..4c5d86908 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,12 @@ ## Features -* In-process OLTP database engine library -* Asynchronous I/O support on Linux with `io_uring` -* SQLite compatibility ([status](COMPAT.md)) - * SQL dialect support - * File format support - * SQLite C API -* JavaScript/WebAssembly bindings (_wip_) -* Support for Linux, macOS, and Windows +Limbo is an in-process OLTP database engine library that has: + +* **Asynchronous I/O** support on Linux with `io_uring` +* **SQLite compatibility** [[doc](COMPAT.md)] for SQL dialect, file formats, and the C API +* **Language bindings** for JavaScript/WebAssembly, Rust, Python, and Java +* **OS support** for Linux, macOS, and Windows ## Getting Started From 0461fd4c41e6110c0e6dce09bfae6778b2d4ef0f Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 11:38:26 +0200 Subject: [PATCH 43/97] Update CHANGELOG --- CHANGELOG.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 698300537..aed946ccf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,61 @@ # Changelog +## 0.0.12 - 2025-01-14 + +### Added + +**Core:** + +* Improve JSON function support (Kacper Madej, Peter Sooley) + +* Support nested parenthesized conditional expressions (Preston Thorpe) + +* Add support for changes() and total_changes() functions (Lemon-Peppermint) + +* Auto-create index in CREATE TABLE when necessary (Jussi Saurio) + +* Add partial support for datetime() function (Preston Thorpe) + +* SQL parser performance improvements (Jussi Saurio) + +**Shell:** + +* Show pretty parse errors in the shell (Samyak Sarnayak) + +* Add CSV import support to shell (Vrishabh) + +* Selectable IO backend with --io={syscall,io-uring} argument (Jorge López Tello) + +**Bindings:** + +* Initial version of Java bindings (Kim Seon Woo) + +* Initial version of Rust bindings (Pekka Enberg) + +* Add OPFS support to Wasm bindings (Elijah Morgan) + +* Support uncorrelated FROM clause subqueries (Jussi Saurio) + +* In-memory support to `sqlite3_open()` (Pekka Enberg) + +### Fixed + +* Make iterate() lazy in JavaScript bindings (Diego Reis) + +* Fix integer overflow output to be same as sqlite3 (Vrishabh) + +* Fix 8-bit serial type to encoding (Preston Thorpe) + +* Query plan optimizer bug fixes (Jussi Saurio) + +* B-Tree balancing fixes (Pere Diaz Bou) + +* Fix index seek wrong on `SeekOp::LT`\`SeekOp::GT` (Kould) + +* Fix arithmetic operations for text values' from Vrishabh + +* Fix quote escape in SQL literals (Vrishabh) + ## 0.0.11 - 2024-12-31 ### Added From b6ae8990e354f44e422cc3bb1c21d933dac5c675 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 11:39:17 +0200 Subject: [PATCH 44/97] Limbo 0.0.12 --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 2 +- bindings/wasm/package-lock.json | 4 ++-- bindings/wasm/package.json | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd7bec345..5312501a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -417,7 +417,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core_tester" -version = "0.0.11" +version = "0.0.12" dependencies = [ "anyhow", "clap", @@ -1059,7 +1059,7 @@ checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "java-limbo" -version = "0.0.11" +version = "0.0.12" dependencies = [ "anyhow", "jni", @@ -1170,7 +1170,7 @@ dependencies = [ [[package]] name = "limbo" -version = "0.0.11" +version = "0.0.12" dependencies = [ "anyhow", "clap", @@ -1186,7 +1186,7 @@ dependencies = [ [[package]] name = "limbo-wasm" -version = "0.0.11" +version = "0.0.12" dependencies = [ "console_error_panic_hook", "js-sys", @@ -1198,7 +1198,7 @@ dependencies = [ [[package]] name = "limbo_core" -version = "0.0.11" +version = "0.0.12" dependencies = [ "bumpalo", "cfg_block", @@ -1238,7 +1238,7 @@ dependencies = [ [[package]] name = "limbo_libsql" -version = "0.0.11" +version = "0.0.12" dependencies = [ "limbo_core", "thiserror 2.0.9", @@ -1247,11 +1247,11 @@ dependencies = [ [[package]] name = "limbo_macros" -version = "0.0.11" +version = "0.0.12" [[package]] name = "limbo_sim" -version = "0.0.11" +version = "0.0.12" dependencies = [ "anarchist-readable-name-generator-lib", "clap", @@ -1265,7 +1265,7 @@ dependencies = [ [[package]] name = "limbo_sqlite3" -version = "0.0.11" +version = "0.0.12" dependencies = [ "env_logger 0.11.5", "libc", @@ -1757,7 +1757,7 @@ dependencies = [ [[package]] name = "py-limbo" -version = "0.0.11" +version = "0.0.12" dependencies = [ "anyhow", "limbo_core", diff --git a/Cargo.toml b/Cargo.toml index bf1eb1062..0d2fc81be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ members = [ exclude = ["perf/latency/limbo"] [workspace.package] -version = "0.0.11" +version = "0.0.12" authors = ["the Limbo authors"] edition = "2021" license = "MIT" diff --git a/bindings/wasm/package-lock.json b/bindings/wasm/package-lock.json index 5028485f2..a41e64210 100644 --- a/bindings/wasm/package-lock.json +++ b/bindings/wasm/package-lock.json @@ -1,12 +1,12 @@ { "name": "limbo-wasm", - "version": "0.0.11", + "version": "0.0.12", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "limbo-wasm", - "version": "0.0.11", + "version": "0.0.12", "license": "MIT", "devDependencies": { "@playwright/test": "^1.49.1", diff --git a/bindings/wasm/package.json b/bindings/wasm/package.json index 7fcc1e9e4..4107b052d 100644 --- a/bindings/wasm/package.json +++ b/bindings/wasm/package.json @@ -3,7 +3,7 @@ "collaborators": [ "the Limbo authors" ], - "version": "0.0.11", + "version": "0.0.12", "license": "MIT", "repository": { "type": "git", From 55e06b0c72e2568f21732ea89bcc2154c625e8ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Tue, 14 Jan 2025 01:15:17 +0100 Subject: [PATCH 45/97] core/io: make file locks non-blocking so they fail right away --- core/io/io_uring.rs | 81 ++++++++++++++++++++------------------------- core/io/unix.rs | 6 ++-- 2 files changed, 38 insertions(+), 49 deletions(-) diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index 4f6001cdf..1598debfa 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -1,11 +1,13 @@ use super::{common, Completion, File, OpenFlags, IO}; use crate::{LimboError, Result}; -use libc::{c_short, fcntl, flock, iovec, F_SETLK}; use log::{debug, trace}; -use nix::fcntl::{FcntlArg, OFlag}; +use rustix::fs::{self, FlockOperation, OFlags}; +use rustix::io_uring::iovec; use std::cell::RefCell; use std::collections::HashMap; use std::fmt; +use std::io::ErrorKind; +use std::os::fd::AsFd; use std::os::unix::io::AsRawFd; use std::rc::Rc; use thiserror::Error; @@ -136,12 +138,12 @@ impl IO for UringIO { .open(path)?; // Let's attempt to enable direct I/O. Not all filesystems support it // so ignore any errors. - let fd = file.as_raw_fd(); + let fd = file.as_fd(); if direct { - match nix::fcntl::fcntl(fd, FcntlArg::F_SETFL(OFlag::O_DIRECT)) { - Ok(_) => {}, + match fs::fcntl_setfl(fd, OFlags::DIRECT) { + Ok(_) => {} Err(error) => debug!("Error {error:?} returned when setting O_DIRECT flag to read file. The performance of the system may be affected"), - }; + } } let uring_file = Rc::new(UringFile { io: self.inner.clone(), @@ -199,52 +201,39 @@ pub struct UringFile { impl File for UringFile { fn lock_file(&self, exclusive: bool) -> Result<()> { - let fd = self.file.as_raw_fd(); - let flock = flock { - l_type: if exclusive { - libc::F_WRLCK as c_short - } else { - libc::F_RDLCK as c_short - }, - l_whence: libc::SEEK_SET as c_short, - l_start: 0, - l_len: 0, // Lock entire file - l_pid: 0, - }; - + let fd = self.file.as_fd(); // F_SETLK is a non-blocking lock. The lock will be released when the file is closed // or the process exits or after an explicit unlock. - let lock_result = unsafe { fcntl(fd, F_SETLK, &flock) }; - if lock_result == -1 { - let err = std::io::Error::last_os_error(); - if err.kind() == std::io::ErrorKind::WouldBlock { - return Err(LimboError::LockingError( - "File is locked by another process".into(), - )); + fs::fcntl_lock( + fd, + if exclusive { + FlockOperation::NonBlockingLockExclusive } else { - return Err(LimboError::IOError(err)); - } - } + FlockOperation::NonBlockingLockShared + }, + ) + .map_err(|e| { + let io_error = std::io::Error::from(e); + let message = match io_error.kind() { + ErrorKind::WouldBlock => { + "Failed locking file. File is locked by another process".to_string() + } + _ => format!("Failed locking file, {}", io_error), + }; + LimboError::LockingError(message) + })?; + Ok(()) } fn unlock_file(&self) -> Result<()> { - let fd = self.file.as_raw_fd(); - let flock = flock { - l_type: libc::F_UNLCK as c_short, - l_whence: libc::SEEK_SET as c_short, - l_start: 0, - l_len: 0, - l_pid: 0, - }; - - let unlock_result = unsafe { fcntl(fd, F_SETLK, &flock) }; - if unlock_result == -1 { - return Err(LimboError::LockingError(format!( + let fd = self.file.as_fd(); + fs::fcntl_lock(fd, FlockOperation::NonBlockingUnlock).map_err(|e| { + LimboError::LockingError(format!( "Failed to release file lock: {}", - std::io::Error::last_os_error() - ))); - } + std::io::Error::from(e) + )) + })?; Ok(()) } @@ -261,7 +250,7 @@ impl File for UringFile { let len = buf.len(); let buf = buf.as_mut_ptr(); let iovec = io.get_iovec(buf, len); - io_uring::opcode::Readv::new(fd, iovec, 1) + io_uring::opcode::Readv::new(fd, iovec as *const iovec as *const libc::iovec, 1) .offset(pos as u64) .build() .user_data(io.ring.get_key()) @@ -282,7 +271,7 @@ impl File for UringFile { let buf = buffer.borrow(); trace!("pwrite(pos = {}, length = {})", pos, buf.len()); let iovec = io.get_iovec(buf.as_ptr(), buf.len()); - io_uring::opcode::Writev::new(fd, iovec, 1) + io_uring::opcode::Writev::new(fd, iovec as *const iovec as *const libc::iovec, 1) .offset(pos as u64) .build() .user_data(io.ring.get_key()) diff --git a/core/io/unix.rs b/core/io/unix.rs index c021c4566..effd94bf5 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -142,9 +142,9 @@ impl File for UnixFile { fs::fcntl_lock( fd, if exclusive { - FlockOperation::LockExclusive + FlockOperation::NonBlockingLockExclusive } else { - FlockOperation::LockShared + FlockOperation::NonBlockingLockShared }, ) .map_err(|e| { @@ -164,7 +164,7 @@ impl File for UnixFile { fn unlock_file(&self) -> Result<()> { let fd = self.file.borrow(); let fd = fd.as_fd(); - fs::fcntl_lock(fd, FlockOperation::Unlock).map_err(|e| { + fs::fcntl_lock(fd, FlockOperation::NonBlockingUnlock).map_err(|e| { LimboError::LockingError(format!( "Failed to release file lock: {}", std::io::Error::from(e) From a16282ea62c6968924648506afee6d3c330f8e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Tue, 14 Jan 2025 11:06:13 +0100 Subject: [PATCH 46/97] core: remove nix as a dependency --- Cargo.lock | 1 - core/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5312501a3..bb69e5c0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1217,7 +1217,6 @@ dependencies = [ "miette", "mimalloc", "mockall", - "nix 0.29.0", "pest", "pest_derive", "polling", diff --git a/core/Cargo.toml b/core/Cargo.toml index c0c12a0a3..0daa58c0d 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -40,7 +40,6 @@ fallible-iterator = "0.3.0" hex = "0.4.3" libc = "0.2.155" log = "0.4.20" -nix = { version = "0.29.0", features = ["fs"] } sieve-cache = "0.1.4" sqlite3-parser = { path = "../vendored/sqlite3-parser" } thiserror = "1.0.61" From 66c9832bec8b71e2296e5aa5d6b4e7fc2872cbd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 19:33:29 +0900 Subject: [PATCH 47/97] Create top level licenses directory --- bindings/java/NOTICE.md => NOTICE.md | 0 .../java/licenses => licenses/bindings/java}/assertj-license.md | 0 .../licenses => licenses/bindings/java}/errorprone-license.md | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename bindings/java/NOTICE.md => NOTICE.md (100%) rename {bindings/java/licenses => licenses/bindings/java}/assertj-license.md (100%) rename {bindings/java/licenses => licenses/bindings/java}/errorprone-license.md (100%) diff --git a/bindings/java/NOTICE.md b/NOTICE.md similarity index 100% rename from bindings/java/NOTICE.md rename to NOTICE.md diff --git a/bindings/java/licenses/assertj-license.md b/licenses/bindings/java/assertj-license.md similarity index 100% rename from bindings/java/licenses/assertj-license.md rename to licenses/bindings/java/assertj-license.md diff --git a/bindings/java/licenses/errorprone-license.md b/licenses/bindings/java/errorprone-license.md similarity index 100% rename from bindings/java/licenses/errorprone-license.md rename to licenses/bindings/java/errorprone-license.md From 8ed2d14ca410ba978c102e9d233490a6ba5f5acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 19:42:09 +0900 Subject: [PATCH 48/97] Reposition licenses in core package --- NOTICE.md | 16 +++++++++++++--- .../licenses => licenses/core}/serde-license.md | 0 .../core}/serde_json5-license.md | 0 3 files changed, 13 insertions(+), 3 deletions(-) rename {core/json/licenses => licenses/core}/serde-license.md (100%) rename {core/json/licenses => licenses/core}/serde_json5-license.md (100%) diff --git a/NOTICE.md b/NOTICE.md index 666b06930..3eeda2182 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,7 +1,7 @@ Limbo ======= -Please visit out github for more information: +Please visit our GitHub for more information: * https://github.com/tursodatabase/limbo @@ -10,10 +10,20 @@ Dependencies This product depends on Error Prone, distributed by the Error Prone project: -* License: licenses/assertj-license.md (Apache License v2.0) +* License: licenses/bindings/java/assertj-license.md (Apache License v2.0) * Homepage: https://github.com/google/error-prone This product depends on AssertJ, distributed by the AssertJ authors: -* License: licenses/errorprone-license.md (Apache License v2.0) +* License: licenses/bindings/java/errorprone-license.md (Apache License v2.0) * Homepage: https://joel-costigliola.github.io/assertj/ + +This product depends on serde, distributed by the serde-rs project: + +* License: licenses/core/serde-license.md (Apache License v2.0, MIT license) +* Homepage: https://github.com/serde-rs/serde + +This product depends on serde_json5, distributed + +* License: licenses/core/serde_json5-license.md (Apache License v2.0) +* Homepage: https://github.com/google/serde_json5 diff --git a/core/json/licenses/serde-license.md b/licenses/core/serde-license.md similarity index 100% rename from core/json/licenses/serde-license.md rename to licenses/core/serde-license.md diff --git a/core/json/licenses/serde_json5-license.md b/licenses/core/serde_json5-license.md similarity index 100% rename from core/json/licenses/serde_json5-license.md rename to licenses/core/serde_json5-license.md From 8e4f6e7d30cf2a9b51fdd4704f7707ad73cd76ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 19:44:13 +0900 Subject: [PATCH 49/97] Split serde-license.md to serde-apache-license.md and serde-mit-license.md --- NOTICE.md | 3 +- licenses/core/serde-apache-license.md | 176 ++++++++++++++++++ ...{serde-license.md => serde-mit-license.md} | 2 +- 3 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 licenses/core/serde-apache-license.md rename licenses/core/{serde-license.md => serde-mit-license.md} (97%) diff --git a/NOTICE.md b/NOTICE.md index 3eeda2182..c735df441 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -20,7 +20,8 @@ This product depends on AssertJ, distributed by the AssertJ authors: This product depends on serde, distributed by the serde-rs project: -* License: licenses/core/serde-license.md (Apache License v2.0, MIT license) +* License: licenses/core/serde-apache-license.md (Apache License v2.0) +* License: licenses/core/serde-mit-license.md (MIT License) * Homepage: https://github.com/serde-rs/serde This product depends on serde_json5, distributed diff --git a/licenses/core/serde-apache-license.md b/licenses/core/serde-apache-license.md new file mode 100644 index 000000000..038d25d68 --- /dev/null +++ b/licenses/core/serde-apache-license.md @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/licenses/core/serde-license.md b/licenses/core/serde-mit-license.md similarity index 97% rename from licenses/core/serde-license.md rename to licenses/core/serde-mit-license.md index 468cd79a8..31aa79387 100644 --- a/licenses/core/serde-license.md +++ b/licenses/core/serde-mit-license.md @@ -20,4 +20,4 @@ SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. \ No newline at end of file +DEALINGS IN THE SOFTWARE. From e687ae3bdf1e91d4b474cc441a70ce45f3eaec26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 19:51:50 +0900 Subject: [PATCH 50/97] Update README.md --- CONTRIBUTING.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cb59600de..2f2b99922 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -142,4 +142,16 @@ Once Maturin is installed, you can build the crate and install it as a Python mo ```bash cd bindings/python && maturin develop -``` \ No newline at end of file +``` + +## Adding Third Party Dependencies + +When you want to add third party dependencies, please follow these steps: + +1. Add Licenses: Place the appropriate licenses for the third-party dependencies under the licenses directory. Ensure + that each license is in a separate file and named appropriately. +2. Update NOTICE.md: Specify the licenses for the third-party dependencies in the NOTICE.md file. Include the name of + the dependency, the license file path, and the homepage of the dependency. + +By following these steps, you ensure that all third-party dependencies are properly documented and their licenses are +included in the project. From eacd7b79452f528f09e13cb073242f6db8d158e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 20:08:13 +0900 Subject: [PATCH 51/97] Change bindings/java to support java 8 --- .github/workflows/java.yml | 2 +- bindings/java/build.gradle.kts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 4a8b0310d..88e3a3976 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -32,7 +32,7 @@ jobs: uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: '17' + java-version: '8' - name: Run Java tests run: make test diff --git a/bindings/java/build.gradle.kts b/bindings/java/build.gradle.kts index 9936bec60..48abcba05 100644 --- a/bindings/java/build.gradle.kts +++ b/bindings/java/build.gradle.kts @@ -4,15 +4,15 @@ import net.ltgt.gradle.errorprone.errorprone plugins { java application - id("net.ltgt.errorprone") version "4.1.0" + id("net.ltgt.errorprone") version "3.1.0" } group = "org.github.tursodatabase" version = "0.0.1-SNAPSHOT" java { - sourceCompatibility = JavaVersion.VERSION_17 - targetCompatibility = JavaVersion.VERSION_17 + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } repositories { @@ -20,8 +20,8 @@ repositories { } dependencies { - errorprone("com.uber.nullaway:nullaway:0.12.3") - errorprone("com.google.errorprone:error_prone_core:2.36.0") + errorprone("com.uber.nullaway:nullaway:0.10.26") // maximum version which supports java 8 + errorprone("com.google.errorprone:error_prone_core:2.11.0") testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") From b3883d03d6b16b19fa7c1d03677957e48dab6657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Tue, 14 Jan 2025 20:14:32 +0900 Subject: [PATCH 52/97] Apply necessary changes for java 8 --- bindings/java/build.gradle.kts | 2 +- .../main/java/org/github/tursodatabase/LimboConnection.java | 2 +- .../java/org/github/tursodatabase/jdbc4/JDBC4Connection.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bindings/java/build.gradle.kts b/bindings/java/build.gradle.kts index 48abcba05..fcdebad3a 100644 --- a/bindings/java/build.gradle.kts +++ b/bindings/java/build.gradle.kts @@ -21,7 +21,7 @@ repositories { dependencies { errorprone("com.uber.nullaway:nullaway:0.10.26") // maximum version which supports java 8 - errorprone("com.google.errorprone:error_prone_core:2.11.0") + errorprone("com.google.errorprone:error_prone_core:2.10.0") // maximum version which supports java 8 testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java index 5bb5e973f..98f0ad04b 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java @@ -46,7 +46,7 @@ public abstract class LimboConnection implements Connection { } private static AbstractDB open(String url, String fileName, Properties properties) throws SQLException { - if (fileName.isBlank()) { + if (fileName.isEmpty()) { throw new IllegalArgumentException("fileName should not be empty"); } diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index 6ffb41e3e..9e67ae501 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -4,6 +4,7 @@ import org.github.tursodatabase.LimboConnection; import org.github.tursodatabase.annotations.SkipNullableCheck; import java.sql.*; +import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.Executor; @@ -150,7 +151,7 @@ public class JDBC4Connection extends LimboConnection { @Override public Map> getTypeMap() throws SQLException { // TODO - return Map.of(); + return new HashMap<>(); } @Override From 0a10d893d9de12ca206ef810e216cce82f5dd820 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Wed, 8 Jan 2025 23:16:57 -0500 Subject: [PATCH 53/97] Sketch out runtime extension loading --- Cargo.lock | 30 +++++++++++++-- Cargo.toml | 2 +- cli/app.rs | 26 ++++++++++++- core/Cargo.toml | 2 + core/error.rs | 2 + core/ext/mod.rs | 39 ++++++++++++++++++-- core/ext/uuid.rs | 6 +-- core/function.rs | 15 ++++++-- core/lib.rs | 53 ++++++++++++++++++++------- core/types.rs | 45 +++++++++++++++++++++-- core/vdbe/mod.rs | 9 ++++- extension_api/Cargo.toml | 9 +++++ extension_api/src/lib.rs | 75 ++++++++++++++++++++++++++++++++++++++ extensions/uuid/Cargo.toml | 11 ++++++ extensions/uuid/src/lib.rs | 0 15 files changed, 291 insertions(+), 33 deletions(-) create mode 100644 extension_api/Cargo.toml create mode 100644 extension_api/src/lib.rs create mode 100644 extensions/uuid/Cargo.toml create mode 100644 extensions/uuid/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index bb69e5c0f..b3f3bb7e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -564,7 +564,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "uuid", + "uuid 1.11.0", ] [[package]] @@ -694,6 +694,10 @@ dependencies = [ "str-buf", ] +[[package]] +name = "extension_api" +version = "0.0.11" + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1137,6 +1141,16 @@ version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libmimalloc-sys" version = "0.1.39" @@ -1204,6 +1218,7 @@ dependencies = [ "cfg_block", "chrono", "criterion", + "extension_api", "fallible-iterator 0.3.0", "getrandom", "hex", @@ -1212,6 +1227,7 @@ dependencies = [ "jsonb", "julian_day_converter", "libc", + "libloading", "limbo_macros", "log", "miette", @@ -1232,7 +1248,7 @@ dependencies = [ "sqlite3-parser", "tempfile", "thiserror 1.0.69", - "uuid", + "uuid 1.11.0", ] [[package]] @@ -2260,7 +2276,7 @@ dependencies = [ "debugid", "memmap2", "stable_deref_trait", - "uuid", + "uuid 1.11.0", ] [[package]] @@ -2502,6 +2518,14 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "0.0.11" +dependencies = [ + "extension_api", + "uuid 1.11.0", +] + [[package]] name = "uuid" version = "1.11.0" diff --git a/Cargo.toml b/Cargo.toml index 0d2fc81be..40f5b2bd4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ members = [ "sqlite3", "core", "simulator", - "test", "macros", + "test", "macros", "extension_api", "extensions/uuid", ] exclude = ["perf/latency/limbo"] diff --git a/cli/app.rs b/cli/app.rs index f114b63a2..325f6fe19 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -129,6 +129,8 @@ pub enum Command { Tables, /// Import data from FILE into TABLE Import, + /// Loads an extension library + LoadExtension, } impl Command { @@ -141,7 +143,12 @@ impl Command { | Self::ShowInfo | Self::Tables | Self::SetOutput => 0, - Self::Open | Self::OutputMode | Self::Cwd | Self::Echo | Self::NullValue => 1, + Self::Open + | Self::OutputMode + | Self::Cwd + | Self::Echo + | Self::NullValue + | Self::LoadExtension => 1, Self::Import => 2, } + 1) // argv0 } @@ -160,6 +167,7 @@ impl Command { Self::NullValue => ".nullvalue ", Self::Echo => ".echo on|off", Self::Tables => ".tables", + Self::LoadExtension => ".load", Self::Import => &IMPORT_HELP, } } @@ -182,6 +190,7 @@ impl FromStr for Command { ".nullvalue" => Ok(Self::NullValue), ".echo" => Ok(Self::Echo), ".import" => Ok(Self::Import), + ".load" => Ok(Self::LoadExtension), _ => Err("Unknown command".to_string()), } } @@ -314,6 +323,16 @@ impl Limbo { }; } + fn handle_load_extension(&mut self) -> Result<(), String> { + let mut args = self.input_buff.split_whitespace(); + let _ = args.next(); + let lib = args + .next() + .ok_or("No library specified") + .map_err(|e| e.to_string())?; + self.conn.load_extension(lib).map_err(|e| e.to_string()) + } + fn display_in_memory(&mut self) -> std::io::Result<()> { if self.opts.db_file == ":memory:" { self.writeln("Connected to a transient in-memory database.")?; @@ -537,6 +556,11 @@ impl Limbo { let _ = self.writeln(e.to_string()); }; } + Command::LoadExtension => { + if let Err(e) = self.handle_load_extension() { + let _ = self.writeln(e.to_string()); + } + } } } else { let _ = self.write_fmt(format_args!( diff --git a/core/Cargo.toml b/core/Cargo.toml index 0daa58c0d..2bbc1347d 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -35,6 +35,7 @@ rustix = "0.38.34" mimalloc = { version = "*", default-features = false } [dependencies] +extension_api = { path = "../extension_api" } cfg_block = "0.1.1" fallible-iterator = "0.3.0" hex = "0.4.3" @@ -58,6 +59,7 @@ bumpalo = { version = "3.16.0", features = ["collections", "boxed"] } limbo_macros = { path = "../macros" } uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true } miette = "7.4.0" +libloading = "0.8.6" [target.'cfg(not(target_family = "windows"))'.dev-dependencies] pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] } diff --git a/core/error.rs b/core/error.rs index 646e85825..e3e176b79 100644 --- a/core/error.rs +++ b/core/error.rs @@ -39,6 +39,8 @@ pub enum LimboError { InvalidModifier(String), #[error("Runtime error: {0}")] Constraint(String), + #[error("Extension error: {0}")] + ExtensionError(String), } #[macro_export] diff --git a/core/ext/mod.rs b/core/ext/mod.rs index cea65a98d..c1718dcc8 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -1,8 +1,39 @@ #[cfg(feature = "uuid")] mod uuid; +use crate::{function::ExternalFunc, Database}; +use std::sync::Arc; + +use extension_api::{AggregateFunction, ExtensionApi, Result, ScalarFunction, VirtualTable}; #[cfg(feature = "uuid")] pub use uuid::{exec_ts_from_uuid7, exec_uuid, exec_uuidblob, exec_uuidstr, UuidFunc}; +impl ExtensionApi for Database { + fn register_scalar_function( + &self, + name: &str, + func: Arc, + ) -> extension_api::Result<()> { + let ext_func = ExternalFunc::new(name, func.clone()); + self.syms + .borrow_mut() + .functions + .insert(name.to_string(), Arc::new(ext_func)); + Ok(()) + } + + fn register_aggregate_function( + &self, + _name: &str, + _func: Arc, + ) -> Result<()> { + todo!("implement aggregate function registration"); + } + + fn register_virtual_table(&self, _name: &str, _table: Arc) -> Result<()> { + todo!("implement virtual table registration"); + } +} + #[derive(Debug, Clone, PartialEq)] pub enum ExtFunc { #[cfg(feature = "uuid")] @@ -31,7 +62,7 @@ impl ExtFunc { } } -pub fn init(db: &mut crate::Database) { - #[cfg(feature = "uuid")] - uuid::init(db); -} +//pub fn init(db: &mut crate::Database) { +// #[cfg(feature = "uuid")] +// uuid::init(db); +//} diff --git a/core/ext/uuid.rs b/core/ext/uuid.rs index 92fdd831a..37e496f00 100644 --- a/core/ext/uuid.rs +++ b/core/ext/uuid.rs @@ -136,9 +136,9 @@ fn uuid_to_unix(uuid: &[u8; 16]) -> u64 { | (uuid[5] as u64) } -pub fn init(db: &mut Database) { - db.define_scalar_function("uuid4", |_args| exec_uuid4()); -} +//pub fn init(db: &mut Database) { +// db.define_scalar_function("uuid4", |_args| exec_uuid4()); +//} #[cfg(test)] #[cfg(feature = "uuid")] diff --git a/core/function.rs b/core/function.rs index 060a677c3..3987b2585 100644 --- a/core/function.rs +++ b/core/function.rs @@ -1,11 +1,20 @@ use crate::ext::ExtFunc; use std::fmt; use std::fmt::{Debug, Display}; -use std::rc::Rc; +use std::sync::Arc; pub struct ExternalFunc { pub name: String, - pub func: Box crate::Result>, + pub func: Arc, +} + +impl ExternalFunc { + pub fn new(name: &str, func: Arc) -> Self { + Self { + name: name.to_string(), + func, + } + } } impl Debug for ExternalFunc { @@ -300,7 +309,7 @@ pub enum Func { #[cfg(feature = "json")] Json(JsonFunc), Extension(ExtFunc), - External(Rc), + External(Arc), } impl Display for Func { diff --git a/core/lib.rs b/core/lib.rs index a80fab83a..21b7cf102 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -17,7 +17,9 @@ mod vdbe; #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +use extension_api::{Extension, ExtensionApi}; use fallible_iterator::FallibleIterator; +use libloading::{Library, Symbol}; use log::trace; use schema::Schema; use sqlite3_parser::ast; @@ -34,12 +36,11 @@ use storage::pager::allocate_page; use storage::sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE}; pub use storage::wal::WalFile; pub use storage::wal::WalFileShared; +pub use types::Value; use util::parse_schema_rows; -use translate::select::prepare_select_plan; -use types::OwnedValue; - pub use error::LimboError; +use translate::select::prepare_select_plan; pub type Result = std::result::Result; use crate::translate::optimizer::optimize_plan; @@ -56,8 +57,6 @@ pub use storage::pager::Page; pub use storage::pager::Pager; pub use storage::wal::CheckpointStatus; pub use storage::wal::Wal; -pub use types::Value; - pub static DATABASE_VERSION: OnceLock = OnceLock::new(); #[derive(Clone)] @@ -135,11 +134,11 @@ impl Database { _shared_wal: shared_wal.clone(), syms, }; - ext::init(&mut db); + // ext::init(&mut db); let db = Arc::new(db); let conn = Rc::new(Connection { db: db.clone(), - pager: pager, + pager, schema: schema.clone(), header, transaction_state: RefCell::new(TransactionState::None), @@ -169,16 +168,31 @@ impl Database { pub fn define_scalar_function>( &self, name: S, - func: impl Fn(&[Value]) -> Result + 'static, + func: Arc, ) { let func = function::ExternalFunc { name: name.as_ref().to_string(), - func: Box::new(func), + func: func.clone(), }; self.syms .borrow_mut() .functions - .insert(name.as_ref().to_string(), Rc::new(func)); + .insert(name.as_ref().to_string(), Arc::new(func)); + } + + pub fn load_extension(&self, path: &str) -> Result<()> { + let lib = + unsafe { Library::new(path).map_err(|e| LimboError::ExtensionError(e.to_string()))? }; + unsafe { + let register: Symbol Box> = + lib.get(b"register_extension") + .map_err(|e| LimboError::ExtensionError(e.to_string()))?; + let extension = register(self); + extension + .load() + .map_err(|e| LimboError::ExtensionError(e.to_string()))?; + } + Ok(()) } } @@ -372,6 +386,10 @@ impl Connection { Ok(()) } + pub fn load_extension(&self, path: &str) -> Result<()> { + Database::load_extension(self.db.as_ref(), path) + } + /// Close a connection and checkpoint. pub fn close(&self) -> Result<()> { loop { @@ -468,15 +486,24 @@ impl Rows { } } -#[derive(Debug)] pub(crate) struct SymbolTable { - pub functions: HashMap>, + pub functions: HashMap>, + extensions: Vec>, +} + +impl std::fmt::Debug for SymbolTable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SymbolTable") + .field("functions", &self.functions) + .finish() + } } impl SymbolTable { pub fn new() -> Self { Self { functions: HashMap::new(), + extensions: Vec::new(), } } @@ -484,7 +511,7 @@ impl SymbolTable { &self, name: &str, _arg_count: usize, - ) -> Option> { + ) -> Option> { self.functions.get(name).cloned() } } diff --git a/core/types.rs b/core/types.rs index d9a496bfb..3d3756799 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1,8 +1,8 @@ -use std::fmt::Display; -use std::rc::Rc; - use crate::error::LimboError; use crate::Result; +use extension_api::Value as ExtValue; +use std::fmt::Display; +use std::rc::Rc; use crate::storage::sqlite3_ondisk::write_varint; @@ -15,6 +15,45 @@ pub enum Value<'a> { Blob(&'a Vec), } +impl From<&OwnedValue> for extension_api::Value { + fn from(value: &OwnedValue) -> Self { + match value { + OwnedValue::Null => extension_api::Value::Null, + OwnedValue::Integer(i) => extension_api::Value::Integer(*i), + OwnedValue::Float(f) => extension_api::Value::Float(*f), + OwnedValue::Text(text) => extension_api::Value::Text(text.value.to_string()), + OwnedValue::Blob(blob) => extension_api::Value::Blob(blob.to_vec()), + OwnedValue::Agg(_) => { + panic!("Cannot convert Aggregate context to extension_api::Value") + } // Handle appropriately + OwnedValue::Record(_) => panic!("Cannot convert Record to extension_api::Value"), // Handle appropriately + } + } +} +impl From for OwnedValue { + fn from(value: ExtValue) -> Self { + match value { + ExtValue::Null => OwnedValue::Null, + ExtValue::Integer(i) => OwnedValue::Integer(i), + ExtValue::Float(f) => OwnedValue::Float(f), + ExtValue::Text(text) => OwnedValue::Text(LimboText::new(Rc::new(text.to_string()))), + ExtValue::Blob(blob) => OwnedValue::Blob(Rc::new(blob.to_vec())), + } + } +} + +impl<'a> From<&'a crate::Value<'a>> for ExtValue { + fn from(value: &'a crate::Value<'a>) -> Self { + match value { + crate::Value::Null => extension_api::Value::Null, + crate::Value::Integer(i) => extension_api::Value::Integer(*i), + crate::Value::Float(f) => extension_api::Value::Float(*f), + crate::Value::Text(t) => extension_api::Value::Text(t.to_string()), + crate::Value::Blob(b) => extension_api::Value::Blob(b.to_vec()), + } + } +} + impl Display for Value<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 28db889cf..f6903619b 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -1872,8 +1872,13 @@ impl Program { _ => unreachable!(), // when more extension types are added }, crate::function::Func::External(f) => { - let result = (f.func)(&[])?; - state.registers[*dest] = result; + let values = &state.registers[*start_reg..*start_reg + arg_count]; + let args: Vec<_> = values.into_iter().map(|v| v.into()).collect(); + let result = f + .func + .execute(args.as_slice()) + .map_err(|e| LimboError::ExtensionError(e.to_string()))?; + state.registers[*dest] = result.into(); } crate::function::Func::Math(math_func) => match math_func.arity() { MathFuncArity::Nullary => match math_func { diff --git a/extension_api/Cargo.toml b/extension_api/Cargo.toml new file mode 100644 index 000000000..73056af33 --- /dev/null +++ b/extension_api/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "extension_api" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] diff --git a/extension_api/src/lib.rs b/extension_api/src/lib.rs new file mode 100644 index 000000000..7ee26e329 --- /dev/null +++ b/extension_api/src/lib.rs @@ -0,0 +1,75 @@ +use std::any::Any; +use std::rc::Rc; +use std::sync::Arc; + +pub type Result = std::result::Result; + +pub trait Extension { + fn load(&self) -> Result<()>; +} + +#[derive(Debug)] +pub enum LimboApiError { + ConnectionError(String), + RegisterFunctionError(String), + ValueError(String), + VTableError(String), +} + +impl From for LimboApiError { + fn from(e: std::io::Error) -> Self { + Self::ConnectionError(e.to_string()) + } +} + +impl std::fmt::Display for LimboApiError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::ConnectionError(e) => write!(f, "Connection error: {e}"), + Self::RegisterFunctionError(e) => write!(f, "Register function error: {e}"), + Self::ValueError(e) => write!(f, "Value error: {e}"), + Self::VTableError(e) => write!(f, "VTable error: {e}"), + } + } +} + +pub trait ExtensionApi { + fn register_scalar_function(&self, name: &str, func: Arc) -> Result<()>; + fn register_aggregate_function( + &self, + name: &str, + func: Arc, + ) -> Result<()>; + fn register_virtual_table(&self, name: &str, table: Arc) -> Result<()>; +} + +pub trait ScalarFunction { + fn execute(&self, args: &[Value]) -> Result; +} + +pub trait AggregateFunction { + fn init(&self) -> Box; + fn step(&self, state: &mut dyn Any, args: &[Value]) -> Result<()>; + fn finalize(&self, state: Box) -> Result; +} + +pub trait VirtualTable { + fn schema(&self) -> &'static str; + fn create_cursor(&self) -> Box; +} + +pub trait Cursor { + fn next(&mut self) -> Result>; +} + +pub struct Row { + pub values: Vec, +} + +pub enum Value { + Text(String), + Blob(Vec), + Integer(i64), + Float(f64), + Null, +} diff --git a/extensions/uuid/Cargo.toml b/extensions/uuid/Cargo.toml new file mode 100644 index 000000000..c6ae90bdf --- /dev/null +++ b/extensions/uuid/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "uuid" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +extension_api = { path = "../../extension_api"} +uuid = { version = "1.11.0", features = ["v4", "v7"] } diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs new file mode 100644 index 000000000..e69de29bb From 3412a3d4c26d0d8a5fc8d2b5d2fd1a929f34bc7a Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sat, 11 Jan 2025 22:48:06 -0500 Subject: [PATCH 54/97] Rough design for extension api/draft extension --- Cargo.lock | 33 +-- Cargo.toml | 2 +- cli/app.rs | 14 +- core/Cargo.toml | 2 +- core/ext/mod.rs | 93 +++---- core/function.rs | 18 +- core/lib.rs | 45 ++-- core/translate/emitter.rs | 9 +- core/translate/expr.rs | 56 ----- core/translate/planner.rs | 19 +- core/translate/select.rs | 30 ++- core/types.rs | 79 +++--- core/vdbe/mod.rs | 57 ++--- extension_api/src/lib.rs | 75 ------ extensions/uuid/Cargo.toml | 8 +- extensions/uuid/src/lib.rs | 62 +++++ {extension_api => limbo_extension}/Cargo.toml | 2 +- limbo_extension/src/lib.rs | 233 ++++++++++++++++++ 18 files changed, 489 insertions(+), 348 deletions(-) delete mode 100644 extension_api/src/lib.rs rename {extension_api => limbo_extension}/Cargo.toml (86%) create mode 100644 limbo_extension/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index b3f3bb7e0..eb09669a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -564,7 +564,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "uuid 1.11.0", + "uuid", ] [[package]] @@ -694,10 +694,6 @@ dependencies = [ "str-buf", ] -[[package]] -name = "extension_api" -version = "0.0.11" - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1218,7 +1214,6 @@ dependencies = [ "cfg_block", "chrono", "criterion", - "extension_api", "fallible-iterator 0.3.0", "getrandom", "hex", @@ -1228,6 +1223,7 @@ dependencies = [ "julian_day_converter", "libc", "libloading", + "limbo_extension", "limbo_macros", "log", "miette", @@ -1248,10 +1244,11 @@ dependencies = [ "sqlite3-parser", "tempfile", "thiserror 1.0.69", - "uuid 1.11.0", + "uuid", ] [[package]] +<<<<<<< HEAD name = "limbo_libsql" version = "0.0.12" dependencies = [ @@ -1260,6 +1257,10 @@ dependencies = [ "tokio", ] +[[package]] +name = "limbo_extension" +version = "0.0.11" + [[package]] name = "limbo_macros" version = "0.0.12" @@ -1288,6 +1289,14 @@ dependencies = [ "log", ] +[[package]] +name = "limbo_uuid" +version = "0.0.11" +dependencies = [ + "limbo_extension", + "uuid", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -2276,7 +2285,7 @@ dependencies = [ "debugid", "memmap2", "stable_deref_trait", - "uuid 1.11.0", + "uuid", ] [[package]] @@ -2518,14 +2527,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "0.0.11" -dependencies = [ - "extension_api", - "uuid 1.11.0", -] - [[package]] name = "uuid" version = "1.11.0" diff --git a/Cargo.toml b/Cargo.toml index 40f5b2bd4..44ec1ef15 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ members = [ "sqlite3", "core", "simulator", - "test", "macros", "extension_api", "extensions/uuid", + "test", "macros", "limbo_extension", "extensions/uuid", ] exclude = ["perf/latency/limbo"] diff --git a/cli/app.rs b/cli/app.rs index 325f6fe19..62108ca6d 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -323,14 +323,8 @@ impl Limbo { }; } - fn handle_load_extension(&mut self) -> Result<(), String> { - let mut args = self.input_buff.split_whitespace(); - let _ = args.next(); - let lib = args - .next() - .ok_or("No library specified") - .map_err(|e| e.to_string())?; - self.conn.load_extension(lib).map_err(|e| e.to_string()) + fn handle_load_extension(&mut self, path: &str) -> Result<(), String> { + self.conn.load_extension(path).map_err(|e| e.to_string()) } fn display_in_memory(&mut self) -> std::io::Result<()> { @@ -557,8 +551,8 @@ impl Limbo { }; } Command::LoadExtension => { - if let Err(e) = self.handle_load_extension() { - let _ = self.writeln(e.to_string()); + if let Err(e) = self.handle_load_extension(args[1]) { + let _ = self.writeln(&e); } } } diff --git a/core/Cargo.toml b/core/Cargo.toml index 2bbc1347d..c6042a298 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -35,7 +35,7 @@ rustix = "0.38.34" mimalloc = { version = "*", default-features = false } [dependencies] -extension_api = { path = "../extension_api" } +limbo_extension = { path = "../limbo_extension" } cfg_block = "0.1.1" fallible-iterator = "0.3.0" hex = "0.4.3" diff --git a/core/ext/mod.rs b/core/ext/mod.rs index c1718dcc8..79936079e 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -1,68 +1,41 @@ -#[cfg(feature = "uuid")] -mod uuid; use crate::{function::ExternalFunc, Database}; -use std::sync::Arc; +use limbo_extension::{ExtensionApi, ResultCode, ScalarFunction, RESULT_ERROR, RESULT_OK}; +pub use limbo_extension::{Value as ExtValue, ValueType as ExtValueType}; +use std::{ + ffi::{c_char, c_void, CStr}, + rc::Rc, +}; -use extension_api::{AggregateFunction, ExtensionApi, Result, ScalarFunction, VirtualTable}; -#[cfg(feature = "uuid")] -pub use uuid::{exec_ts_from_uuid7, exec_uuid, exec_uuidblob, exec_uuidstr, UuidFunc}; - -impl ExtensionApi for Database { - fn register_scalar_function( - &self, - name: &str, - func: Arc, - ) -> extension_api::Result<()> { - let ext_func = ExternalFunc::new(name, func.clone()); - self.syms - .borrow_mut() - .functions - .insert(name.to_string(), Arc::new(ext_func)); - Ok(()) - } - - fn register_aggregate_function( - &self, - _name: &str, - _func: Arc, - ) -> Result<()> { - todo!("implement aggregate function registration"); - } - - fn register_virtual_table(&self, _name: &str, _table: Arc) -> Result<()> { - todo!("implement virtual table registration"); - } +extern "C" fn register_scalar_function( + ctx: *mut c_void, + name: *const c_char, + func: ScalarFunction, +) -> ResultCode { + let c_str = unsafe { CStr::from_ptr(name) }; + let name_str = match c_str.to_str() { + Ok(s) => s.to_string(), + Err(_) => return RESULT_ERROR, + }; + let db = unsafe { &*(ctx as *const Database) }; + db.register_scalar_function_impl(name_str, func) } -#[derive(Debug, Clone, PartialEq)] -pub enum ExtFunc { - #[cfg(feature = "uuid")] - Uuid(UuidFunc), -} +impl Database { + fn register_scalar_function_impl(&self, name: String, func: ScalarFunction) -> ResultCode { + self.syms.borrow_mut().functions.insert( + name.to_string(), + Rc::new(ExternalFunc { + name: name.to_string(), + func, + }), + ); + RESULT_OK + } -#[allow(unreachable_patterns)] // TODO: remove when more extension funcs added -impl std::fmt::Display for ExtFunc { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - #[cfg(feature = "uuid")] - Self::Uuid(uuidfn) => write!(f, "{}", uuidfn), - _ => write!(f, "unknown"), + pub fn build_limbo_extension(&self) -> ExtensionApi { + ExtensionApi { + ctx: self as *const _ as *mut c_void, + register_scalar_function, } } } - -#[allow(unreachable_patterns)] -impl ExtFunc { - pub fn resolve_function(name: &str, num_args: usize) -> Option { - match name { - #[cfg(feature = "uuid")] - name => UuidFunc::resolve_function(name, num_args), - _ => None, - } - } -} - -//pub fn init(db: &mut crate::Database) { -// #[cfg(feature = "uuid")] -// uuid::init(db); -//} diff --git a/core/function.rs b/core/function.rs index 3987b2585..68b5ef005 100644 --- a/core/function.rs +++ b/core/function.rs @@ -1,15 +1,16 @@ -use crate::ext::ExtFunc; use std::fmt; use std::fmt::{Debug, Display}; -use std::sync::Arc; +use std::rc::Rc; + +use limbo_extension::ScalarFunction; pub struct ExternalFunc { pub name: String, - pub func: Arc, + pub func: ScalarFunction, } impl ExternalFunc { - pub fn new(name: &str, func: Arc) -> Self { + pub fn new(name: &str, func: ScalarFunction) -> Self { Self { name: name.to_string(), func, @@ -308,8 +309,7 @@ pub enum Func { Math(MathFunc), #[cfg(feature = "json")] Json(JsonFunc), - Extension(ExtFunc), - External(Arc), + External(Rc), } impl Display for Func { @@ -320,7 +320,6 @@ impl Display for Func { Self::Math(math_func) => write!(f, "{}", math_func), #[cfg(feature = "json")] Self::Json(json_func) => write!(f, "{}", json_func), - Self::Extension(ext_func) => write!(f, "{}", ext_func), Self::External(generic_func) => write!(f, "{}", generic_func), } } @@ -427,10 +426,7 @@ impl Func { "tan" => Ok(Self::Math(MathFunc::Tan)), "tanh" => Ok(Self::Math(MathFunc::Tanh)), "trunc" => Ok(Self::Math(MathFunc::Trunc)), - _ => match ExtFunc::resolve_function(name, arg_count) { - Some(ext_func) => Ok(Self::Extension(ext_func)), - None => Err(()), - }, + _ => Err(()), } } } diff --git a/core/lib.rs b/core/lib.rs index 21b7cf102..4b3e8155b 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -17,9 +17,9 @@ mod vdbe; #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; -use extension_api::{Extension, ExtensionApi}; use fallible_iterator::FallibleIterator; use libloading::{Library, Symbol}; +use limbo_extension::{ExtensionApi, ExtensionEntryPoint, RESULT_OK}; use log::trace; use schema::Schema; use sqlite3_parser::ast; @@ -134,7 +134,6 @@ impl Database { _shared_wal: shared_wal.clone(), syms, }; - // ext::init(&mut db); let db = Arc::new(db); let conn = Rc::new(Connection { db: db.clone(), @@ -168,31 +167,37 @@ impl Database { pub fn define_scalar_function>( &self, name: S, - func: Arc, + func: limbo_extension::ScalarFunction, ) { let func = function::ExternalFunc { name: name.as_ref().to_string(), - func: func.clone(), + func, }; self.syms .borrow_mut() .functions - .insert(name.as_ref().to_string(), Arc::new(func)); + .insert(name.as_ref().to_string(), func.into()); } pub fn load_extension(&self, path: &str) -> Result<()> { + let api = Box::new(self.build_limbo_extension()); let lib = unsafe { Library::new(path).map_err(|e| LimboError::ExtensionError(e.to_string()))? }; - unsafe { - let register: Symbol Box> = - lib.get(b"register_extension") - .map_err(|e| LimboError::ExtensionError(e.to_string()))?; - let extension = register(self); - extension - .load() - .map_err(|e| LimboError::ExtensionError(e.to_string()))?; + let entry: Symbol = unsafe { + lib.get(b"register_extension") + .map_err(|e| LimboError::ExtensionError(e.to_string()))? + }; + let api_ptr: *const ExtensionApi = Box::into_raw(api); + let result_code = entry(api_ptr); + if result_code == RESULT_OK { + self.syms.borrow_mut().extensions.push((lib, api_ptr)); + Ok(()) + } else { + let _ = unsafe { Box::from_raw(api_ptr.cast_mut()) }; // own this again so we dont leak + Err(LimboError::ExtensionError( + "Extension registration failed".to_string(), + )) } - Ok(()) } } @@ -321,7 +326,11 @@ impl Connection { Cmd::ExplainQueryPlan(stmt) => { match stmt { ast::Stmt::Select(select) => { - let mut plan = prepare_select_plan(&self.schema.borrow(), *select)?; + let mut plan = prepare_select_plan( + &self.schema.borrow(), + *select, + &self.db.syms.borrow(), + )?; optimize_plan(&mut plan)?; println!("{}", plan); } @@ -487,8 +496,8 @@ impl Rows { } pub(crate) struct SymbolTable { - pub functions: HashMap>, - extensions: Vec>, + pub functions: HashMap>, + extensions: Vec<(libloading::Library, *const ExtensionApi)>, } impl std::fmt::Debug for SymbolTable { @@ -511,7 +520,7 @@ impl SymbolTable { &self, name: &str, _arg_count: usize, - ) -> Option> { + ) -> Option> { self.functions.get(name).cloned() } } diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index e376da160..aa1aaecf9 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -35,14 +35,13 @@ impl<'a> Resolver<'a> { } pub fn resolve_function(&self, func_name: &str, arg_count: usize) -> Option { - let func_type = match Func::resolve_function(&func_name, arg_count).ok() { + match Func::resolve_function(func_name, arg_count).ok() { Some(func) => Some(func), None => self .symbol_table - .resolve_function(&func_name, arg_count) - .map(|func| Func::External(func)), - }; - func_type + .resolve_function(func_name, arg_count) + .map(|arg| Func::External(arg.clone())), + } } pub fn resolve_cached_expr_reg(&self, expr: &ast::Expr) -> Option { diff --git a/core/translate/expr.rs b/core/translate/expr.rs index dd35a5d26..a464e5279 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -1,7 +1,5 @@ use sqlite3_parser::ast::{self, UnaryOperator}; -#[cfg(feature = "uuid")] -use crate::ext::{ExtFunc, UuidFunc}; #[cfg(feature = "json")] use crate::function::JsonFunc; use crate::function::{Func, FuncCtx, MathFuncArity, ScalarFunc}; @@ -1428,60 +1426,6 @@ pub fn translate_expr( } } } - Func::Extension(ext_func) => match ext_func { - #[cfg(feature = "uuid")] - ExtFunc::Uuid(ref uuid_fn) => match uuid_fn { - UuidFunc::UuidStr | UuidFunc::UuidBlob | UuidFunc::Uuid7TS => { - let args = expect_arguments_exact!(args, 1, ext_func); - let regs = program.alloc_register(); - translate_expr(program, referenced_tables, &args[0], regs, resolver)?; - program.emit_insn(Insn::Function { - constant_mask: 0, - start_reg: regs, - dest: target_register, - func: func_ctx, - }); - Ok(target_register) - } - UuidFunc::Uuid4Str => { - if args.is_some() { - crate::bail_parse_error!( - "{} function with arguments", - ext_func.to_string() - ); - } - let regs = program.alloc_register(); - program.emit_insn(Insn::Function { - constant_mask: 0, - start_reg: regs, - dest: target_register, - func: func_ctx, - }); - Ok(target_register) - } - UuidFunc::Uuid7 => { - let args = expect_arguments_max!(args, 1, ext_func); - let mut start_reg = None; - if let Some(arg) = args.first() { - start_reg = Some(translate_and_mark( - program, - referenced_tables, - arg, - resolver, - )?); - } - program.emit_insn(Insn::Function { - constant_mask: 0, - start_reg: start_reg.unwrap_or(target_register), - dest: target_register, - func: func_ctx, - }); - Ok(target_register) - } - }, - #[allow(unreachable_patterns)] - _ => unreachable!("{ext_func} not implemented yet"), - }, Func::Math(math_func) => match math_func.arity() { MathFuncArity::Nullary => { if args.is_some() { diff --git a/core/translate/planner.rs b/core/translate/planner.rs index 64a0ffb04..f5c835f8e 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -1,6 +1,7 @@ use super::{ plan::{Aggregate, Plan, SelectQueryType, SourceOperator, TableReference, TableReferenceType}, select::prepare_select_plan, + SymbolTable, }; use crate::{ function::Func, @@ -259,6 +260,7 @@ fn parse_from_clause_table( table: ast::SelectTable, operator_id_counter: &mut OperatorIdCounter, cur_table_index: usize, + syms: &SymbolTable, ) -> Result<(TableReference, SourceOperator)> { match table { ast::SelectTable::Table(qualified_name, maybe_alias, _) => { @@ -289,7 +291,7 @@ fn parse_from_clause_table( )) } ast::SelectTable::Select(subselect, maybe_alias) => { - let Plan::Select(mut subplan) = prepare_select_plan(schema, *subselect)? else { + let Plan::Select(mut subplan) = prepare_select_plan(schema, *subselect, syms)? else { unreachable!(); }; subplan.query_type = SelectQueryType::Subquery { @@ -322,6 +324,7 @@ pub fn parse_from( schema: &Schema, mut from: Option, operator_id_counter: &mut OperatorIdCounter, + syms: &SymbolTable, ) -> Result<(SourceOperator, Vec)> { if from.as_ref().and_then(|f| f.select.as_ref()).is_none() { return Ok(( @@ -339,7 +342,7 @@ pub fn parse_from( let select_owned = *std::mem::take(&mut from_owned.select).unwrap(); let joins_owned = std::mem::take(&mut from_owned.joins).unwrap_or_default(); let (table_reference, mut operator) = - parse_from_clause_table(schema, select_owned, operator_id_counter, table_index)?; + parse_from_clause_table(schema, select_owned, operator_id_counter, table_index, syms)?; tables.push(table_reference); table_index += 1; @@ -350,7 +353,14 @@ pub fn parse_from( is_outer_join: outer, using, predicates, - } = parse_join(schema, join, operator_id_counter, &mut tables, table_index)?; + } = parse_join( + schema, + join, + operator_id_counter, + &mut tables, + table_index, + syms, + )?; operator = SourceOperator::Join { left: Box::new(operator), right: Box::new(right), @@ -394,6 +404,7 @@ fn parse_join( operator_id_counter: &mut OperatorIdCounter, tables: &mut Vec, table_index: usize, + syms: &SymbolTable, ) -> Result { let ast::JoinedSelectTable { operator: join_operator, @@ -402,7 +413,7 @@ fn parse_join( } = join; let (table_reference, source_operator) = - parse_from_clause_table(schema, table, operator_id_counter, table_index)?; + parse_from_clause_table(schema, table, operator_id_counter, table_index, syms)?; tables.push(table_reference); diff --git a/core/translate/select.rs b/core/translate/select.rs index b1be01169..dfbd4c2fb 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -20,12 +20,15 @@ pub fn translate_select( select: ast::Select, syms: &SymbolTable, ) -> Result<()> { - let mut select_plan = prepare_select_plan(schema, select)?; optimize_plan(&mut select_plan)?; emit_program(program, select_plan, syms) } -pub fn prepare_select_plan(schema: &Schema, select: ast::Select) -> Result { +pub fn prepare_select_plan( + schema: &Schema, + select: ast::Select, + syms: &SymbolTable, +) -> Result { match *select.body.select { ast::OneSelect::Select { mut columns, @@ -42,7 +45,8 @@ pub fn prepare_select_plan(schema: &Schema, select: ast::Select) -> Result let mut operator_id_counter = OperatorIdCounter::new(); // Parse the FROM clause - let (source, referenced_tables) = parse_from(schema, from, &mut operator_id_counter)?; + let (source, referenced_tables) = + parse_from(schema, from, &mut operator_id_counter, syms)?; let mut plan = SelectPlan { source, @@ -142,7 +146,25 @@ pub fn prepare_select_plan(schema: &Schema, select: ast::Select) -> Result contains_aggregates, }); } - _ => {} + Err(_) => { + if syms.functions.contains_key(&name.0) { + // TODO: future extensions can be aggregate functions + log::debug!( + "Resolving {} function from symbol table", + name.0 + ); + plan.result_columns.push(ResultSetColumn { + name: get_name( + maybe_alias.as_ref(), + expr, + &plan.referenced_tables, + || format!("expr_{}", result_column_idx), + ), + expr: expr.clone(), + contains_aggregates: false, + }); + } + } } } ast::Expr::FunctionCallStar { diff --git a/core/types.rs b/core/types.rs index 3d3756799..b35d0dc98 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1,11 +1,10 @@ use crate::error::LimboError; +use crate::ext::{ExtValue, ExtValueType}; +use crate::storage::sqlite3_ondisk::write_varint; use crate::Result; -use extension_api::Value as ExtValue; use std::fmt::Display; use std::rc::Rc; -use crate::storage::sqlite3_ondisk::write_varint; - #[derive(Debug, Clone, PartialEq)] pub enum Value<'a> { Null, @@ -15,45 +14,6 @@ pub enum Value<'a> { Blob(&'a Vec), } -impl From<&OwnedValue> for extension_api::Value { - fn from(value: &OwnedValue) -> Self { - match value { - OwnedValue::Null => extension_api::Value::Null, - OwnedValue::Integer(i) => extension_api::Value::Integer(*i), - OwnedValue::Float(f) => extension_api::Value::Float(*f), - OwnedValue::Text(text) => extension_api::Value::Text(text.value.to_string()), - OwnedValue::Blob(blob) => extension_api::Value::Blob(blob.to_vec()), - OwnedValue::Agg(_) => { - panic!("Cannot convert Aggregate context to extension_api::Value") - } // Handle appropriately - OwnedValue::Record(_) => panic!("Cannot convert Record to extension_api::Value"), // Handle appropriately - } - } -} -impl From for OwnedValue { - fn from(value: ExtValue) -> Self { - match value { - ExtValue::Null => OwnedValue::Null, - ExtValue::Integer(i) => OwnedValue::Integer(i), - ExtValue::Float(f) => OwnedValue::Float(f), - ExtValue::Text(text) => OwnedValue::Text(LimboText::new(Rc::new(text.to_string()))), - ExtValue::Blob(blob) => OwnedValue::Blob(Rc::new(blob.to_vec())), - } - } -} - -impl<'a> From<&'a crate::Value<'a>> for ExtValue { - fn from(value: &'a crate::Value<'a>) -> Self { - match value { - crate::Value::Null => extension_api::Value::Null, - crate::Value::Integer(i) => extension_api::Value::Integer(*i), - crate::Value::Float(f) => extension_api::Value::Float(*f), - crate::Value::Text(t) => extension_api::Value::Text(t.to_string()), - crate::Value::Blob(b) => extension_api::Value::Blob(b.to_vec()), - } - } -} - impl Display for Value<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -132,6 +92,41 @@ impl Display for OwnedValue { } } } +impl OwnedValue { + pub fn to_ffi(&self) -> ExtValue { + match self { + Self::Null => ExtValue::null(), + Self::Integer(i) => ExtValue::from_integer(*i), + Self::Float(fl) => ExtValue::from_float(*fl), + Self::Text(s) => ExtValue::from_text(s.value.to_string()), + Self::Blob(b) => ExtValue::from_blob(b), + Self::Agg(_) => todo!(), + Self::Record(_) => todo!(), + } + } + pub fn from_ffi(v: &ExtValue) -> Self { + match v.value_type { + ExtValueType::Null => OwnedValue::Null, + ExtValueType::Integer => OwnedValue::Integer(v.integer), + ExtValueType::Float => OwnedValue::Float(v.float), + ExtValueType::Text => { + if v.text.is_null() { + OwnedValue::Null + } else { + OwnedValue::build_text(std::rc::Rc::new(v.text.to_string())) + } + } + ExtValueType::Blob => { + if v.blob.data.is_null() { + OwnedValue::Null + } else { + let bytes = unsafe { std::slice::from_raw_parts(v.blob.data, v.blob.size) }; + OwnedValue::Blob(std::rc::Rc::new(bytes.to_vec())) + } + } + } + } +} #[derive(Debug, Clone, PartialEq)] pub enum AggContext { diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index f6903619b..075dd5dab 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -25,8 +25,7 @@ pub mod likeop; pub mod sorter; use crate::error::{LimboError, SQLITE_CONSTRAINT_PRIMARYKEY}; -#[cfg(feature = "uuid")] -use crate::ext::{exec_ts_from_uuid7, exec_uuid, exec_uuidblob, exec_uuidstr, ExtFunc, UuidFunc}; +use crate::ext::ExtValue; use crate::function::{AggFunc, FuncCtx, MathFunc, MathFuncArity, ScalarFunc}; use crate::pseudo::PseudoCursor; use crate::result::LimboResult; @@ -53,9 +52,10 @@ use rand::distributions::{Distribution, Uniform}; use rand::{thread_rng, Rng}; use regex::{Regex, RegexBuilder}; use sorter::Sorter; -use std::borrow::{Borrow, BorrowMut}; +use std::borrow::BorrowMut; use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; +use std::os::raw::c_void; use std::rc::{Rc, Weak}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -1838,47 +1838,20 @@ impl Program { state.registers[*dest] = exec_replace(source, pattern, replacement); } }, - #[allow(unreachable_patterns)] - crate::function::Func::Extension(extfn) => match extfn { - #[cfg(feature = "uuid")] - ExtFunc::Uuid(uuidfn) => match uuidfn { - UuidFunc::Uuid4Str => { - state.registers[*dest] = exec_uuid(uuidfn, None)? - } - UuidFunc::Uuid7 => match arg_count { - 0 => { - state.registers[*dest] = - exec_uuid(uuidfn, None).unwrap_or(OwnedValue::Null); - } - 1 => { - let reg_value = state.registers[*start_reg].borrow(); - state.registers[*dest] = exec_uuid(uuidfn, Some(reg_value)) - .unwrap_or(OwnedValue::Null); - } - _ => unreachable!(), - }, - _ => { - // remaining accept 1 arg - let reg_value = state.registers[*start_reg].borrow(); - state.registers[*dest] = match uuidfn { - UuidFunc::Uuid7TS => Some(exec_ts_from_uuid7(reg_value)), - UuidFunc::UuidStr => exec_uuidstr(reg_value).ok(), - UuidFunc::UuidBlob => exec_uuidblob(reg_value).ok(), - _ => unreachable!(), - } - .unwrap_or(OwnedValue::Null); - } - }, - _ => unreachable!(), // when more extension types are added - }, crate::function::Func::External(f) => { let values = &state.registers[*start_reg..*start_reg + arg_count]; - let args: Vec<_> = values.into_iter().map(|v| v.into()).collect(); - let result = f - .func - .execute(args.as_slice()) - .map_err(|e| LimboError::ExtensionError(e.to_string()))?; - state.registers[*dest] = result.into(); + let c_values: Vec<*const c_void> = values + .iter() + .map(|ov| &ov.to_ffi() as *const _ as *const c_void) + .collect(); + let argv_ptr = if c_values.is_empty() { + std::ptr::null() + } else { + c_values.as_ptr() + }; + let result_c_value: ExtValue = (f.func)(arg_count as i32, argv_ptr); + let result_ov = OwnedValue::from_ffi(&result_c_value); + state.registers[*dest] = result_ov; } crate::function::Func::Math(math_func) => match math_func.arity() { MathFuncArity::Nullary => match math_func { diff --git a/extension_api/src/lib.rs b/extension_api/src/lib.rs deleted file mode 100644 index 7ee26e329..000000000 --- a/extension_api/src/lib.rs +++ /dev/null @@ -1,75 +0,0 @@ -use std::any::Any; -use std::rc::Rc; -use std::sync::Arc; - -pub type Result = std::result::Result; - -pub trait Extension { - fn load(&self) -> Result<()>; -} - -#[derive(Debug)] -pub enum LimboApiError { - ConnectionError(String), - RegisterFunctionError(String), - ValueError(String), - VTableError(String), -} - -impl From for LimboApiError { - fn from(e: std::io::Error) -> Self { - Self::ConnectionError(e.to_string()) - } -} - -impl std::fmt::Display for LimboApiError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::ConnectionError(e) => write!(f, "Connection error: {e}"), - Self::RegisterFunctionError(e) => write!(f, "Register function error: {e}"), - Self::ValueError(e) => write!(f, "Value error: {e}"), - Self::VTableError(e) => write!(f, "VTable error: {e}"), - } - } -} - -pub trait ExtensionApi { - fn register_scalar_function(&self, name: &str, func: Arc) -> Result<()>; - fn register_aggregate_function( - &self, - name: &str, - func: Arc, - ) -> Result<()>; - fn register_virtual_table(&self, name: &str, table: Arc) -> Result<()>; -} - -pub trait ScalarFunction { - fn execute(&self, args: &[Value]) -> Result; -} - -pub trait AggregateFunction { - fn init(&self) -> Box; - fn step(&self, state: &mut dyn Any, args: &[Value]) -> Result<()>; - fn finalize(&self, state: Box) -> Result; -} - -pub trait VirtualTable { - fn schema(&self) -> &'static str; - fn create_cursor(&self) -> Box; -} - -pub trait Cursor { - fn next(&mut self) -> Result>; -} - -pub struct Row { - pub values: Vec, -} - -pub enum Value { - Text(String), - Blob(Vec), - Integer(i64), - Float(f64), - Null, -} diff --git a/extensions/uuid/Cargo.toml b/extensions/uuid/Cargo.toml index c6ae90bdf..1b9eb10a2 100644 --- a/extensions/uuid/Cargo.toml +++ b/extensions/uuid/Cargo.toml @@ -1,11 +1,15 @@ [package] -name = "uuid" +name = "limbo_uuid" version.workspace = true authors.workspace = true edition.workspace = true license.workspace = true repository.workspace = true +[lib] +crate-type = ["cdylib"] + + [dependencies] -extension_api = { path = "../../extension_api"} +limbo_extension = { path = "../../limbo_extension"} uuid = { version = "1.11.0", features = ["v4", "v7"] } diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index e69de29bb..c9950be8f 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -0,0 +1,62 @@ +use limbo_extension::{ + declare_scalar_functions, register_extension, register_scalar_functions, Value, +}; + +register_extension! { + scalars: { + "uuid4_str" => uuid4_str, + "uuid4" => uuid4_blob, + "uuid_str" => uuid_str, + "uuid_blob" => uuid_blob, + }, +} + +declare_scalar_functions! { + #[args(min = 0, max = 0)] + fn uuid4_str(_args: &[Value]) -> Value { + let uuid = uuid::Uuid::new_v4().to_string(); + Value::from_text(uuid) + } + #[args(min = 0, max = 0)] + fn uuid4_blob(_args: &[Value]) -> Value { + let uuid = uuid::Uuid::new_v4(); + let bytes = uuid.as_bytes(); + Value::from_blob(bytes) + } + + #[args(min = 1, max = 1)] + fn uuid_str(args: &[Value]) -> Value { + if args.len() != 1 { + return Value::null(); + } + if args[0].value_type != limbo_extension::ValueType::Blob { + return Value::null(); + } + let data_ptr = args[0].blob.data; + let size = args[0].blob.size; + if data_ptr.is_null() || size != 16 { + return Value::null(); + } + let slice = unsafe{ std::slice::from_raw_parts(data_ptr, size)}; + let parsed = uuid::Uuid::from_slice(slice).ok().map(|u| u.to_string()); + match parsed { + Some(s) => Value::from_text(s), + None => Value::null() + } + } + + #[args(min = 1, max = 1)] + fn uuid_blob(args: &[Value]) -> Value { + if args.len() != 1 { + return Value::null(); + } + if args[0].value_type != limbo_extension::ValueType::Text { + return Value::null(); + } + let text = args[0].text.to_string(); + match uuid::Uuid::parse_str(&text) { + Ok(uuid) => Value::from_blob(uuid.as_bytes()), + Err(_) => Value::null() + } + } +} diff --git a/extension_api/Cargo.toml b/limbo_extension/Cargo.toml similarity index 86% rename from extension_api/Cargo.toml rename to limbo_extension/Cargo.toml index 73056af33..d3ac246d2 100644 --- a/extension_api/Cargo.toml +++ b/limbo_extension/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "extension_api" +name = "limbo_extension" version.workspace = true authors.workspace = true edition.workspace = true diff --git a/limbo_extension/src/lib.rs b/limbo_extension/src/lib.rs new file mode 100644 index 000000000..6704f63ec --- /dev/null +++ b/limbo_extension/src/lib.rs @@ -0,0 +1,233 @@ +use std::ffi::CString; +use std::os::raw::{c_char, c_void}; + +pub type ResultCode = i32; + +pub const RESULT_OK: ResultCode = 0; +pub const RESULT_ERROR: ResultCode = 1; +// TODO: more error types + +pub type ExtensionEntryPoint = extern "C" fn(api: *const ExtensionApi) -> ResultCode; +pub type ScalarFunction = extern "C" fn(argc: i32, *const *const c_void) -> Value; + +#[repr(C)] +pub struct ExtensionApi { + pub ctx: *mut c_void, + pub register_scalar_function: + extern "C" fn(ctx: *mut c_void, name: *const c_char, func: ScalarFunction) -> ResultCode, +} + +#[macro_export] +macro_rules! register_extension { + ( + scalars: { $( $scalar_name:expr => $scalar_func:ident ),* $(,)? }, + //aggregates: { $( $agg_name:expr => ($step_func:ident, $finalize_func:ident) ),* $(,)? }, + //virtual_tables: { $( $vt_name:expr => $vt_impl:expr ),* $(,)? } + ) => { + #[no_mangle] + pub unsafe extern "C" fn register_extension(api: *const $crate::ExtensionApi) -> $crate::ResultCode { + if api.is_null() { + return $crate::RESULT_ERROR; + } + + register_scalar_functions! { api, $( $scalar_name => $scalar_func ),* } + // TODO: + //register_aggregate_functions! { $( $agg_name => ($step_func, $finalize_func) ),* } + //register_virtual_tables! { $( $vt_name => $vt_impl ),* } + $crate::RESULT_OK + } + } +} + +#[macro_export] +macro_rules! register_scalar_functions { + ( $api:expr, $( $fname:expr => $fptr:ident ),* ) => { + unsafe { + $( + let cname = std::ffi::CString::new($fname).unwrap(); + ((*$api).register_scalar_function)((*$api).ctx, cname.as_ptr(), $fptr); + )* + } + } +} + +/// Provide a cleaner interface to define scalar functions to extension authors +/// . e.g. +/// ``` +/// fn scalar_func(args: &[Value]) -> Value { +/// if args.len() != 1 { +/// return Value::null(); +/// } +/// Value::from_integer(args[0].integer * 2) +/// } +/// ``` +/// +#[macro_export] +macro_rules! declare_scalar_functions { + ( + $( + #[args(min = $min_args:literal, max = $max_args:literal)] + fn $func_name:ident ($args:ident : &[Value]) -> Value $body:block + )* + ) => { + $( + extern "C" fn $func_name( + argc: i32, + argv: *const *const std::os::raw::c_void + ) -> $crate::Value { + if !($min_args..=$max_args).contains(&argc) { + println!("{}: Invalid argument count", stringify!($func_name)); + return $crate::Value::null();// TODO: error code + } + if argc == 0 || argv.is_null() { + let $args: &[$crate::Value] = &[]; + $body + } else { + unsafe { + let ptr_slice = std::slice::from_raw_parts(argv, argc as usize); + let mut values = Vec::with_capacity(argc as usize); + for &ptr in ptr_slice { + let val_ptr = ptr as *const $crate::Value; + if val_ptr.is_null() { + values.push($crate::Value::null()); + } else { + values.push(std::ptr::read(val_ptr)); + } + } + let $args: &[$crate::Value] = &values[..]; + $body + } + } + } + )* + }; +} + +#[derive(PartialEq, Eq)] +#[repr(C)] +pub enum ValueType { + Null, + Integer, + Float, + Text, + Blob, +} + +// TODO: perf, these can be better expressed +#[repr(C)] +pub struct Value { + pub value_type: ValueType, + pub integer: i64, + pub float: f64, + pub text: TextValue, + pub blob: Blob, +} + +#[repr(C)] +pub struct TextValue { + text: *const c_char, + len: usize, +} + +impl std::fmt::Display for TextValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.text.is_null() { + return write!(f, ""); + } + let slice = unsafe { std::slice::from_raw_parts(self.text as *const u8, self.len) }; + match std::str::from_utf8(slice) { + Ok(s) => write!(f, "{}", s), + Err(e) => write!(f, "", e), + } + } +} + +impl TextValue { + pub fn is_null(&self) -> bool { + self.text.is_null() + } + + pub fn new(text: *const c_char, len: usize) -> Self { + Self { text, len } + } + + pub fn null() -> Self { + Self { + text: std::ptr::null(), + len: 0, + } + } +} + +#[repr(C)] +pub struct Blob { + pub data: *const u8, + pub size: usize, +} + +impl Blob { + pub fn new(data: *const u8, size: usize) -> Self { + Self { data, size } + } + pub fn null() -> Self { + Self { + data: std::ptr::null(), + size: 0, + } + } +} + +impl Value { + pub fn null() -> Self { + Self { + value_type: ValueType::Null, + integer: 0, + float: 0.0, + text: TextValue::null(), + blob: Blob::null(), + } + } + + pub fn from_integer(value: i64) -> Self { + Self { + value_type: ValueType::Integer, + integer: value, + float: 0.0, + text: TextValue::null(), + blob: Blob::null(), + } + } + pub fn from_float(value: f64) -> Self { + Self { + value_type: ValueType::Float, + integer: 0, + float: value, + text: TextValue::null(), + blob: Blob::null(), + } + } + + pub fn from_text(value: String) -> Self { + let cstr = CString::new(&*value).unwrap(); + let ptr = cstr.as_ptr(); + let len = value.len(); + std::mem::forget(cstr); + Self { + value_type: ValueType::Text, + integer: 0, + float: 0.0, + text: TextValue::new(ptr, len), + blob: Blob::null(), + } + } + + pub fn from_blob(value: &[u8]) -> Self { + Self { + value_type: ValueType::Blob, + integer: 0, + float: 0.0, + text: TextValue::null(), + blob: Blob::new(value.as_ptr(), value.len()), + } + } +} From c565fba1951c3a9f92cd3ceed837a96c4bd39385 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 12 Jan 2025 11:35:02 -0500 Subject: [PATCH 55/97] Adjust types in extension API --- Cargo.lock | 4 + core/ext/mod.rs | 4 +- core/types.rs | 47 ++++++++---- extensions/uuid/Cargo.toml | 3 +- extensions/uuid/src/lib.rs | 35 ++++----- limbo_extension/Cargo.toml | 1 + limbo_extension/src/lib.rs | 145 ++++++++++++++++++++----------------- 7 files changed, 139 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb09669a3..d3fffc7a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1260,6 +1260,9 @@ dependencies = [ [[package]] name = "limbo_extension" version = "0.0.11" +dependencies = [ + "log", +] [[package]] name = "limbo_macros" @@ -1294,6 +1297,7 @@ name = "limbo_uuid" version = "0.0.11" dependencies = [ "limbo_extension", + "log", "uuid", ] diff --git a/core/ext/mod.rs b/core/ext/mod.rs index 79936079e..179b3e7c6 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -1,6 +1,8 @@ use crate::{function::ExternalFunc, Database}; +pub use limbo_extension::{ + Blob as ExtBlob, TextValue as ExtTextValue, Value as ExtValue, ValueType as ExtValueType, +}; use limbo_extension::{ExtensionApi, ResultCode, ScalarFunction, RESULT_ERROR, RESULT_OK}; -pub use limbo_extension::{Value as ExtValue, ValueType as ExtValueType}; use std::{ ffi::{c_char, c_void, CStr}, rc::Rc, diff --git a/core/types.rs b/core/types.rs index b35d0dc98..92d02fce5 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1,5 +1,5 @@ use crate::error::LimboError; -use crate::ext::{ExtValue, ExtValueType}; +use crate::ext::{ExtBlob, ExtTextValue, ExtValue, ExtValueType}; use crate::storage::sqlite3_ondisk::write_varint; use crate::Result; use std::fmt::Display; @@ -92,37 +92,54 @@ impl Display for OwnedValue { } } } + impl OwnedValue { pub fn to_ffi(&self) -> ExtValue { match self { Self::Null => ExtValue::null(), Self::Integer(i) => ExtValue::from_integer(*i), Self::Float(fl) => ExtValue::from_float(*fl), - Self::Text(s) => ExtValue::from_text(s.value.to_string()), - Self::Blob(b) => ExtValue::from_blob(b), - Self::Agg(_) => todo!(), - Self::Record(_) => todo!(), + Self::Text(text) => ExtValue::from_text(text.value.to_string()), + Self::Blob(blob) => ExtValue::from_blob(blob.to_vec()), + Self::Agg(_) => todo!("Aggregate values not yet supported"), + Self::Record(_) => todo!("Record values not yet supported"), } } + pub fn from_ffi(v: &ExtValue) -> Self { + if v.value.is_null() { + return OwnedValue::Null; + } match v.value_type { ExtValueType::Null => OwnedValue::Null, - ExtValueType::Integer => OwnedValue::Integer(v.integer), - ExtValueType::Float => OwnedValue::Float(v.float), + ExtValueType::Integer => { + let int_ptr = v.value as *mut i64; + let integer = unsafe { *int_ptr }; + OwnedValue::Integer(integer) + } + ExtValueType::Float => { + let float_ptr = v.value as *mut f64; + let float = unsafe { *float_ptr }; + OwnedValue::Float(float) + } ExtValueType::Text => { - if v.text.is_null() { + if v.value.is_null() { OwnedValue::Null } else { - OwnedValue::build_text(std::rc::Rc::new(v.text.to_string())) + let Some(text) = ExtTextValue::from_value(v) else { + return OwnedValue::Null; + }; + OwnedValue::build_text(std::rc::Rc::new(unsafe { text.as_str().to_string() })) } } ExtValueType::Blob => { - if v.blob.data.is_null() { - OwnedValue::Null - } else { - let bytes = unsafe { std::slice::from_raw_parts(v.blob.data, v.blob.size) }; - OwnedValue::Blob(std::rc::Rc::new(bytes.to_vec())) - } + let blob_ptr = v.value as *mut ExtBlob; + let blob = unsafe { + let slice = + std::slice::from_raw_parts((*blob_ptr).data, (*blob_ptr).size as usize); + slice.to_vec() + }; + OwnedValue::Blob(std::rc::Rc::new(blob)) } } } diff --git a/extensions/uuid/Cargo.toml b/extensions/uuid/Cargo.toml index 1b9eb10a2..ed2c43e87 100644 --- a/extensions/uuid/Cargo.toml +++ b/extensions/uuid/Cargo.toml @@ -7,9 +7,10 @@ license.workspace = true repository.workspace = true [lib] -crate-type = ["cdylib"] +crate-type = ["cdylib", "lib"] [dependencies] limbo_extension = { path = "../../limbo_extension"} uuid = { version = "1.11.0", features = ["v4", "v7"] } +log = "0.4.20" diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index c9950be8f..940658155 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -1,5 +1,5 @@ use limbo_extension::{ - declare_scalar_functions, register_extension, register_scalar_functions, Value, + declare_scalar_functions, register_extension, register_scalar_functions, Blob, TextValue, Value, }; register_extension! { @@ -17,46 +17,47 @@ declare_scalar_functions! { let uuid = uuid::Uuid::new_v4().to_string(); Value::from_text(uuid) } + #[args(min = 0, max = 0)] fn uuid4_blob(_args: &[Value]) -> Value { let uuid = uuid::Uuid::new_v4(); let bytes = uuid.as_bytes(); - Value::from_blob(bytes) + Value::from_blob(bytes.to_vec()) } #[args(min = 1, max = 1)] fn uuid_str(args: &[Value]) -> Value { - if args.len() != 1 { - return Value::null(); - } if args[0].value_type != limbo_extension::ValueType::Blob { + log::debug!("uuid_str was passed a non-blob arg"); return Value::null(); } - let data_ptr = args[0].blob.data; - let size = args[0].blob.size; - if data_ptr.is_null() || size != 16 { - return Value::null(); - } - let slice = unsafe{ std::slice::from_raw_parts(data_ptr, size)}; + if let Some(blob) = Blob::from_value(&args[0]) { + let slice = unsafe{ std::slice::from_raw_parts(blob.data, blob.size as usize)}; let parsed = uuid::Uuid::from_slice(slice).ok().map(|u| u.to_string()); match parsed { Some(s) => Value::from_text(s), None => Value::null() } + } else { + Value::null() + } } #[args(min = 1, max = 1)] fn uuid_blob(args: &[Value]) -> Value { - if args.len() != 1 { - return Value::null(); - } if args[0].value_type != limbo_extension::ValueType::Text { + log::debug!("uuid_blob was passed a non-text arg"); return Value::null(); } - let text = args[0].text.to_string(); - match uuid::Uuid::parse_str(&text) { - Ok(uuid) => Value::from_blob(uuid.as_bytes()), + if let Some(text) = TextValue::from_value(&args[0]) { + match uuid::Uuid::parse_str(unsafe {text.as_str()}) { + Ok(uuid) => { + Value::from_blob(uuid.as_bytes().to_vec()) + } Err(_) => Value::null() } + } else { + Value::null() + } } } diff --git a/limbo_extension/Cargo.toml b/limbo_extension/Cargo.toml index d3ac246d2..2928ed853 100644 --- a/limbo_extension/Cargo.toml +++ b/limbo_extension/Cargo.toml @@ -7,3 +7,4 @@ license.workspace = true repository.workspace = true [dependencies] +log = "0.4.20" diff --git a/limbo_extension/src/lib.rs b/limbo_extension/src/lib.rs index 6704f63ec..3daf99755 100644 --- a/limbo_extension/src/lib.rs +++ b/limbo_extension/src/lib.rs @@ -1,4 +1,3 @@ -use std::ffi::CString; use std::os::raw::{c_char, c_void}; pub type ResultCode = i32; @@ -76,8 +75,7 @@ macro_rules! declare_scalar_functions { argv: *const *const std::os::raw::c_void ) -> $crate::Value { if !($min_args..=$max_args).contains(&argc) { - println!("{}: Invalid argument count", stringify!($func_name)); - return $crate::Value::null();// TODO: error code + return $crate::Value::null(); } if argc == 0 || argv.is_null() { let $args: &[$crate::Value] = &[]; @@ -103,8 +101,8 @@ macro_rules! declare_scalar_functions { }; } -#[derive(PartialEq, Eq)] #[repr(C)] +#[derive(PartialEq, Eq)] pub enum ValueType { Null, Integer, @@ -113,45 +111,20 @@ pub enum ValueType { Blob, } -// TODO: perf, these can be better expressed #[repr(C)] pub struct Value { pub value_type: ValueType, - pub integer: i64, - pub float: f64, - pub text: TextValue, - pub blob: Blob, + pub value: *mut c_void, } #[repr(C)] pub struct TextValue { - text: *const c_char, - len: usize, + pub text: *const u8, + pub len: u32, } -impl std::fmt::Display for TextValue { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if self.text.is_null() { - return write!(f, ""); - } - let slice = unsafe { std::slice::from_raw_parts(self.text as *const u8, self.len) }; - match std::str::from_utf8(slice) { - Ok(s) => write!(f, "{}", s), - Err(e) => write!(f, "", e), - } - } -} - -impl TextValue { - pub fn is_null(&self) -> bool { - self.text.is_null() - } - - pub fn new(text: *const c_char, len: usize) -> Self { - Self { text, len } - } - - pub fn null() -> Self { +impl Default for TextValue { + fn default() -> Self { Self { text: std::ptr::null(), len: 0, @@ -159,21 +132,49 @@ impl TextValue { } } +impl TextValue { + pub fn new(text: *const u8, len: usize) -> Self { + Self { + text, + len: len as u32, + } + } + + pub fn from_value(value: &Value) -> Option<&Self> { + if value.value_type != ValueType::Text { + return None; + } + unsafe { Some(&*(value.value as *const TextValue)) } + } + + /// # Safety + /// The caller must ensure that the text is a valid UTF-8 string + pub unsafe fn as_str(&self) -> &str { + if self.text.is_null() { + return ""; + } + unsafe { + std::str::from_utf8_unchecked(std::slice::from_raw_parts(self.text, self.len as usize)) + } + } +} + #[repr(C)] pub struct Blob { pub data: *const u8, - pub size: usize, + pub size: u64, } impl Blob { - pub fn new(data: *const u8, size: usize) -> Self { + pub fn new(data: *const u8, size: u64) -> Self { Self { data, size } } - pub fn null() -> Self { - Self { - data: std::ptr::null(), - size: 0, + + pub fn from_value(value: &Value) -> Option<&Self> { + if value.value_type != ValueType::Blob { + return None; } + unsafe { Some(&*(value.value as *const Blob)) } } } @@ -181,53 +182,65 @@ impl Value { pub fn null() -> Self { Self { value_type: ValueType::Null, - integer: 0, - float: 0.0, - text: TextValue::null(), - blob: Blob::null(), + value: std::ptr::null_mut(), } } pub fn from_integer(value: i64) -> Self { + let boxed = Box::new(value); Self { value_type: ValueType::Integer, - integer: value, - float: 0.0, - text: TextValue::null(), - blob: Blob::null(), + value: Box::into_raw(boxed) as *mut c_void, } } + pub fn from_float(value: f64) -> Self { + let boxed = Box::new(value); Self { value_type: ValueType::Float, - integer: 0, - float: value, - text: TextValue::null(), - blob: Blob::null(), + value: Box::into_raw(boxed) as *mut c_void, } } - pub fn from_text(value: String) -> Self { - let cstr = CString::new(&*value).unwrap(); - let ptr = cstr.as_ptr(); - let len = value.len(); - std::mem::forget(cstr); + pub fn from_text(s: String) -> Self { + let text_value = TextValue::new(s.as_ptr(), s.len()); + let boxed_text = Box::new(text_value); + std::mem::forget(s); Self { value_type: ValueType::Text, - integer: 0, - float: 0.0, - text: TextValue::new(ptr, len), - blob: Blob::null(), + value: Box::into_raw(boxed_text) as *mut c_void, } } - pub fn from_blob(value: &[u8]) -> Self { + pub fn from_blob(value: Vec) -> Self { + let boxed = Box::new(Blob::new(value.as_ptr(), value.len() as u64)); + std::mem::forget(value); Self { value_type: ValueType::Blob, - integer: 0, - float: 0.0, - text: TextValue::null(), - blob: Blob::new(value.as_ptr(), value.len()), + value: Box::into_raw(boxed) as *mut c_void, } } + + pub unsafe fn free(&mut self) { + if self.value.is_null() { + return; + } + match self.value_type { + ValueType::Integer => { + let _ = Box::from_raw(self.value as *mut i64); + } + ValueType::Float => { + let _ = Box::from_raw(self.value as *mut f64); + } + ValueType::Text => { + let _ = Box::from_raw(self.value as *mut TextValue); + } + ValueType::Blob => { + let _ = Box::from_raw(self.value as *mut Blob); + } + ValueType::Null => {} + } + + self.value = std::ptr::null_mut(); + } } From 852817c9ff4f3b5833f46a9baea17fbd5cb29943 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 12 Jan 2025 12:19:05 -0500 Subject: [PATCH 56/97] Have args macro in extension take a range --- extensions/uuid/src/lib.rs | 43 ++++++++++++++++++++++++++++++++++---- limbo_extension/src/lib.rs | 17 +++++++++------ 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index 940658155..7166e4bb1 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -6,26 +6,61 @@ register_extension! { scalars: { "uuid4_str" => uuid4_str, "uuid4" => uuid4_blob, + "uuid7_str" => uuid7_str, + "uuid7" => uuid7_blob, "uuid_str" => uuid_str, "uuid_blob" => uuid_blob, }, } declare_scalar_functions! { - #[args(min = 0, max = 0)] + #[args(0)] fn uuid4_str(_args: &[Value]) -> Value { let uuid = uuid::Uuid::new_v4().to_string(); Value::from_text(uuid) } - #[args(min = 0, max = 0)] + #[args(0)] fn uuid4_blob(_args: &[Value]) -> Value { let uuid = uuid::Uuid::new_v4(); let bytes = uuid.as_bytes(); Value::from_blob(bytes.to_vec()) } - #[args(min = 1, max = 1)] + #[args(0..=1)] + fn uuid7_str(args: &[Value]) -> Value { + let timestamp = if args.is_empty() { + let ctx = uuid::ContextV7::new(); + uuid::Timestamp::now(ctx) + } else if args[0].value_type == limbo_extension::ValueType::Integer { + let ctx = uuid::ContextV7::new(); + let int = args[0].value as i64; + uuid::Timestamp::from_unix(ctx, int as u64, 0) + } else { + return Value::null(); + }; + let uuid = uuid::Uuid::new_v7(timestamp); + Value::from_text(uuid.to_string()) + } + + #[args(0..=1)] + fn uuid7_blob(args: &[Value]) -> Value { + let timestamp = if args.is_empty() { + let ctx = uuid::ContextV7::new(); + uuid::Timestamp::now(ctx) + } else if args[0].value_type == limbo_extension::ValueType::Integer { + let ctx = uuid::ContextV7::new(); + let int = args[0].value as i64; + uuid::Timestamp::from_unix(ctx, int as u64, 0) + } else { + return Value::null(); + }; + let uuid = uuid::Uuid::new_v7(timestamp); + let bytes = uuid.as_bytes(); + Value::from_blob(bytes.to_vec()) + } + + #[args(1)] fn uuid_str(args: &[Value]) -> Value { if args[0].value_type != limbo_extension::ValueType::Blob { log::debug!("uuid_str was passed a non-blob arg"); @@ -43,7 +78,7 @@ declare_scalar_functions! { } } - #[args(min = 1, max = 1)] + #[args(1)] fn uuid_blob(args: &[Value]) -> Value { if args[0].value_type != limbo_extension::ValueType::Text { log::debug!("uuid_blob was passed a non-text arg"); diff --git a/limbo_extension/src/lib.rs b/limbo_extension/src/lib.rs index 3daf99755..a1e77d53d 100644 --- a/limbo_extension/src/lib.rs +++ b/limbo_extension/src/lib.rs @@ -53,6 +53,7 @@ macro_rules! register_scalar_functions { /// Provide a cleaner interface to define scalar functions to extension authors /// . e.g. /// ``` +/// #[args(1)] /// fn scalar_func(args: &[Value]) -> Value { /// if args.len() != 1 { /// return Value::null(); @@ -65,7 +66,7 @@ macro_rules! register_scalar_functions { macro_rules! declare_scalar_functions { ( $( - #[args(min = $min_args:literal, max = $max_args:literal)] + #[args($($args_count:tt)+)] fn $func_name:ident ($args:ident : &[Value]) -> Value $body:block )* ) => { @@ -74,28 +75,32 @@ macro_rules! declare_scalar_functions { argc: i32, argv: *const *const std::os::raw::c_void ) -> $crate::Value { - if !($min_args..=$max_args).contains(&argc) { + let valid_args = { + match argc { + $($args_count)+ => true, + _ => false, + } + }; + if !valid_args { return $crate::Value::null(); } if argc == 0 || argv.is_null() { let $args: &[$crate::Value] = &[]; $body } else { - unsafe { - let ptr_slice = std::slice::from_raw_parts(argv, argc as usize); + let ptr_slice = unsafe{ std::slice::from_raw_parts(argv, argc as usize)}; let mut values = Vec::with_capacity(argc as usize); for &ptr in ptr_slice { let val_ptr = ptr as *const $crate::Value; if val_ptr.is_null() { values.push($crate::Value::null()); } else { - values.push(std::ptr::read(val_ptr)); + unsafe{values.push(std::ptr::read(val_ptr))}; } } let $args: &[$crate::Value] = &values[..]; $body } - } } )* }; From 98eff6cf7a2a75bdce9e78acf949fff1367ad83d Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 12 Jan 2025 15:24:50 -0500 Subject: [PATCH 57/97] Enable passing arguments to external functions --- core/translate/expr.rs | 12 +++++- core/translate/select.rs | 11 +++--- core/vdbe/mod.rs | 42 +++++++++++++------- extensions/uuid/src/lib.rs | 34 ++++++++++++++++- limbo_extension/src/lib.rs | 78 +++++++++++++++++++++++++++----------- 5 files changed, 132 insertions(+), 45 deletions(-) diff --git a/core/translate/expr.rs b/core/translate/expr.rs index a464e5279..d13902433 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -762,13 +762,23 @@ pub fn translate_expr( crate::bail_parse_error!("aggregation function in non-aggregation context") } Func::External(_) => { - let regs = program.alloc_register(); + let regs = program.alloc_registers(args_count); + for (i, arg_expr) in args.iter().enumerate() { + translate_expr( + program, + referenced_tables, + &arg_expr[i], + regs + i, + resolver, + )?; + } program.emit_insn(Insn::Function { constant_mask: 0, start_reg: regs, dest: target_register, func: func_ctx, }); + Ok(target_register) } #[cfg(feature = "json")] diff --git a/core/translate/select.rs b/core/translate/select.rs index dfbd4c2fb..fa5361205 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -148,10 +148,9 @@ pub fn prepare_select_plan( } Err(_) => { if syms.functions.contains_key(&name.0) { - // TODO: future extensions can be aggregate functions - log::debug!( - "Resolving {} function from symbol table", - name.0 + let contains_aggregates = resolve_aggregates( + expr, + &mut aggregate_expressions, ); plan.result_columns.push(ResultSetColumn { name: get_name( @@ -161,7 +160,7 @@ pub fn prepare_select_plan( || format!("expr_{}", result_column_idx), ), expr: expr.clone(), - contains_aggregates: false, + contains_aggregates, }); } } @@ -202,7 +201,7 @@ pub fn prepare_select_plan( } expr => { let contains_aggregates = - resolve_aggregates(&expr, &mut aggregate_expressions); + resolve_aggregates(expr, &mut aggregate_expressions); plan.result_columns.push(ResultSetColumn { name: get_name( maybe_alias.as_ref(), diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 075dd5dab..580834d39 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -55,7 +55,6 @@ use sorter::Sorter; use std::borrow::BorrowMut; use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; -use std::os::raw::c_void; use std::rc::{Rc, Weak}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -147,6 +146,33 @@ macro_rules! return_if_io { }; } +macro_rules! call_external_function { + ( + $func_ptr:expr, + $dest_register:expr, + $state:expr, + $arg_count:expr, + $start_reg:expr + ) => {{ + if $arg_count == 0 { + let result_c_value: ExtValue = ($func_ptr)(0, std::ptr::null()); + let result_ov = OwnedValue::from_ffi(&result_c_value); + $state.registers[$dest_register] = result_ov; + } else { + let register_slice = &$state.registers[$start_reg..$start_reg + $arg_count]; + let mut ext_values: Vec = Vec::with_capacity($arg_count); + for ov in register_slice.iter() { + let val = ov.to_ffi(); + ext_values.push(val); + } + let argv_ptr = ext_values.as_ptr(); + let result_c_value: ExtValue = ($func_ptr)($arg_count as i32, argv_ptr); + let result_ov = OwnedValue::from_ffi(&result_c_value); + $state.registers[$dest_register] = result_ov; + } + }}; +} + struct RegexCache { like: HashMap, glob: HashMap, @@ -1839,19 +1865,7 @@ impl Program { } }, crate::function::Func::External(f) => { - let values = &state.registers[*start_reg..*start_reg + arg_count]; - let c_values: Vec<*const c_void> = values - .iter() - .map(|ov| &ov.to_ffi() as *const _ as *const c_void) - .collect(); - let argv_ptr = if c_values.is_empty() { - std::ptr::null() - } else { - c_values.as_ptr() - }; - let result_c_value: ExtValue = (f.func)(arg_count as i32, argv_ptr); - let result_ov = OwnedValue::from_ffi(&result_c_value); - state.registers[*dest] = result_ov; + call_external_function! {f.func, *dest, state, arg_count, *start_reg }; } crate::function::Func::Math(math_func) => match math_func.arity() { MathFuncArity::Nullary => match math_func { diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index 7166e4bb1..7380f82d3 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -1,5 +1,6 @@ use limbo_extension::{ - declare_scalar_functions, register_extension, register_scalar_functions, Blob, TextValue, Value, + declare_scalar_functions, register_extension, register_scalar_functions, Blob, TextValue, + Value, ValueType, }; register_extension! { @@ -10,6 +11,7 @@ register_extension! { "uuid7" => uuid7_blob, "uuid_str" => uuid_str, "uuid_blob" => uuid_blob, + "exec_ts_from_uuid7" => exec_ts_from_uuid7, }, } @@ -60,6 +62,26 @@ declare_scalar_functions! { Value::from_blob(bytes.to_vec()) } + #[args(1)] + fn exec_ts_from_uuid7(args: &[Value]) -> Value { + match args[0].value_type { + ValueType::Blob => { + let blob = Blob::from_value(&args[0]).unwrap(); + let slice = unsafe{ std::slice::from_raw_parts(blob.data, blob.size as usize)}; + let uuid = uuid::Uuid::from_slice(slice).unwrap(); + let unix = uuid_to_unix(uuid.as_bytes()); + Value::from_integer(unix as i64) + } + ValueType::Text => { + let text = TextValue::from_value(&args[0]).unwrap(); + let uuid = uuid::Uuid::parse_str(unsafe {text.as_str()}).unwrap(); + let unix = uuid_to_unix(uuid.as_bytes()); + Value::from_integer(unix as i64) + } + _ => Value::null(), + } + } + #[args(1)] fn uuid_str(args: &[Value]) -> Value { if args[0].value_type != limbo_extension::ValueType::Blob { @@ -96,3 +118,13 @@ declare_scalar_functions! { } } } + +#[inline(always)] +fn uuid_to_unix(uuid: &[u8; 16]) -> u64 { + ((uuid[0] as u64) << 40) + | ((uuid[1] as u64) << 32) + | ((uuid[2] as u64) << 24) + | ((uuid[3] as u64) << 16) + | ((uuid[4] as u64) << 8) + | (uuid[5] as u64) +} diff --git a/limbo_extension/src/lib.rs b/limbo_extension/src/lib.rs index a1e77d53d..7bfc881e5 100644 --- a/limbo_extension/src/lib.rs +++ b/limbo_extension/src/lib.rs @@ -7,7 +7,7 @@ pub const RESULT_ERROR: ResultCode = 1; // TODO: more error types pub type ExtensionEntryPoint = extern "C" fn(api: *const ExtensionApi) -> ResultCode; -pub type ScalarFunction = extern "C" fn(argc: i32, *const *const c_void) -> Value; +pub type ScalarFunction = extern "C" fn(argc: i32, *const Value) -> Value; #[repr(C)] pub struct ExtensionApi { @@ -54,12 +54,13 @@ macro_rules! register_scalar_functions { /// . e.g. /// ``` /// #[args(1)] -/// fn scalar_func(args: &[Value]) -> Value { -/// if args.len() != 1 { -/// return Value::null(); -/// } +/// fn scalar_double(args: &[Value]) -> Value { /// Value::from_integer(args[0].integer * 2) /// } +/// +/// #[args(0..=2)] +/// fn scalar_sum(args: &[Value]) -> Value { +/// Value::from_integer(args.iter().map(|v| v.integer).sum()) /// ``` /// #[macro_export] @@ -73,7 +74,7 @@ macro_rules! declare_scalar_functions { $( extern "C" fn $func_name( argc: i32, - argv: *const *const std::os::raw::c_void + argv: *const $crate::Value ) -> $crate::Value { let valid_args = { match argc { @@ -85,22 +86,14 @@ macro_rules! declare_scalar_functions { return $crate::Value::null(); } if argc == 0 || argv.is_null() { + log::debug!("{} was called with no arguments", stringify!($func_name)); let $args: &[$crate::Value] = &[]; $body } else { - let ptr_slice = unsafe{ std::slice::from_raw_parts(argv, argc as usize)}; - let mut values = Vec::with_capacity(argc as usize); - for &ptr in ptr_slice { - let val_ptr = ptr as *const $crate::Value; - if val_ptr.is_null() { - values.push($crate::Value::null()); - } else { - unsafe{values.push(std::ptr::read(val_ptr))}; - } - } - let $args: &[$crate::Value] = &values[..]; - $body - } + let ptr_slice = unsafe{ std::slice::from_raw_parts(argv, argc as usize)}; + let $args: &[$crate::Value] = ptr_slice; + $body + } } )* }; @@ -122,12 +115,42 @@ pub struct Value { pub value: *mut c_void, } +impl std::fmt::Debug for Value { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.value_type { + ValueType::Null => write!(f, "Value {{ Null }}"), + ValueType::Integer => write!(f, "Value {{ Integer: {} }}", unsafe { + *(self.value as *const i64) + }), + ValueType::Float => write!(f, "Value {{ Float: {} }}", unsafe { + *(self.value as *const f64) + }), + ValueType::Text => write!(f, "Value {{ Text: {:?} }}", unsafe { + &*(self.value as *const TextValue) + }), + ValueType::Blob => write!(f, "Value {{ Blob: {:?} }}", unsafe { + &*(self.value as *const Blob) + }), + } + } +} + #[repr(C)] pub struct TextValue { pub text: *const u8, pub len: u32, } +impl std::fmt::Debug for TextValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "TextValue {{ text: {:?}, len: {} }}", + self.text, self.len + ) + } +} + impl Default for TextValue { fn default() -> Self { Self { @@ -170,6 +193,12 @@ pub struct Blob { pub size: u64, } +impl std::fmt::Debug for Blob { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Blob {{ data: {:?}, size: {} }}", self.data, self.size) + } +} + impl Blob { pub fn new(data: *const u8, size: u64) -> Self { Self { data, size } @@ -208,12 +237,15 @@ impl Value { } pub fn from_text(s: String) -> Self { - let text_value = TextValue::new(s.as_ptr(), s.len()); - let boxed_text = Box::new(text_value); - std::mem::forget(s); + let buffer = s.into_boxed_str(); + let ptr = buffer.as_ptr(); + let len = buffer.len(); + std::mem::forget(buffer); + let text_value = TextValue::new(ptr, len); + let text_box = Box::new(text_value); Self { value_type: ValueType::Text, - value: Box::into_raw(boxed_text) as *mut c_void, + value: Box::into_raw(text_box) as *mut c_void, } } From 6e05258d368aa3bbe2ded88f4b49cd0234b72618 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 12 Jan 2025 15:48:26 -0500 Subject: [PATCH 58/97] Add safety comments and clean up extension types --- core/types.rs | 12 ++++-------- extensions/uuid/src/lib.rs | 11 ++++++----- limbo_extension/src/lib.rs | 29 ++++++++++++++++++----------- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/core/types.rs b/core/types.rs index 92d02fce5..3d84da194 100644 --- a/core/types.rs +++ b/core/types.rs @@ -123,14 +123,10 @@ impl OwnedValue { OwnedValue::Float(float) } ExtValueType::Text => { - if v.value.is_null() { - OwnedValue::Null - } else { - let Some(text) = ExtTextValue::from_value(v) else { - return OwnedValue::Null; - }; - OwnedValue::build_text(std::rc::Rc::new(unsafe { text.as_str().to_string() })) - } + let Some(text) = (unsafe { ExtTextValue::from_value(v) }) else { + return OwnedValue::Null; + }; + OwnedValue::build_text(std::rc::Rc::new(unsafe { text.as_str().to_string() })) } ExtValueType::Blob => { let blob_ptr = v.value as *mut ExtBlob; diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index 7380f82d3..df8581955 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -73,7 +73,9 @@ declare_scalar_functions! { Value::from_integer(unix as i64) } ValueType::Text => { - let text = TextValue::from_value(&args[0]).unwrap(); + let Some(text) = (unsafe {TextValue::from_value(&args[0])}) else { + return Value::null(); + }; let uuid = uuid::Uuid::parse_str(unsafe {text.as_str()}).unwrap(); let unix = uuid_to_unix(uuid.as_bytes()); Value::from_integer(unix as i64) @@ -106,16 +108,15 @@ declare_scalar_functions! { log::debug!("uuid_blob was passed a non-text arg"); return Value::null(); } - if let Some(text) = TextValue::from_value(&args[0]) { + let Some(text) = (unsafe { TextValue::from_value(&args[0])}) else { + return Value::null(); + }; match uuid::Uuid::parse_str(unsafe {text.as_str()}) { Ok(uuid) => { Value::from_blob(uuid.as_bytes().to_vec()) } Err(_) => Value::null() } - } else { - Value::null() - } } } diff --git a/limbo_extension/src/lib.rs b/limbo_extension/src/lib.rs index 7bfc881e5..f8eb19dcf 100644 --- a/limbo_extension/src/lib.rs +++ b/limbo_extension/src/lib.rs @@ -137,8 +137,8 @@ impl std::fmt::Debug for Value { #[repr(C)] pub struct TextValue { - pub text: *const u8, - pub len: u32, + text: *const u8, + len: u32, } impl std::fmt::Debug for TextValue { @@ -168,22 +168,27 @@ impl TextValue { } } - pub fn from_value(value: &Value) -> Option<&Self> { + /// # Safety + /// Safe to call if the pointer is null, returns None + /// if the value is not a text type or if the value is null + pub unsafe fn from_value(value: &Value) -> Option<&Self> { if value.value_type != ValueType::Text { return None; } - unsafe { Some(&*(value.value as *const TextValue)) } + if value.value.is_null() { + return None; + } + Some(&*(value.value as *const TextValue)) } /// # Safety - /// The caller must ensure that the text is a valid UTF-8 string + /// If self.text is null we safely return an empty string but + /// the caller must ensure that the underlying value is valid utf8 pub unsafe fn as_str(&self) -> &str { if self.text.is_null() { return ""; } - unsafe { - std::str::from_utf8_unchecked(std::slice::from_raw_parts(self.text, self.len as usize)) - } + std::str::from_utf8_unchecked(std::slice::from_raw_parts(self.text, self.len as usize)) } } @@ -258,7 +263,11 @@ impl Value { } } - pub unsafe fn free(&mut self) { + /// # Safety + /// consumes the value while freeing the underlying memory with null check. + /// however this does assume that the type was properly constructed with + /// the appropriate value_type and value. + pub unsafe fn free(self) { if self.value.is_null() { return; } @@ -277,7 +286,5 @@ impl Value { } ValueType::Null => {} } - - self.value = std::ptr::null_mut(); } } From 3099e5c9ba237948dd0928f6ae0e9a904da8e24d Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 12 Jan 2025 16:35:56 -0500 Subject: [PATCH 59/97] Improve api, standardize conversions between types, finish extension --- core/ext/mod.rs | 7 +++-- core/lib.rs | 6 ++-- core/types.rs | 11 +++---- extensions/uuid/src/lib.rs | 61 ++++++++++++++++++++++++-------------- limbo_extension/src/lib.rs | 47 ++++++++++++++++++++++++----- 5 files changed, 92 insertions(+), 40 deletions(-) diff --git a/core/ext/mod.rs b/core/ext/mod.rs index 179b3e7c6..f1758324b 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -1,8 +1,6 @@ use crate::{function::ExternalFunc, Database}; -pub use limbo_extension::{ - Blob as ExtBlob, TextValue as ExtTextValue, Value as ExtValue, ValueType as ExtValueType, -}; use limbo_extension::{ExtensionApi, ResultCode, ScalarFunction, RESULT_ERROR, RESULT_OK}; +pub use limbo_extension::{Value as ExtValue, ValueType as ExtValueType}; use std::{ ffi::{c_char, c_void, CStr}, rc::Rc, @@ -18,6 +16,9 @@ extern "C" fn register_scalar_function( Ok(s) => s.to_string(), Err(_) => return RESULT_ERROR, }; + if ctx.is_null() { + return RESULT_ERROR; + } let db = unsafe { &*(ctx as *const Database) }; db.register_scalar_function_impl(name_str, func) } diff --git a/core/lib.rs b/core/lib.rs index 4b3e8155b..f99bd3db3 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -126,7 +126,7 @@ impl Database { let header = db_header; let schema = Rc::new(RefCell::new(Schema::new())); let syms = Rc::new(RefCell::new(SymbolTable::new())); - let mut db = Database { + let db = Database { pager: pager.clone(), schema: schema.clone(), header: header.clone(), @@ -193,7 +193,9 @@ impl Database { self.syms.borrow_mut().extensions.push((lib, api_ptr)); Ok(()) } else { - let _ = unsafe { Box::from_raw(api_ptr.cast_mut()) }; // own this again so we dont leak + if !api_ptr.is_null() { + let _ = unsafe { Box::from_raw(api_ptr.cast_mut()) }; + } Err(LimboError::ExtensionError( "Extension registration failed".to_string(), )) diff --git a/core/types.rs b/core/types.rs index 3d84da194..82e8c787b 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1,5 +1,5 @@ use crate::error::LimboError; -use crate::ext::{ExtBlob, ExtTextValue, ExtValue, ExtValueType}; +use crate::ext::{ExtValue, ExtValueType}; use crate::storage::sqlite3_ondisk::write_varint; use crate::Result; use std::fmt::Display; @@ -123,16 +123,17 @@ impl OwnedValue { OwnedValue::Float(float) } ExtValueType::Text => { - let Some(text) = (unsafe { ExtTextValue::from_value(v) }) else { + let Some(text) = v.to_text() else { return OwnedValue::Null; }; OwnedValue::build_text(std::rc::Rc::new(unsafe { text.as_str().to_string() })) } ExtValueType::Blob => { - let blob_ptr = v.value as *mut ExtBlob; + let Some(blob_ptr) = v.to_blob() else { + return OwnedValue::Null; + }; let blob = unsafe { - let slice = - std::slice::from_raw_parts((*blob_ptr).data, (*blob_ptr).size as usize); + let slice = std::slice::from_raw_parts(blob_ptr.data, blob_ptr.size as usize); slice.to_vec() }; OwnedValue::Blob(std::rc::Rc::new(blob)) diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index df8581955..d88f2f887 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -1,6 +1,5 @@ use limbo_extension::{ - declare_scalar_functions, register_extension, register_scalar_functions, Blob, TextValue, - Value, ValueType, + declare_scalar_functions, register_extension, register_scalar_functions, Value, ValueType, }; register_extension! { @@ -11,7 +10,7 @@ register_extension! { "uuid7" => uuid7_blob, "uuid_str" => uuid_str, "uuid_blob" => uuid_blob, - "exec_ts_from_uuid7" => exec_ts_from_uuid7, + "uuid7_timestamp_ms" => exec_ts_from_uuid7, }, } @@ -32,14 +31,35 @@ declare_scalar_functions! { #[args(0..=1)] fn uuid7_str(args: &[Value]) -> Value { let timestamp = if args.is_empty() { - let ctx = uuid::ContextV7::new(); + let ctx = uuid::ContextV7::new(); uuid::Timestamp::now(ctx) - } else if args[0].value_type == limbo_extension::ValueType::Integer { + } else { + let arg = &args[0]; + match arg.value_type { + ValueType::Integer => { let ctx = uuid::ContextV7::new(); - let int = args[0].value as i64; + let Some(int) = arg.to_integer() else { + return Value::null(); + }; uuid::Timestamp::from_unix(ctx, int as u64, 0) - } else { - return Value::null(); + } + ValueType::Text => { + let Some(text) = arg.to_text() else { + return Value::null(); + }; + let parsed = unsafe{text.as_str()}.parse::(); + match parsed { + Ok(unix) => { + if unix <= 0 { + return Value::null(); + } + uuid::Timestamp::from_unix(uuid::ContextV7::new(), unix as u64, 0) + } + Err(_) => return Value::null(), + } + } + _ => return Value::null(), + } }; let uuid = uuid::Uuid::new_v7(timestamp); Value::from_text(uuid.to_string()) @@ -52,7 +72,9 @@ declare_scalar_functions! { uuid::Timestamp::now(ctx) } else if args[0].value_type == limbo_extension::ValueType::Integer { let ctx = uuid::ContextV7::new(); - let int = args[0].value as i64; + let Some(int) = args[0].to_integer() else { + return Value::null(); + }; uuid::Timestamp::from_unix(ctx, int as u64, 0) } else { return Value::null(); @@ -66,14 +88,16 @@ declare_scalar_functions! { fn exec_ts_from_uuid7(args: &[Value]) -> Value { match args[0].value_type { ValueType::Blob => { - let blob = Blob::from_value(&args[0]).unwrap(); + let Some(blob) = &args[0].to_blob() else { + return Value::null(); + }; let slice = unsafe{ std::slice::from_raw_parts(blob.data, blob.size as usize)}; let uuid = uuid::Uuid::from_slice(slice).unwrap(); let unix = uuid_to_unix(uuid.as_bytes()); Value::from_integer(unix as i64) } ValueType::Text => { - let Some(text) = (unsafe {TextValue::from_value(&args[0])}) else { + let Some(text) = args[0].to_text() else { return Value::null(); }; let uuid = uuid::Uuid::parse_str(unsafe {text.as_str()}).unwrap(); @@ -86,29 +110,20 @@ declare_scalar_functions! { #[args(1)] fn uuid_str(args: &[Value]) -> Value { - if args[0].value_type != limbo_extension::ValueType::Blob { - log::debug!("uuid_str was passed a non-blob arg"); + let Some(blob) = args[0].to_blob() else { return Value::null(); - } - if let Some(blob) = Blob::from_value(&args[0]) { + }; let slice = unsafe{ std::slice::from_raw_parts(blob.data, blob.size as usize)}; let parsed = uuid::Uuid::from_slice(slice).ok().map(|u| u.to_string()); match parsed { Some(s) => Value::from_text(s), None => Value::null() } - } else { - Value::null() - } } #[args(1)] fn uuid_blob(args: &[Value]) -> Value { - if args[0].value_type != limbo_extension::ValueType::Text { - log::debug!("uuid_blob was passed a non-text arg"); - return Value::null(); - } - let Some(text) = (unsafe { TextValue::from_value(&args[0])}) else { + let Some(text) = args[0].to_text() else { return Value::null(); }; match uuid::Uuid::parse_str(unsafe {text.as_str()}) { diff --git a/limbo_extension/src/lib.rs b/limbo_extension/src/lib.rs index f8eb19dcf..d07cc8ea7 100644 --- a/limbo_extension/src/lib.rs +++ b/limbo_extension/src/lib.rs @@ -208,13 +208,6 @@ impl Blob { pub fn new(data: *const u8, size: u64) -> Self { Self { data, size } } - - pub fn from_value(value: &Value) -> Option<&Self> { - if value.value_type != ValueType::Blob { - return None; - } - unsafe { Some(&*(value.value as *const Blob)) } - } } impl Value { @@ -225,6 +218,46 @@ impl Value { } } + pub fn to_float(&self) -> Option { + if self.value_type != ValueType::Float { + return None; + } + if self.value.is_null() { + return None; + } + Some(unsafe { *(self.value as *const f64) }) + } + + pub fn to_text(&self) -> Option<&TextValue> { + if self.value_type != ValueType::Text { + return None; + } + if self.value.is_null() { + return None; + } + unsafe { Some(&*(self.value as *const TextValue)) } + } + + pub fn to_blob(&self) -> Option<&Blob> { + if self.value_type != ValueType::Blob { + return None; + } + if self.value.is_null() { + return None; + } + unsafe { Some(&*(self.value as *const Blob)) } + } + + pub fn to_integer(&self) -> Option { + if self.value_type != ValueType::Integer { + return None; + } + if self.value.is_null() { + return None; + } + Some(unsafe { *(self.value as *const i64) }) + } + pub fn from_integer(value: i64) -> Self { let boxed = Box::new(value); Self { From e4ce6402ebe357ea2de795c749556f86e9cd9039 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 12 Jan 2025 16:47:43 -0500 Subject: [PATCH 60/97] Remove previous uuid implementation --- core/ext/uuid.rs | 343 ------------------------------------- extensions/uuid/src/lib.rs | 4 +- 2 files changed, 3 insertions(+), 344 deletions(-) delete mode 100644 core/ext/uuid.rs diff --git a/core/ext/uuid.rs b/core/ext/uuid.rs deleted file mode 100644 index 37e496f00..000000000 --- a/core/ext/uuid.rs +++ /dev/null @@ -1,343 +0,0 @@ -use super::ExtFunc; -use crate::{ - types::{LimboText, OwnedValue}, - Database, LimboError, -}; -use std::rc::Rc; -use uuid::{ContextV7, Timestamp, Uuid}; - -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum UuidFunc { - Uuid4Str, - Uuid7, - Uuid7TS, - UuidStr, - UuidBlob, -} - -impl UuidFunc { - pub fn resolve_function(name: &str, num_args: usize) -> Option { - match name { - "uuid4_str" => Some(ExtFunc::Uuid(Self::Uuid4Str)), - "uuid7" if num_args < 2 => Some(ExtFunc::Uuid(Self::Uuid7)), - "uuid_str" if num_args == 1 => Some(ExtFunc::Uuid(Self::UuidStr)), - "uuid_blob" if num_args == 1 => Some(ExtFunc::Uuid(Self::UuidBlob)), - "uuid7_timestamp_ms" if num_args == 1 => Some(ExtFunc::Uuid(Self::Uuid7TS)), - // postgres_compatability - "gen_random_uuid" => Some(ExtFunc::Uuid(Self::Uuid4Str)), - _ => None, - } - } -} - -impl std::fmt::Display for UuidFunc { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Uuid4Str => write!(f, "uuid4_str"), - Self::Uuid7 => write!(f, "uuid7"), - Self::Uuid7TS => write!(f, "uuid7_timestamp_ms"), - Self::UuidStr => write!(f, "uuid_str"), - Self::UuidBlob => write!(f, "uuid_blob"), - } - } -} - -pub fn exec_uuid(var: &UuidFunc, sec: Option<&OwnedValue>) -> crate::Result { - match var { - UuidFunc::Uuid4Str => Ok(OwnedValue::Text(LimboText::new(Rc::new( - Uuid::new_v4().to_string(), - )))), - UuidFunc::Uuid7 => { - let uuid = match sec { - Some(OwnedValue::Integer(ref seconds)) => { - let ctx = ContextV7::new(); - if *seconds < 0 { - // not valid unix timestamp, error or null? - return Ok(OwnedValue::Null); - } - Uuid::new_v7(Timestamp::from_unix(ctx, *seconds as u64, 0)) - } - _ => Uuid::now_v7(), - }; - Ok(OwnedValue::Blob(Rc::new(uuid.into_bytes().to_vec()))) - } - _ => unreachable!(), - } -} - -pub fn exec_uuid4() -> crate::Result { - Ok(OwnedValue::Blob(Rc::new( - Uuid::new_v4().into_bytes().to_vec(), - ))) -} - -pub fn exec_uuidstr(reg: &OwnedValue) -> crate::Result { - match reg { - OwnedValue::Blob(blob) => { - let uuid = Uuid::from_slice(blob).map_err(|e| LimboError::ParseError(e.to_string()))?; - Ok(OwnedValue::Text(LimboText::new(Rc::new(uuid.to_string())))) - } - OwnedValue::Text(ref val) => { - let uuid = - Uuid::parse_str(&val.value).map_err(|e| LimboError::ParseError(e.to_string()))?; - Ok(OwnedValue::Text(LimboText::new(Rc::new(uuid.to_string())))) - } - OwnedValue::Null => Ok(OwnedValue::Null), - _ => Err(LimboError::ParseError( - "Invalid argument type for UUID function".to_string(), - )), - } -} - -pub fn exec_uuidblob(reg: &OwnedValue) -> crate::Result { - match reg { - OwnedValue::Text(val) => { - let uuid = - Uuid::parse_str(&val.value).map_err(|e| LimboError::ParseError(e.to_string()))?; - Ok(OwnedValue::Blob(Rc::new(uuid.as_bytes().to_vec()))) - } - OwnedValue::Blob(blob) => { - let uuid = Uuid::from_slice(blob).map_err(|e| LimboError::ParseError(e.to_string()))?; - Ok(OwnedValue::Blob(Rc::new(uuid.as_bytes().to_vec()))) - } - OwnedValue::Null => Ok(OwnedValue::Null), - _ => Err(LimboError::ParseError( - "Invalid argument type for UUID function".to_string(), - )), - } -} - -pub fn exec_ts_from_uuid7(reg: &OwnedValue) -> OwnedValue { - let uuid = match reg { - OwnedValue::Blob(blob) => { - Uuid::from_slice(blob).map_err(|e| LimboError::ParseError(e.to_string())) - } - OwnedValue::Text(val) => { - Uuid::parse_str(&val.value).map_err(|e| LimboError::ParseError(e.to_string())) - } - _ => Err(LimboError::ParseError( - "Invalid argument type for UUID function".to_string(), - )), - }; - match uuid { - Ok(uuid) => OwnedValue::Integer(uuid_to_unix(uuid.as_bytes()) as i64), - // display error? sqlean seems to set value to null - Err(_) => OwnedValue::Null, - } -} - -#[inline(always)] -fn uuid_to_unix(uuid: &[u8; 16]) -> u64 { - ((uuid[0] as u64) << 40) - | ((uuid[1] as u64) << 32) - | ((uuid[2] as u64) << 24) - | ((uuid[3] as u64) << 16) - | ((uuid[4] as u64) << 8) - | (uuid[5] as u64) -} - -//pub fn init(db: &mut Database) { -// db.define_scalar_function("uuid4", |_args| exec_uuid4()); -//} - -#[cfg(test)] -#[cfg(feature = "uuid")] -pub mod test { - use super::UuidFunc; - use crate::types::OwnedValue; - #[test] - fn test_exec_uuid_v4blob() { - use super::exec_uuid4; - use uuid::Uuid; - let owned_val = exec_uuid4(); - match owned_val { - Ok(OwnedValue::Blob(blob)) => { - assert_eq!(blob.len(), 16); - let uuid = Uuid::from_slice(&blob); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 4); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - } - - #[test] - fn test_exec_uuid_v4str() { - use super::{exec_uuid, UuidFunc}; - use uuid::Uuid; - let func = UuidFunc::Uuid4Str; - let owned_val = exec_uuid(&func, None); - match owned_val { - Ok(OwnedValue::Text(v4str)) => { - assert_eq!(v4str.value.len(), 36); - let uuid = Uuid::parse_str(&v4str.value); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 4); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - } - - #[test] - fn test_exec_uuid_v7_now() { - use super::{exec_uuid, UuidFunc}; - use uuid::Uuid; - let func = UuidFunc::Uuid7; - let owned_val = exec_uuid(&func, None); - match owned_val { - Ok(OwnedValue::Blob(blob)) => { - assert_eq!(blob.len(), 16); - let uuid = Uuid::from_slice(&blob); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 7); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - } - - #[test] - fn test_exec_uuid_v7_with_input() { - use super::{exec_uuid, UuidFunc}; - use uuid::Uuid; - let func = UuidFunc::Uuid7; - let owned_val = exec_uuid(&func, Some(&OwnedValue::Integer(946702800))); - match owned_val { - Ok(OwnedValue::Blob(blob)) => { - assert_eq!(blob.len(), 16); - let uuid = Uuid::from_slice(&blob); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 7); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - } - - #[test] - fn test_exec_uuid_v7_now_to_timestamp() { - use super::{exec_ts_from_uuid7, exec_uuid, UuidFunc}; - use uuid::Uuid; - let func = UuidFunc::Uuid7; - let owned_val = exec_uuid(&func, None); - match owned_val { - Ok(OwnedValue::Blob(ref blob)) => { - assert_eq!(blob.len(), 16); - let uuid = Uuid::from_slice(blob); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 7); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - let result = exec_ts_from_uuid7(&owned_val.expect("uuid7")); - if let OwnedValue::Integer(ref ts) = result { - let unixnow = (std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs() - * 1000) as i64; - assert!(*ts >= unixnow - 1000); - } - } - - #[test] - fn test_exec_uuid_v7_to_timestamp() { - use super::{exec_ts_from_uuid7, exec_uuid, UuidFunc}; - use uuid::Uuid; - let func = UuidFunc::Uuid7; - let owned_val = exec_uuid(&func, Some(&OwnedValue::Integer(946702800))); - match owned_val { - Ok(OwnedValue::Blob(ref blob)) => { - assert_eq!(blob.len(), 16); - let uuid = Uuid::from_slice(blob); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 7); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - let result = exec_ts_from_uuid7(&owned_val.expect("uuid7")); - assert_eq!(result, OwnedValue::Integer(946702800 * 1000)); - if let OwnedValue::Integer(ts) = result { - let time = chrono::DateTime::from_timestamp(ts / 1000, 0); - assert_eq!( - time.unwrap(), - "2000-01-01T05:00:00Z" - .parse::>() - .unwrap() - ); - } - } - - #[test] - fn test_exec_uuid_v4_str_to_blob() { - use super::{exec_uuid, exec_uuidblob, UuidFunc}; - use uuid::Uuid; - let owned_val = exec_uuidblob( - &exec_uuid(&UuidFunc::Uuid4Str, None).expect("uuid v4 string to generate"), - ); - match owned_val { - Ok(OwnedValue::Blob(blob)) => { - assert_eq!(blob.len(), 16); - let uuid = Uuid::from_slice(&blob); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 4); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - } - - #[test] - fn test_exec_uuid_v7_str_to_blob() { - use super::{exec_uuid, exec_uuidblob, exec_uuidstr, UuidFunc}; - use uuid::Uuid; - // convert a v7 blob to a string then back to a blob - let owned_val = exec_uuidblob( - &exec_uuidstr(&exec_uuid(&UuidFunc::Uuid7, None).expect("uuid v7 blob to generate")) - .expect("uuid v7 string to generate"), - ); - match owned_val { - Ok(OwnedValue::Blob(blob)) => { - assert_eq!(blob.len(), 16); - let uuid = Uuid::from_slice(&blob); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 7); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - } - - #[test] - fn test_exec_uuid_v4_blob_to_str() { - use super::{exec_uuid4, exec_uuidstr}; - use uuid::Uuid; - // convert a v4 blob to a string - let owned_val = exec_uuidstr(&exec_uuid4().expect("uuid v7 blob to generate")); - match owned_val { - Ok(OwnedValue::Text(v4str)) => { - assert_eq!(v4str.value.len(), 36); - let uuid = Uuid::parse_str(&v4str.value); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 4); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - } - - #[test] - fn test_exec_uuid_v7_blob_to_str() { - use super::{exec_uuid, exec_uuidstr}; - use uuid::Uuid; - // convert a v7 blob to a string - let owned_val = exec_uuidstr( - &exec_uuid(&UuidFunc::Uuid7, Some(&OwnedValue::Integer(123456789))) - .expect("uuid v7 blob to generate"), - ); - match owned_val { - Ok(OwnedValue::Text(v7str)) => { - assert_eq!(v7str.value.len(), 36); - let uuid = Uuid::parse_str(&v7str.value); - assert!(uuid.is_ok()); - assert_eq!(uuid.unwrap().get_version_num(), 7); - } - _ => panic!("exec_uuid did not return a Blob variant"), - } - } -} diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index d88f2f887..92e9d5d4b 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -100,7 +100,9 @@ declare_scalar_functions! { let Some(text) = args[0].to_text() else { return Value::null(); }; - let uuid = uuid::Uuid::parse_str(unsafe {text.as_str()}).unwrap(); + let Ok(uuid) = uuid::Uuid::parse_str(unsafe {text.as_str()}) else { + return Value::null(); + }; let unix = uuid_to_unix(uuid.as_bytes()); Value::from_integer(unix as i64) } From 9c208dc866d0e4cb0e4053cdb0a83b9b71ab0038 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 12 Jan 2025 17:32:20 -0500 Subject: [PATCH 61/97] Add tests for first extension --- Cargo.lock | 17 +++-- Makefile | 9 ++- cli/app.rs | 5 +- core/lib.rs | 7 ++ core/translate/select.rs | 1 + core/types.rs | 25 +++---- extensions/uuid/src/lib.rs | 19 +++-- limbo_extension/src/lib.rs | 62 ++++++----------- testing/extensions.py | 139 +++++++++++++++++++++++++++++++++++++ 9 files changed, 205 insertions(+), 79 deletions(-) create mode 100755 testing/extensions.py diff --git a/Cargo.lock b/Cargo.lock index d3fffc7a0..7de3e3500 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1248,7 +1248,13 @@ dependencies = [ ] [[package]] -<<<<<<< HEAD +name = "limbo_extension" +version = "0.0.12" +dependencies = [ + "log", +] + +[[package]] name = "limbo_libsql" version = "0.0.12" dependencies = [ @@ -1257,13 +1263,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "limbo_extension" -version = "0.0.11" -dependencies = [ - "log", -] - [[package]] name = "limbo_macros" version = "0.0.12" @@ -1294,7 +1293,7 @@ dependencies = [ [[package]] name = "limbo_uuid" -version = "0.0.11" +version = "0.0.12" dependencies = [ "limbo_extension", "log", diff --git a/Makefile b/Makefile index 30d84bcc4..109a3f147 100644 --- a/Makefile +++ b/Makefile @@ -62,10 +62,15 @@ limbo-wasm: cargo build --package limbo-wasm --target wasm32-wasi .PHONY: limbo-wasm -test: limbo test-compat test-sqlite3 test-shell +test: limbo test-compat test-sqlite3 test-shell test-extensions .PHONY: test -test-shell: limbo +test-extensions: limbo + cargo build --package limbo_uuid + ./testing/extensions.py +.PHONY: test-extensions + +test-shell: limbo SQLITE_EXEC=$(SQLITE_EXEC) ./testing/shelltests.py .PHONY: test-shell diff --git a/cli/app.rs b/cli/app.rs index 62108ca6d..a0a7b5d99 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -323,6 +323,7 @@ impl Limbo { }; } + #[cfg(not(target_family = "wasm"))] fn handle_load_extension(&mut self, path: &str) -> Result<(), String> { self.conn.load_extension(path).map_err(|e| e.to_string()) } @@ -550,7 +551,9 @@ impl Limbo { let _ = self.writeln(e.to_string()); }; } - Command::LoadExtension => { + Command::LoadExtension => + { + #[cfg(not(target_family = "wasm"))] if let Err(e) = self.handle_load_extension(args[1]) { let _ = self.writeln(&e); } diff --git a/core/lib.rs b/core/lib.rs index f99bd3db3..8f8914b0f 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -18,7 +18,9 @@ mod vdbe; static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; use fallible_iterator::FallibleIterator; +#[cfg(not(target_family = "wasm"))] use libloading::{Library, Symbol}; +#[cfg(not(target_family = "wasm"))] use limbo_extension::{ExtensionApi, ExtensionEntryPoint, RESULT_OK}; use log::trace; use schema::Schema; @@ -179,6 +181,7 @@ impl Database { .insert(name.as_ref().to_string(), func.into()); } + #[cfg(not(target_family = "wasm"))] pub fn load_extension(&self, path: &str) -> Result<()> { let api = Box::new(self.build_limbo_extension()); let lib = @@ -397,6 +400,7 @@ impl Connection { Ok(()) } + #[cfg(not(target_family = "wasm"))] pub fn load_extension(&self, path: &str) -> Result<()> { Database::load_extension(self.db.as_ref(), path) } @@ -499,6 +503,7 @@ impl Rows { pub(crate) struct SymbolTable { pub functions: HashMap>, + #[cfg(not(target_family = "wasm"))] extensions: Vec<(libloading::Library, *const ExtensionApi)>, } @@ -514,6 +519,8 @@ impl SymbolTable { pub fn new() -> Self { Self { functions: HashMap::new(), + // TODO: wasm libs will be very different + #[cfg(not(target_family = "wasm"))] extensions: Vec::new(), } } diff --git a/core/translate/select.rs b/core/translate/select.rs index fa5361205..768474a8e 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -20,6 +20,7 @@ pub fn translate_select( select: ast::Select, syms: &SymbolTable, ) -> Result<()> { + let mut select_plan = prepare_select_plan(schema, select, syms)?; optimize_plan(&mut select_plan)?; emit_program(program, select_plan, syms) } diff --git a/core/types.rs b/core/types.rs index 82e8c787b..81d9d5328 100644 --- a/core/types.rs +++ b/core/types.rs @@ -107,35 +107,30 @@ impl OwnedValue { } pub fn from_ffi(v: &ExtValue) -> Self { - if v.value.is_null() { - return OwnedValue::Null; - } - match v.value_type { + match v.value_type() { ExtValueType::Null => OwnedValue::Null, ExtValueType::Integer => { - let int_ptr = v.value as *mut i64; - let integer = unsafe { *int_ptr }; - OwnedValue::Integer(integer) + let Some(int) = v.to_integer() else { + return OwnedValue::Null; + }; + OwnedValue::Integer(int) } ExtValueType::Float => { - let float_ptr = v.value as *mut f64; - let float = unsafe { *float_ptr }; + let Some(float) = v.to_float() else { + return OwnedValue::Null; + }; OwnedValue::Float(float) } ExtValueType::Text => { let Some(text) = v.to_text() else { return OwnedValue::Null; }; - OwnedValue::build_text(std::rc::Rc::new(unsafe { text.as_str().to_string() })) + OwnedValue::build_text(std::rc::Rc::new(text)) } ExtValueType::Blob => { - let Some(blob_ptr) = v.to_blob() else { + let Some(blob) = v.to_blob() else { return OwnedValue::Null; }; - let blob = unsafe { - let slice = std::slice::from_raw_parts(blob_ptr.data, blob_ptr.size as usize); - slice.to_vec() - }; OwnedValue::Blob(std::rc::Rc::new(blob)) } } diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index 92e9d5d4b..e6a3a4f9b 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -35,7 +35,7 @@ declare_scalar_functions! { uuid::Timestamp::now(ctx) } else { let arg = &args[0]; - match arg.value_type { + match arg.value_type() { ValueType::Integer => { let ctx = uuid::ContextV7::new(); let Some(int) = arg.to_integer() else { @@ -47,8 +47,7 @@ declare_scalar_functions! { let Some(text) = arg.to_text() else { return Value::null(); }; - let parsed = unsafe{text.as_str()}.parse::(); - match parsed { + match text.parse::() { Ok(unix) => { if unix <= 0 { return Value::null(); @@ -70,7 +69,7 @@ declare_scalar_functions! { let timestamp = if args.is_empty() { let ctx = uuid::ContextV7::new(); uuid::Timestamp::now(ctx) - } else if args[0].value_type == limbo_extension::ValueType::Integer { + } else if args[0].value_type() == limbo_extension::ValueType::Integer { let ctx = uuid::ContextV7::new(); let Some(int) = args[0].to_integer() else { return Value::null(); @@ -86,13 +85,12 @@ declare_scalar_functions! { #[args(1)] fn exec_ts_from_uuid7(args: &[Value]) -> Value { - match args[0].value_type { + match args[0].value_type() { ValueType::Blob => { let Some(blob) = &args[0].to_blob() else { return Value::null(); }; - let slice = unsafe{ std::slice::from_raw_parts(blob.data, blob.size as usize)}; - let uuid = uuid::Uuid::from_slice(slice).unwrap(); + let uuid = uuid::Uuid::from_slice(blob.as_slice()).unwrap(); let unix = uuid_to_unix(uuid.as_bytes()); Value::from_integer(unix as i64) } @@ -100,7 +98,7 @@ declare_scalar_functions! { let Some(text) = args[0].to_text() else { return Value::null(); }; - let Ok(uuid) = uuid::Uuid::parse_str(unsafe {text.as_str()}) else { + let Ok(uuid) = uuid::Uuid::parse_str(&text) else { return Value::null(); }; let unix = uuid_to_unix(uuid.as_bytes()); @@ -115,8 +113,7 @@ declare_scalar_functions! { let Some(blob) = args[0].to_blob() else { return Value::null(); }; - let slice = unsafe{ std::slice::from_raw_parts(blob.data, blob.size as usize)}; - let parsed = uuid::Uuid::from_slice(slice).ok().map(|u| u.to_string()); + let parsed = uuid::Uuid::from_slice(blob.as_slice()).ok().map(|u| u.to_string()); match parsed { Some(s) => Value::from_text(s), None => Value::null() @@ -128,7 +125,7 @@ declare_scalar_functions! { let Some(text) = args[0].to_text() else { return Value::null(); }; - match uuid::Uuid::parse_str(unsafe {text.as_str()}) { + match uuid::Uuid::parse_str(&text) { Ok(uuid) => { Value::from_blob(uuid.as_bytes().to_vec()) } diff --git a/limbo_extension/src/lib.rs b/limbo_extension/src/lib.rs index d07cc8ea7..0666c588b 100644 --- a/limbo_extension/src/lib.rs +++ b/limbo_extension/src/lib.rs @@ -50,19 +50,6 @@ macro_rules! register_scalar_functions { } } -/// Provide a cleaner interface to define scalar functions to extension authors -/// . e.g. -/// ``` -/// #[args(1)] -/// fn scalar_double(args: &[Value]) -> Value { -/// Value::from_integer(args[0].integer * 2) -/// } -/// -/// #[args(0..=2)] -/// fn scalar_sum(args: &[Value]) -> Value { -/// Value::from_integer(args.iter().map(|v| v.integer).sum()) -/// ``` -/// #[macro_export] macro_rules! declare_scalar_functions { ( @@ -100,7 +87,7 @@ macro_rules! declare_scalar_functions { } #[repr(C)] -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Clone, Copy)] pub enum ValueType { Null, Integer, @@ -111,8 +98,8 @@ pub enum ValueType { #[repr(C)] pub struct Value { - pub value_type: ValueType, - pub value: *mut c_void, + value_type: ValueType, + value: *mut c_void, } impl std::fmt::Debug for Value { @@ -161,41 +148,27 @@ impl Default for TextValue { } impl TextValue { - pub fn new(text: *const u8, len: usize) -> Self { + pub(crate) fn new(text: *const u8, len: usize) -> Self { Self { text, len: len as u32, } } - /// # Safety - /// Safe to call if the pointer is null, returns None - /// if the value is not a text type or if the value is null - pub unsafe fn from_value(value: &Value) -> Option<&Self> { - if value.value_type != ValueType::Text { - return None; - } - if value.value.is_null() { - return None; - } - Some(&*(value.value as *const TextValue)) - } - - /// # Safety - /// If self.text is null we safely return an empty string but - /// the caller must ensure that the underlying value is valid utf8 - pub unsafe fn as_str(&self) -> &str { + fn as_str(&self) -> &str { if self.text.is_null() { return ""; } - std::str::from_utf8_unchecked(std::slice::from_raw_parts(self.text, self.len as usize)) + unsafe { + std::str::from_utf8_unchecked(std::slice::from_raw_parts(self.text, self.len as usize)) + } } } #[repr(C)] pub struct Blob { - pub data: *const u8, - pub size: u64, + data: *const u8, + size: u64, } impl std::fmt::Debug for Blob { @@ -218,6 +191,10 @@ impl Value { } } + pub fn value_type(&self) -> ValueType { + self.value_type + } + pub fn to_float(&self) -> Option { if self.value_type != ValueType::Float { return None; @@ -228,24 +205,27 @@ impl Value { Some(unsafe { *(self.value as *const f64) }) } - pub fn to_text(&self) -> Option<&TextValue> { + pub fn to_text(&self) -> Option { if self.value_type != ValueType::Text { return None; } if self.value.is_null() { return None; } - unsafe { Some(&*(self.value as *const TextValue)) } + let txt = unsafe { &*(self.value as *const TextValue) }; + Some(String::from(txt.as_str())) } - pub fn to_blob(&self) -> Option<&Blob> { + pub fn to_blob(&self) -> Option> { if self.value_type != ValueType::Blob { return None; } if self.value.is_null() { return None; } - unsafe { Some(&*(self.value as *const Blob)) } + let blob = unsafe { &*(self.value as *const Blob) }; + let slice = unsafe { std::slice::from_raw_parts(blob.data, blob.size as usize) }; + Some(slice.to_vec()) } pub fn to_integer(&self) -> Option { diff --git a/testing/extensions.py b/testing/extensions.py new file mode 100755 index 000000000..74383be94 --- /dev/null +++ b/testing/extensions.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +import os +import subprocess +import select +import time +import uuid + +sqlite_exec = "./target/debug/limbo" +sqlite_flags = os.getenv("SQLITE_FLAGS", "-q").split(" ") + + +def init_limbo(): + pipe = subprocess.Popen( + [sqlite_exec, *sqlite_flags], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + ) + return pipe + + +def execute_sql(pipe, sql): + end_suffix = "END_OF_RESULT" + write_to_pipe(pipe, sql) + write_to_pipe(pipe, f"SELECT '{end_suffix}';\n") + stdout = pipe.stdout + stderr = pipe.stderr + output = "" + while True: + ready_to_read, _, error_in_pipe = select.select( + [stdout, stderr], [], [stdout, stderr] + ) + ready_to_read_or_err = set(ready_to_read + error_in_pipe) + if stderr in ready_to_read_or_err: + exit_on_error(stderr) + + if stdout in ready_to_read_or_err: + fragment = stdout.read(select.PIPE_BUF) + output += fragment.decode() + if output.rstrip().endswith(end_suffix): + output = output.rstrip().removesuffix(end_suffix) + break + output = strip_each_line(output) + return output + + +def strip_each_line(lines: str) -> str: + lines = lines.split("\n") + lines = [line.strip() for line in lines if line != ""] + return "\n".join(lines) + + +def write_to_pipe(pipe, command): + if pipe.stdin is None: + raise RuntimeError("Failed to write to shell") + pipe.stdin.write((command + "\n").encode()) + pipe.stdin.flush() + + +def exit_on_error(stderr): + while True: + ready_to_read, _, _ = select.select([stderr], [], []) + if not ready_to_read: + break + print(stderr.read().decode(), end="") + exit(1) + + +def run_test(pipe, sql, validator=None): + print(f"Running test: {sql}") + result = execute_sql(pipe, sql) + if validator is not None: + if not validator(result): + print(f"Test FAILED: {sql}") + print(f"Returned: {result}") + raise Exception("Validation failed") + print("Test PASSED") + + +def validate_blob(result): + # HACK: blobs are difficult to test because the shell + # tries to return them as utf8 strings, so we call hex + # and assert they are valid hex digits + return int(result, 16) is not None + + +def validate_string_uuid(result): + return len(result) == 36 and result.count("-") == 4 + + +def returns_null(result): + return result == "" or result == b"\n" or result == b"" + + +def assert_now_unixtime(result): + return result == str(int(time.time())) + + +def assert_specific_time(result): + return result == "1736720789" + + +def main(): + specific_time = "01945ca0-3189-76c0-9a8f-caf310fc8b8e" + extension_path = "./target/debug/liblimbo_uuid.so" + pipe = init_limbo() + try: + # before extension loads, assert no function + run_test(pipe, "SELECT uuid4();", returns_null) + run_test(pipe, "SELECT uuid4_str();", returns_null) + run_test(pipe, f".load {extension_path}", returns_null) + print("Extension loaded successfully.") + run_test(pipe, "SELECT hex(uuid4());", validate_blob) + run_test(pipe, "SELECT uuid4_str();", validate_string_uuid) + run_test(pipe, "SELECT hex(uuid7());", validate_blob) + run_test( + pipe, + "SELECT uuid7_timestamp_ms(uuid7()) / 1000;", + ) + run_test(pipe, "SELECT uuid7_str();", validate_string_uuid) + run_test(pipe, "SELECT uuid_str(uuid7());", validate_string_uuid) + run_test(pipe, "SELECT hex(uuid_blob(uuid7_str()));", validate_blob) + run_test(pipe, "SELECT uuid_str(uuid_blob(uuid7_str()));", validate_string_uuid) + run_test( + pipe, + f"SELECT uuid7_timestamp_ms('{specific_time}') / 1000;", + assert_specific_time, + ) + except Exception as e: + print(f"Test FAILED: {e}") + pipe.terminate() + exit(1) + pipe.terminate() + print("All tests passed successfully.") + + +if __name__ == "__main__": + main() From fcfee24c5074fb92fc7947a915c1f60c265e2f0e Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Tue, 14 Jan 2025 16:06:42 +0200 Subject: [PATCH 62/97] Remove mark_last_insn_constant() from places where it is not safe to do so --- core/translate/expr.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/translate/expr.rs b/core/translate/expr.rs index d13902433..eadbc40e1 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -1651,7 +1651,6 @@ pub fn translate_expr( dest: target_register, }); } - program.mark_last_insn_constant(); Ok(target_register) } (UnaryOperator::Negative | UnaryOperator::Positive, _) => { @@ -1690,7 +1689,6 @@ pub fn translate_expr( dest: target_register, }); } - program.mark_last_insn_constant(); Ok(target_register) } (UnaryOperator::BitwiseNot, ast::Expr::Literal(ast::Literal::Null)) => { @@ -1698,7 +1696,6 @@ pub fn translate_expr( dest: target_register, dest_end: None, }); - program.mark_last_insn_constant(); Ok(target_register) } (UnaryOperator::BitwiseNot, _) => { From 3cbb2d2d7c5f43d2b245d5051d80f2ca5a91b4e3 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Tue, 14 Jan 2025 16:22:16 +0200 Subject: [PATCH 63/97] Add regression test for multi insert with unary operator --- test/src/lib.rs | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/test/src/lib.rs b/test/src/lib.rs index 931c9b1bf..9aa9116d5 100644 --- a/test/src/lib.rs +++ b/test/src/lib.rs @@ -110,6 +110,73 @@ mod tests { Ok(()) } + #[test] + /// There was a regression with inserting multiple rows with a column containing an unary operator :) + /// https://github.com/tursodatabase/limbo/pull/679 + fn test_regression_multi_row_insert() -> anyhow::Result<()> { + let _ = env_logger::try_init(); + let tmp_db = TempDatabase::new("CREATE TABLE test (x REAL);"); + let conn = tmp_db.connect_limbo(); + + let insert_query = "INSERT INTO test VALUES (-2), (-3), (-1)"; + let list_query = "SELECT * FROM test"; + + match conn.query(insert_query) { + Ok(Some(ref mut rows)) => loop { + match rows.next_row()? { + StepResult::IO => { + tmp_db.io.run_once()?; + } + StepResult::Done => break, + _ => unreachable!(), + } + }, + Ok(None) => {} + Err(err) => { + eprintln!("{}", err); + } + }; + + do_flush(&conn, &tmp_db)?; + + let mut current_read_index = 1; + let expected_ids = vec![-3, -2, -1]; + let mut actual_ids = Vec::new(); + match conn.query(list_query) { + Ok(Some(ref mut rows)) => loop { + match rows.next_row()? { + StepResult::Row(row) => { + let first_value = row.values.first().expect("missing id"); + let id = match first_value { + Value::Float(f) => *f as i32, + _ => panic!("expected float"), + }; + actual_ids.push(id); + current_read_index += 1; + } + StepResult::IO => { + tmp_db.io.run_once()?; + } + StepResult::Interrupt => break, + StepResult::Done => break, + StepResult::Busy => { + panic!("Database is busy"); + } + } + }, + Ok(None) => {} + Err(err) => { + eprintln!("{}", err); + } + } + + assert_eq!(current_read_index, 4); // Verify we read all rows + // sort ids + actual_ids.sort(); + assert_eq!(actual_ids, expected_ids); + Ok(()) + } + #[test] fn test_simple_overflow_page() -> anyhow::Result<()> { let _ = env_logger::try_init(); From 5305a9d0fdeed927eeadd29fa9966968c66b6adc Mon Sep 17 00:00:00 2001 From: Kould Date: Fri, 3 Jan 2025 01:42:36 +0800 Subject: [PATCH 64/97] feat: support keyword `rowid` --- core/translate/expr.rs | 9 ++ core/translate/planner.rs | 93 +++++++++++++------ testing/join.test | 16 ++++ testing/select.test | 8 ++ .../sqlite3-parser/src/parser/ast/check.rs | 3 + vendored/sqlite3-parser/src/parser/ast/fmt.rs | 1 + vendored/sqlite3-parser/src/parser/ast/mod.rs | 7 ++ 7 files changed, 111 insertions(+), 26 deletions(-) diff --git a/core/translate/expr.rs b/core/translate/expr.rs index d13902433..965326131 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -1545,6 +1545,15 @@ pub fn translate_expr( } } } + ast::Expr::RowId { database: _, table } => { + let tbl_ref = referenced_tables.as_ref().unwrap().get(*table).unwrap(); + let cursor_id = program.resolve_cursor_id(&tbl_ref.table_identifier); + program.emit_insn(Insn::RowId { + cursor_id, + dest: target_register, + }); + Ok(target_register) + } ast::Expr::InList { .. } => todo!(), ast::Expr::InSelect { .. } => todo!(), ast::Expr::InTable { .. } => todo!(), diff --git a/core/translate/planner.rs b/core/translate/planner.rs index f5c835f8e..998145263 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -12,6 +12,8 @@ use crate::{ }; use sqlite3_parser::ast::{self, Expr, FromClause, JoinType, Limit}; +pub const ROWID: &'static str = "rowid"; + pub struct OperatorIdCounter { id: usize, } @@ -102,8 +104,18 @@ pub fn bind_column_references( if id.0.eq_ignore_ascii_case("true") || id.0.eq_ignore_ascii_case("false") { return Ok(()); } - let mut match_result = None; let normalized_id = normalize_ident(id.0.as_str()); + + if referenced_tables.len() > 0 { + if let Some(row_id_expr) = + parse_row_id(&normalized_id, 0, || referenced_tables.len() != 1)? + { + *expr = row_id_expr; + + return Ok(()); + } + } + let mut match_result = None; for (tbl_idx, table) in referenced_tables.iter().enumerate() { let col_idx = table .columns() @@ -140,6 +152,12 @@ pub fn bind_column_references( } let tbl_idx = matching_tbl_idx.unwrap(); let normalized_id = normalize_ident(id.0.as_str()); + + if let Some(row_id_expr) = parse_row_id(&normalized_id, tbl_idx, || false)? { + *expr = row_id_expr; + + return Ok(()); + } let col_idx = referenced_tables[tbl_idx] .columns() .iter() @@ -209,7 +227,7 @@ pub fn bind_column_references( Ok(()) } // Already bound earlier - ast::Expr::Column { .. } => Ok(()), + ast::Expr::Column { .. } | ast::Expr::RowId { .. } => Ok(()), ast::Expr::DoublyQualified(_, _, _) => todo!(), ast::Expr::Exists(_) => todo!(), ast::Expr::FunctionCallStar { .. } => Ok(()), @@ -491,17 +509,23 @@ fn parse_join( let left_tables = &tables[..table_index]; assert!(!left_tables.is_empty()); let right_table = &tables[table_index]; - let mut left_col = None; + let mut left_col = + parse_row_id(&name_normalized, 0, || left_tables.len() != 1)?; for (left_table_idx, left_table) in left_tables.iter().enumerate() { + if left_col.is_some() { + break; + } left_col = left_table .columns() .iter() .enumerate() .find(|(_, col)| col.name == name_normalized) - .map(|(idx, col)| (left_table_idx, idx, col)); - if left_col.is_some() { - break; - } + .map(|(idx, col)| ast::Expr::Column { + database: None, + table: left_table_idx, + column: idx, + is_rowid_alias: col.is_rowid_alias, + }); } if left_col.is_none() { crate::bail_parse_error!( @@ -509,33 +533,33 @@ fn parse_join( distinct_name.0 ); } - let right_col = right_table - .columns() - .iter() - .enumerate() - .find(|(_, col)| col.name == name_normalized); + let right_col = + parse_row_id(&name_normalized, right_table.table_index, || false)?.or_else( + || { + right_table + .table + .columns() + .iter() + .enumerate() + .find(|(_, col)| col.name == name_normalized) + .map(|(i, col)| ast::Expr::Column { + database: None, + table: right_table.table_index, + column: i, + is_rowid_alias: col.is_rowid_alias, + }) + }, + ); if right_col.is_none() { crate::bail_parse_error!( "cannot join using column {} - column not present in all tables", distinct_name.0 ); } - let (left_table_idx, left_col_idx, left_col) = left_col.unwrap(); - let (right_col_idx, right_col) = right_col.unwrap(); using_predicates.push(ast::Expr::Binary( - Box::new(ast::Expr::Column { - database: None, - table: left_table_idx, - column: left_col_idx, - is_rowid_alias: left_col.is_rowid_alias, - }), + Box::new(left_col.unwrap()), ast::Operator::Equals, - Box::new(ast::Expr::Column { - database: None, - table: right_table.table_index, - column: right_col_idx, - is_rowid_alias: right_col.is_rowid_alias, - }), + Box::new(right_col.unwrap()), )); } predicates = Some(using_predicates); @@ -582,3 +606,20 @@ pub fn break_predicate_at_and_boundaries( } } } + +fn parse_row_id(column_name: &str, table_id: usize, fn_check: F) -> Result> +where + F: FnOnce() -> bool, +{ + if column_name.eq_ignore_ascii_case(ROWID) { + if fn_check() { + crate::bail_parse_error!("ROWID is ambiguous"); + } + + return Ok(Some(ast::Expr::RowId { + database: None, // TODO: support different databases + table: table_id, + })); + } + Ok(None) +} diff --git a/testing/join.test b/testing/join.test index 7ebcdd6c5..2b6d6c45d 100755 --- a/testing/join.test +++ b/testing/join.test @@ -106,6 +106,22 @@ Jamie|coat Jamie|accessories Cindy|} +do_execsql_test left-join-row-id { + select u.rowid, p.rowid from users u left join products as p on u.rowid = p.rowid where u.rowid >= 10 limit 5; +} {10|10 +11|11 +12| +13| +14|} + +do_execsql_test left-join-row-id-2 { + select u.rowid, p.rowid from users u left join products as p using(rowid) where u.rowid >= 10 limit 5; +} {10|10 +11|11 +12| +13| +14|} + do_execsql_test left-join-constant-condition-true { select u.first_name, p.name from users u left join products as p on true limit 1; } {Jamie|hat} diff --git a/testing/select.test b/testing/select.test index 49f8021bc..9babd104c 100755 --- a/testing/select.test +++ b/testing/select.test @@ -80,6 +80,14 @@ do_execsql_test select_with_quoting_2 { select "users".`id` from users where `users`.[id] = 5; } {5} +do_execsql_test select-rowid { + select rowid, first_name from users u where rowid = 5; +} {5|Edward} + +do_execsql_test select-rowid-2 { + select users.rowid, first_name from users u where rowid = 5; +} {5|Edward} + do_execsql_test seekrowid { select * from users u where u.id = 5; } {"5|Edward|Miller|christiankramer@example.com|725-281-1033|08522 English Plain|Lake Keith|ID|23283|15"} diff --git a/vendored/sqlite3-parser/src/parser/ast/check.rs b/vendored/sqlite3-parser/src/parser/ast/check.rs index 11a3cb031..e1e0eecd3 100644 --- a/vendored/sqlite3-parser/src/parser/ast/check.rs +++ b/vendored/sqlite3-parser/src/parser/ast/check.rs @@ -194,6 +194,9 @@ impl CreateTableBody { { let mut generated_count = 0; for c in columns.values() { + if c.col_name == "rowid" { + return Err(custom_err!("cannot use reserved word: ROWID")); + } for cs in &c.constraints { if let ColumnConstraint::Generated { .. } = cs.constraint { generated_count += 1; diff --git a/vendored/sqlite3-parser/src/parser/ast/fmt.rs b/vendored/sqlite3-parser/src/parser/ast/fmt.rs index 39c929409..34a7fa3f0 100644 --- a/vendored/sqlite3-parser/src/parser/ast/fmt.rs +++ b/vendored/sqlite3-parser/src/parser/ast/fmt.rs @@ -728,6 +728,7 @@ impl ToTokens for Expr { } s.append(TK_RP, None) } + Self::RowId { .. } => Ok(()), Self::Subquery(query) => { s.append(TK_LP, None)?; query.to_tokens(s)?; diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 50c7bcacf..466a718ab 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -338,6 +338,13 @@ pub enum Expr { /// is the column a rowid alias is_rowid_alias: bool, }, + /// `ROWID` + RowId { + /// the x in `x.y.z`. index of the db in catalog. + database: Option, + /// the y in `x.y.z`. index of the table in catalog. + table: usize, + }, /// `IN` InList { /// expression From 1bf651bd375d406a866eca44dfeb3c707b7f5b60 Mon Sep 17 00:00:00 2001 From: Kould Date: Tue, 14 Jan 2025 22:56:49 +0800 Subject: [PATCH 65/97] chore: rollback using rowid(sqlite3 unsupported) --- core/translate/planner.rs | 54 +++++++++++++++++---------------------- testing/join.test | 8 ------ testing/select.test | 2 +- 3 files changed, 25 insertions(+), 39 deletions(-) diff --git a/core/translate/planner.rs b/core/translate/planner.rs index 998145263..d15a2fab6 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -509,23 +509,17 @@ fn parse_join( let left_tables = &tables[..table_index]; assert!(!left_tables.is_empty()); let right_table = &tables[table_index]; - let mut left_col = - parse_row_id(&name_normalized, 0, || left_tables.len() != 1)?; + let mut left_col = None; for (left_table_idx, left_table) in left_tables.iter().enumerate() { - if left_col.is_some() { - break; - } left_col = left_table .columns() .iter() .enumerate() .find(|(_, col)| col.name == name_normalized) - .map(|(idx, col)| ast::Expr::Column { - database: None, - table: left_table_idx, - column: idx, - is_rowid_alias: col.is_rowid_alias, - }); + .map(|(idx, col)| (left_table_idx, idx, col)); + if left_col.is_some() { + break; + } } if left_col.is_none() { crate::bail_parse_error!( @@ -533,33 +527,33 @@ fn parse_join( distinct_name.0 ); } - let right_col = - parse_row_id(&name_normalized, right_table.table_index, || false)?.or_else( - || { - right_table - .table - .columns() - .iter() - .enumerate() - .find(|(_, col)| col.name == name_normalized) - .map(|(i, col)| ast::Expr::Column { - database: None, - table: right_table.table_index, - column: i, - is_rowid_alias: col.is_rowid_alias, - }) - }, - ); + let right_col = right_table + .columns() + .iter() + .enumerate() + .find(|(_, col)| col.name == name_normalized); if right_col.is_none() { crate::bail_parse_error!( "cannot join using column {} - column not present in all tables", distinct_name.0 ); } + let (left_table_idx, left_col_idx, left_col) = left_col.unwrap(); + let (right_col_idx, right_col) = right_col.unwrap(); using_predicates.push(ast::Expr::Binary( - Box::new(left_col.unwrap()), + Box::new(ast::Expr::Column { + database: None, + table: left_table_idx, + column: left_col_idx, + is_rowid_alias: left_col.is_rowid_alias, + }), ast::Operator::Equals, - Box::new(right_col.unwrap()), + Box::new(ast::Expr::Column { + database: None, + table: right_table.table_index, + column: right_col_idx, + is_rowid_alias: right_col.is_rowid_alias, + }), )); } predicates = Some(using_predicates); diff --git a/testing/join.test b/testing/join.test index 2b6d6c45d..64b3dcbd3 100755 --- a/testing/join.test +++ b/testing/join.test @@ -114,14 +114,6 @@ do_execsql_test left-join-row-id { 13| 14|} -do_execsql_test left-join-row-id-2 { - select u.rowid, p.rowid from users u left join products as p using(rowid) where u.rowid >= 10 limit 5; -} {10|10 -11|11 -12| -13| -14|} - do_execsql_test left-join-constant-condition-true { select u.first_name, p.name from users u left join products as p on true limit 1; } {Jamie|hat} diff --git a/testing/select.test b/testing/select.test index 9babd104c..e39030a22 100755 --- a/testing/select.test +++ b/testing/select.test @@ -85,7 +85,7 @@ do_execsql_test select-rowid { } {5|Edward} do_execsql_test select-rowid-2 { - select users.rowid, first_name from users u where rowid = 5; + select u.rowid, first_name from users u where rowid = 5; } {5|Edward} do_execsql_test seekrowid { From 1df1f7afc55e0b74bb91044fa33e3d6ca12dd7b0 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 17:19:58 +0200 Subject: [PATCH 66/97] Add scripts/run-sim helper ...to run the simulator in a loop with different seeds. --- scripts/run-sim | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 scripts/run-sim diff --git a/scripts/run-sim b/scripts/run-sim new file mode 100755 index 000000000..69192eaca --- /dev/null +++ b/scripts/run-sim @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +while true; do + cargo run -p limbo_sim +done From 5b4c7ec7f5c7c04bb15731086a55cf5d0d56c256 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 17:24:14 +0200 Subject: [PATCH 67/97] simulator: Rename stats in SimulatorFile --- simulator/runner/file.rs | 29 ++++++++++++++++++++--------- simulator/runner/io.rs | 6 +++--- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/simulator/runner/file.rs b/simulator/runner/file.rs index e0153f2b3..bc23eff89 100644 --- a/simulator/runner/file.rs +++ b/simulator/runner/file.rs @@ -4,11 +4,22 @@ use limbo_core::{File, Result}; pub(crate) struct SimulatorFile { pub(crate) inner: Rc, pub(crate) fault: RefCell, + + /// Number of `pread` function calls (both success and failures). + pub(crate) nr_pread_calls: RefCell, + + /// Number of `pread` function calls with injected fault. pub(crate) nr_pread_faults: RefCell, + + /// Number of `pwrite` function calls (both success and failures). + pub(crate) nr_pwrite_calls: RefCell, + + /// Number of `pwrite` function calls with injected fault. pub(crate) nr_pwrite_faults: RefCell, - pub(crate) writes: RefCell, - pub(crate) reads: RefCell, - pub(crate) syncs: RefCell, + + /// Number of `sync` function calls (both success and failures). + pub(crate) nr_sync_calls: RefCell, + pub(crate) page_size: usize, } @@ -22,9 +33,9 @@ impl SimulatorFile { "pread faults: {}, pwrite faults: {}, reads: {}, writes: {}, syncs: {}", *self.nr_pread_faults.borrow(), *self.nr_pwrite_faults.borrow(), - *self.reads.borrow(), - *self.writes.borrow(), - *self.syncs.borrow(), + *self.nr_pread_calls.borrow(), + *self.nr_pwrite_calls.borrow(), + *self.nr_sync_calls.borrow(), ); } } @@ -49,13 +60,13 @@ impl limbo_core::File for SimulatorFile { } fn pread(&self, pos: usize, c: Rc) -> Result<()> { + *self.nr_pread_calls.borrow_mut() += 1; if *self.fault.borrow() { *self.nr_pread_faults.borrow_mut() += 1; return Err(limbo_core::LimboError::InternalError( "Injected fault".into(), )); } - *self.reads.borrow_mut() += 1; self.inner.pread(pos, c) } @@ -65,18 +76,18 @@ impl limbo_core::File for SimulatorFile { buffer: Rc>, c: Rc, ) -> Result<()> { + *self.nr_pwrite_calls.borrow_mut() += 1; if *self.fault.borrow() { *self.nr_pwrite_faults.borrow_mut() += 1; return Err(limbo_core::LimboError::InternalError( "Injected fault".into(), )); } - *self.writes.borrow_mut() += 1; self.inner.pwrite(pos, buffer, c) } fn sync(&self, c: Rc) -> Result<()> { - *self.syncs.borrow_mut() += 1; + *self.nr_sync_calls.borrow_mut() += 1; self.inner.sync(c) } diff --git a/simulator/runner/io.rs b/simulator/runner/io.rs index c039764b0..ce4f45231 100644 --- a/simulator/runner/io.rs +++ b/simulator/runner/io.rs @@ -60,9 +60,9 @@ impl IO for SimulatorIO { fault: RefCell::new(false), nr_pread_faults: RefCell::new(0), nr_pwrite_faults: RefCell::new(0), - reads: RefCell::new(0), - writes: RefCell::new(0), - syncs: RefCell::new(0), + nr_pread_calls: RefCell::new(0), + nr_pwrite_calls: RefCell::new(0), + nr_sync_calls: RefCell::new(0), page_size: self.page_size, }); self.files.borrow_mut().push(file.clone()); From 14ec057a3414dd6fdc790ac30528e6af1f35f47c Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 17:30:11 +0200 Subject: [PATCH 68/97] simulator: Make stats printout prettier ``` op calls faults --------- -------- -------- pread 3 0 pwrite 1 0 sync 0 0 --------- -------- -------- total 4 0 ``` --- simulator/runner/file.rs | 25 ++++++++++++++++++++----- simulator/runner/io.rs | 3 +++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/simulator/runner/file.rs b/simulator/runner/file.rs index bc23eff89..685a60af2 100644 --- a/simulator/runner/file.rs +++ b/simulator/runner/file.rs @@ -29,14 +29,29 @@ impl SimulatorFile { } pub(crate) fn print_stats(&self) { + println!("op calls faults"); + println!("--------- -------- --------"); println!( - "pread faults: {}, pwrite faults: {}, reads: {}, writes: {}, syncs: {}", - *self.nr_pread_faults.borrow(), - *self.nr_pwrite_faults.borrow(), + "pread {:8} {:8}", *self.nr_pread_calls.borrow(), - *self.nr_pwrite_calls.borrow(), - *self.nr_sync_calls.borrow(), + *self.nr_pread_faults.borrow() ); + println!( + "pwrite {:8} {:8}", + *self.nr_pwrite_calls.borrow(), + *self.nr_pwrite_faults.borrow() + ); + println!( + "sync {:8} {:8}", + *self.nr_sync_calls.borrow(), + 0 // No fault counter for sync + ); + println!("--------- -------- --------"); + let sum_calls = *self.nr_pread_calls.borrow() + + *self.nr_pwrite_calls.borrow() + + *self.nr_sync_calls.borrow(); + let sum_faults = *self.nr_pread_faults.borrow() + *self.nr_pwrite_faults.borrow(); + println!("total {:8} {:8}", sum_calls, sum_faults); } } diff --git a/simulator/runner/io.rs b/simulator/runner/io.rs index ce4f45231..e7802b40f 100644 --- a/simulator/runner/io.rs +++ b/simulator/runner/io.rs @@ -42,7 +42,10 @@ impl SimulatorIO { pub(crate) fn print_stats(&self) { println!("run_once faults: {}", self.nr_run_once_faults.borrow()); for file in self.files.borrow().iter() { + println!(); + println!("==========================="); file.print_stats(); + println!(); } } } From e1f5fa875ea6339ec3a8c7723299d56bb3a1b37f Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 17:37:53 +0200 Subject: [PATCH 69/97] simulator: Make simulator runs longer by default --- simulator/runner/cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index 8ad42c8b3..71923a92e 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -18,14 +18,14 @@ pub struct SimulatorCLI { short = 'n', long, help = "change the maximum size of the randomly generated sequence of interactions", - default_value_t = 1024 + default_value_t = 20000 )] pub maximum_size: usize, #[clap( short = 'k', long, help = "change the minimum size of the randomly generated sequence of interactions", - default_value_t = 1 + default_value_t = 10000 )] pub minimum_size: usize, #[clap( From 3c6c6041ffa7c7e0c41676c322f753cce0268701 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 17:41:07 +0200 Subject: [PATCH 70/97] simulator: Log query errors with debug level ...it is totally fine for a SQL query to fail as part of the simulation. For example, if we attempt to create a table that already exists, the expectation is that the query fails. No need to spam the logs. --- simulator/generation/plan.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index dbf6dd7f6..367a18c02 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -209,7 +209,7 @@ impl Interaction { let rows = conn.query(&query_str); if rows.is_err() { let err = rows.err(); - log::error!( + log::debug!( "Error running query '{}': {:?}", &query_str[0..query_str.len().min(4096)], err From d355ce785cac40e1a88ebf084014a7198ac43773 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 17:54:17 +0200 Subject: [PATCH 71/97] core/storage: Remove debug printout --- core/storage/sqlite3_ondisk.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 8dbda073b..5b54be756 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -1245,7 +1245,6 @@ pub fn begin_write_wal_frame( *write_counter.borrow_mut() += 1; let write_complete = { let buf_copy = buffer.clone(); - log::info!("finished"); Box::new(move |bytes_written: i32| { let buf_copy = buf_copy.clone(); let buf_len = buf_copy.borrow().len(); From 30a380cab137083d8f5a2260794aa86507a1954e Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 17:54:29 +0200 Subject: [PATCH 72/97] simulator: Move more logging under trace level --- simulator/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simulator/main.rs b/simulator/main.rs index 52c33d5ec..504adf50b 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -269,7 +269,7 @@ fn execute_plan( let interaction = &plan.plan[plan.interaction_pointer]; if let SimConnection::Disconnected = connection { - log::info!("connecting {}", connection_index); + log::trace!("connecting {}", connection_index); env.connections[connection_index] = SimConnection::Connected(env.db.connect()); } else { match execute_interaction(env, connection_index, interaction, &mut plan.stack) { @@ -293,7 +293,7 @@ fn execute_interaction( interaction: &Interaction, stack: &mut Vec, ) -> Result<()> { - log::info!("executing: {}", interaction); + log::trace!("executing: {}", interaction); match interaction { generation::plan::Interaction::Query(_) => { let conn = match &mut env.connections[connection_index] { From 0c7ebd4df5eaa20b964c862e01e33a57e4e041b5 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 17:54:39 +0200 Subject: [PATCH 73/97] simulator: Enable info-level logging by default --- simulator/main.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/simulator/main.rs b/simulator/main.rs index 504adf50b..3d9752b77 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -20,7 +20,7 @@ mod model; mod runner; fn main() { - let _ = env_logger::try_init(); + init_logger(); let cli_opts = SimulatorCLI::parse(); @@ -326,3 +326,11 @@ fn compare_equal_rows(a: &[Vec], b: &[Vec]) { } } } + +fn init_logger() { + env_logger::Builder::from_env(env_logger::Env::default().filter_or("RUST_LOG", "info")) + .format_timestamp(None) + .format_module_path(false) + .format_target(false) + .init(); +} From 053bc0b8cfb2a3cc8644dfd96bad3a905032796b Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 18:37:09 +0200 Subject: [PATCH 74/97] Add Jussi to .github.json --- .github.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github.json b/.github.json index 410ca2471..6297affda 100644 --- a/.github.json +++ b/.github.json @@ -6,5 +6,9 @@ "pereman2": { "name": "Pere Diaz Bou", "email": "pere-altea@homail.com" + }, + "jussisaurio": { + "name": "Jussi Saurio", + "email": "jussi.saurio@gmail.com" } } From a9ffa7215127f45d1d765566a644cb11b8552028 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 18:40:03 +0200 Subject: [PATCH 75/97] simulator: Replace println() calls with log::info() --- simulator/runner/file.rs | 14 +++++++------- simulator/runner/io.rs | 7 +++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/simulator/runner/file.rs b/simulator/runner/file.rs index 685a60af2..17651d221 100644 --- a/simulator/runner/file.rs +++ b/simulator/runner/file.rs @@ -29,29 +29,29 @@ impl SimulatorFile { } pub(crate) fn print_stats(&self) { - println!("op calls faults"); - println!("--------- -------- --------"); - println!( + log::info!("op calls faults"); + log::info!("--------- -------- --------"); + log::info!( "pread {:8} {:8}", *self.nr_pread_calls.borrow(), *self.nr_pread_faults.borrow() ); - println!( + log::info!( "pwrite {:8} {:8}", *self.nr_pwrite_calls.borrow(), *self.nr_pwrite_faults.borrow() ); - println!( + log::info!( "sync {:8} {:8}", *self.nr_sync_calls.borrow(), 0 // No fault counter for sync ); - println!("--------- -------- --------"); + log::info!("--------- -------- --------"); let sum_calls = *self.nr_pread_calls.borrow() + *self.nr_pwrite_calls.borrow() + *self.nr_sync_calls.borrow(); let sum_faults = *self.nr_pread_faults.borrow() + *self.nr_pwrite_faults.borrow(); - println!("total {:8} {:8}", sum_calls, sum_faults); + log::info!("total {:8} {:8}", sum_calls, sum_faults); } } diff --git a/simulator/runner/io.rs b/simulator/runner/io.rs index e7802b40f..2da707de7 100644 --- a/simulator/runner/io.rs +++ b/simulator/runner/io.rs @@ -40,12 +40,11 @@ impl SimulatorIO { } pub(crate) fn print_stats(&self) { - println!("run_once faults: {}", self.nr_run_once_faults.borrow()); + log::info!("run_once faults: {}", self.nr_run_once_faults.borrow()); for file in self.files.borrow().iter() { - println!(); - println!("==========================="); + log::info!(""); + log::info!("==========================="); file.print_stats(); - println!(); } } } From 343ccb3f72e409575fdec540349e206901f525e4 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Tue, 14 Jan 2025 11:49:34 -0500 Subject: [PATCH 76/97] Replace declare_scalar_functions in extension API with proc macro --- Cargo.lock | 46 +++++---- extensions/uuid/src/lib.rs | 196 +++++++++++++++++++------------------ limbo_extension/Cargo.toml | 1 + limbo_extension/src/lib.rs | 39 +------- macros/Cargo.toml | 5 + macros/src/args.rs | 63 ++++++++++++ macros/src/lib.rs | 122 +++++++++++++++++++++++ 7 files changed, 319 insertions(+), 153 deletions(-) create mode 100644 macros/src/args.rs diff --git a/Cargo.lock b/Cargo.lock index 7de3e3500..9cbce1207 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -331,7 +331,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -665,7 +665,7 @@ checksum = "3bf679796c0322556351f287a51b49e48f7c4986e727b5dd78c972d30e2e16cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -809,7 +809,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1251,6 +1251,7 @@ dependencies = [ name = "limbo_extension" version = "0.0.12" dependencies = [ + "limbo_macros", "log", ] @@ -1266,6 +1267,11 @@ dependencies = [ [[package]] name = "limbo_macros" version = "0.0.12" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] [[package]] name = "limbo_sim" @@ -1374,7 +1380,7 @@ checksum = "23c9b935fbe1d6cbd1dac857b54a688145e2d93f48db36010514d0f612d0ad67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1435,7 +1441,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1595,7 +1601,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1841,7 +1847,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1854,7 +1860,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1868,9 +1874,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -2024,7 +2030,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.90", + "syn 2.0.96", "unicode-ident", ] @@ -2137,7 +2143,7 @@ checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2315,9 +2321,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -2404,7 +2410,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2415,7 +2421,7 @@ checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2454,7 +2460,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2588,7 +2594,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "wasm-bindgen-shared", ] @@ -2623,7 +2629,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2916,5 +2922,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index e6a3a4f9b..f8d8f3816 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -1,5 +1,5 @@ use limbo_extension::{ - declare_scalar_functions, register_extension, register_scalar_functions, Value, ValueType, + export_scalar, register_extension, register_scalar_functions, Value, ValueType, }; register_extension! { @@ -11,31 +11,34 @@ register_extension! { "uuid_str" => uuid_str, "uuid_blob" => uuid_blob, "uuid7_timestamp_ms" => exec_ts_from_uuid7, + "gen_random_uuid" => uuid4_str, }, } -declare_scalar_functions! { - #[args(0)] - fn uuid4_str(_args: &[Value]) -> Value { - let uuid = uuid::Uuid::new_v4().to_string(); - Value::from_text(uuid) - } +#[export_scalar] +#[args(0)] +fn uuid4_str(_args: &[Value]) -> Value { + let uuid = uuid::Uuid::new_v4().to_string(); + Value::from_text(uuid) +} - #[args(0)] - fn uuid4_blob(_args: &[Value]) -> Value { - let uuid = uuid::Uuid::new_v4(); - let bytes = uuid.as_bytes(); - Value::from_blob(bytes.to_vec()) - } +#[export_scalar] +#[args(0)] +fn uuid4_blob(_args: &[Value]) -> Value { + let uuid = uuid::Uuid::new_v4(); + let bytes = uuid.as_bytes(); + Value::from_blob(bytes.to_vec()) +} - #[args(0..=1)] - fn uuid7_str(args: &[Value]) -> Value { - let timestamp = if args.is_empty() { - let ctx = uuid::ContextV7::new(); - uuid::Timestamp::now(ctx) - } else { - let arg = &args[0]; - match arg.value_type() { +#[export_scalar] +#[args(0..=1)] +fn uuid7_str(args: &[Value]) -> Value { + let timestamp = if args.is_empty() { + let ctx = uuid::ContextV7::new(); + uuid::Timestamp::now(ctx) + } else { + let arg = &args[0]; + match arg.value_type() { ValueType::Integer => { let ctx = uuid::ContextV7::new(); let Some(int) = arg.to_integer() else { @@ -43,94 +46,97 @@ declare_scalar_functions! { }; uuid::Timestamp::from_unix(ctx, int as u64, 0) } - ValueType::Text => { - let Some(text) = arg.to_text() else { - return Value::null(); - }; + ValueType::Text => { + let Some(text) = arg.to_text() else { + return Value::null(); + }; match text.parse::() { Ok(unix) => { - if unix <= 0 { - return Value::null(); - } + if unix <= 0 { + return Value::null(); + } uuid::Timestamp::from_unix(uuid::ContextV7::new(), unix as u64, 0) } Err(_) => return Value::null(), } } _ => return Value::null(), - } - }; - let uuid = uuid::Uuid::new_v7(timestamp); - Value::from_text(uuid.to_string()) - } + } + }; + let uuid = uuid::Uuid::new_v7(timestamp); + Value::from_text(uuid.to_string()) +} - #[args(0..=1)] - fn uuid7_blob(args: &[Value]) -> Value { - let timestamp = if args.is_empty() { - let ctx = uuid::ContextV7::new(); - uuid::Timestamp::now(ctx) - } else if args[0].value_type() == limbo_extension::ValueType::Integer { - let ctx = uuid::ContextV7::new(); - let Some(int) = args[0].to_integer() else { - return Value::null(); - }; - uuid::Timestamp::from_unix(ctx, int as u64, 0) - } else { +#[export_scalar] +#[args(0..=1)] +fn uuid7_blob(args: &[Value]) -> Value { + let timestamp = if args.is_empty() { + let ctx = uuid::ContextV7::new(); + uuid::Timestamp::now(ctx) + } else if args[0].value_type() == limbo_extension::ValueType::Integer { + let ctx = uuid::ContextV7::new(); + let Some(int) = args[0].to_integer() else { + return Value::null(); + }; + uuid::Timestamp::from_unix(ctx, int as u64, 0) + } else { + return Value::null(); + }; + let uuid = uuid::Uuid::new_v7(timestamp); + let bytes = uuid.as_bytes(); + Value::from_blob(bytes.to_vec()) +} + +#[export_scalar] +#[args(1)] +fn exec_ts_from_uuid7(args: &[Value]) -> Value { + match args[0].value_type() { + ValueType::Blob => { + let Some(blob) = &args[0].to_blob() else { return Value::null(); - }; - let uuid = uuid::Uuid::new_v7(timestamp); - let bytes = uuid.as_bytes(); - Value::from_blob(bytes.to_vec()) - } - - #[args(1)] - fn exec_ts_from_uuid7(args: &[Value]) -> Value { - match args[0].value_type() { - ValueType::Blob => { - let Some(blob) = &args[0].to_blob() else { - return Value::null(); - }; - let uuid = uuid::Uuid::from_slice(blob.as_slice()).unwrap(); - let unix = uuid_to_unix(uuid.as_bytes()); - Value::from_integer(unix as i64) - } - ValueType::Text => { - let Some(text) = args[0].to_text() else { - return Value::null(); - }; - let Ok(uuid) = uuid::Uuid::parse_str(&text) else { - return Value::null(); - }; - let unix = uuid_to_unix(uuid.as_bytes()); - Value::from_integer(unix as i64) - } - _ => Value::null(), + }; + let uuid = uuid::Uuid::from_slice(blob.as_slice()).unwrap(); + let unix = uuid_to_unix(uuid.as_bytes()); + Value::from_integer(unix as i64) } - } - - #[args(1)] - fn uuid_str(args: &[Value]) -> Value { - let Some(blob) = args[0].to_blob() else { - return Value::null(); - }; - let parsed = uuid::Uuid::from_slice(blob.as_slice()).ok().map(|u| u.to_string()); - match parsed { - Some(s) => Value::from_text(s), - None => Value::null() + ValueType::Text => { + let Some(text) = args[0].to_text() else { + return Value::null(); + }; + let Ok(uuid) = uuid::Uuid::parse_str(&text) else { + return Value::null(); + }; + let unix = uuid_to_unix(uuid.as_bytes()); + Value::from_integer(unix as i64) } + _ => Value::null(), } +} - #[args(1)] - fn uuid_blob(args: &[Value]) -> Value { - let Some(text) = args[0].to_text() else { - return Value::null(); - }; - match uuid::Uuid::parse_str(&text) { - Ok(uuid) => { - Value::from_blob(uuid.as_bytes().to_vec()) - } - Err(_) => Value::null() - } +#[export_scalar] +#[args(1)] +fn uuid_str(args: &[Value]) -> Value { + let Some(blob) = args[0].to_blob() else { + return Value::null(); + }; + let parsed = uuid::Uuid::from_slice(blob.as_slice()) + .ok() + .map(|u| u.to_string()); + match parsed { + Some(s) => Value::from_text(s), + None => Value::null(), + } +} + +#[export_scalar] +#[args(1)] +fn uuid_blob(args: &[Value]) -> Value { + let Some(text) = args[0].to_text() else { + return Value::null(); + }; + match uuid::Uuid::parse_str(&text) { + Ok(uuid) => Value::from_blob(uuid.as_bytes().to_vec()), + Err(_) => Value::null(), } } diff --git a/limbo_extension/Cargo.toml b/limbo_extension/Cargo.toml index 2928ed853..94c0229e5 100644 --- a/limbo_extension/Cargo.toml +++ b/limbo_extension/Cargo.toml @@ -8,3 +8,4 @@ repository.workspace = true [dependencies] log = "0.4.20" +limbo_macros = { path = "../macros" } diff --git a/limbo_extension/src/lib.rs b/limbo_extension/src/lib.rs index 0666c588b..ab598cc09 100644 --- a/limbo_extension/src/lib.rs +++ b/limbo_extension/src/lib.rs @@ -1,7 +1,6 @@ use std::os::raw::{c_char, c_void}; - pub type ResultCode = i32; - +pub use limbo_macros::export_scalar; pub const RESULT_OK: ResultCode = 0; pub const RESULT_ERROR: ResultCode = 1; // TODO: more error types @@ -50,42 +49,6 @@ macro_rules! register_scalar_functions { } } -#[macro_export] -macro_rules! declare_scalar_functions { - ( - $( - #[args($($args_count:tt)+)] - fn $func_name:ident ($args:ident : &[Value]) -> Value $body:block - )* - ) => { - $( - extern "C" fn $func_name( - argc: i32, - argv: *const $crate::Value - ) -> $crate::Value { - let valid_args = { - match argc { - $($args_count)+ => true, - _ => false, - } - }; - if !valid_args { - return $crate::Value::null(); - } - if argc == 0 || argv.is_null() { - log::debug!("{} was called with no arguments", stringify!($func_name)); - let $args: &[$crate::Value] = &[]; - $body - } else { - let ptr_slice = unsafe{ std::slice::from_raw_parts(argv, argc as usize)}; - let $args: &[$crate::Value] = ptr_slice; - $body - } - } - )* - }; -} - #[repr(C)] #[derive(PartialEq, Eq, Clone, Copy)] pub enum ValueType { diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 78a3805c6..fb41bc18b 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -11,3 +11,8 @@ description = "The Limbo database library" [lib] proc-macro = true + +[dependencies] +quote = "1.0.38" +proc-macro2 = "1.0.38" +syn = { version = "2.0.96", features = ["full"]} diff --git a/macros/src/args.rs b/macros/src/args.rs new file mode 100644 index 000000000..d0988b5e9 --- /dev/null +++ b/macros/src/args.rs @@ -0,0 +1,63 @@ +use syn::parse::{Parse, ParseStream, Result as ParseResult}; +use syn::{LitInt, Token}; +#[derive(Debug)] +pub enum ArgsSpec { + Exact(i32), + Range { + lower: i32, + upper: i32, + inclusive: bool, + }, +} + +pub struct ArgsAttr { + pub spec: ArgsSpec, +} + +impl Parse for ArgsAttr { + fn parse(input: ParseStream) -> ParseResult { + if input.peek(LitInt) { + let start_lit = input.parse::()?; + let start_val = start_lit.base10_parse::()?; + + if input.is_empty() { + return Ok(ArgsAttr { + spec: ArgsSpec::Exact(start_val), + }); + } + if input.peek(Token![..=]) { + let _dots = input.parse::()?; + let end_lit = input.parse::()?; + let end_val = end_lit.base10_parse::()?; + Ok(ArgsAttr { + spec: ArgsSpec::Range { + lower: start_val, + upper: end_val, + inclusive: true, + }, + }) + } else if input.peek(Token![..]) { + let _dots = input.parse::()?; + let end_lit = input.parse::()?; + let end_val = end_lit.base10_parse::()?; + Ok(ArgsAttr { + spec: ArgsSpec::Range { + lower: start_val, + upper: end_val, + inclusive: false, + }, + }) + } else { + Err(syn::Error::new_spanned( + start_lit, + "Expected '..' or '..=' for a range, or nothing for a single integer.", + )) + } + } else { + Err(syn::Error::new( + input.span(), + "Expected an integer or a range expression, like `0`, `0..2`, or `0..=2`.", + )) + } + } +} diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 1cbf31e7a..5b21e2a90 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1,3 +1,5 @@ +mod args; +use args::{ArgsAttr, ArgsSpec}; extern crate proc_macro; use proc_macro::{token_stream::IntoIter, Group, TokenStream, TokenTree}; use std::collections::HashMap; @@ -133,3 +135,123 @@ fn generate_get_description( ); enum_impl.parse().unwrap() } + +use quote::quote; +use syn::{parse_macro_input, Attribute, Block, ItemFn}; +/// Macro to transform the preferred API for scalar functions in extensions into +/// an FFI-compatible function signature while validating argc +#[proc_macro_attribute] +pub fn export_scalar(_attr: TokenStream, item: TokenStream) -> TokenStream { + let mut input_fn = parse_macro_input!(item as ItemFn); + + let fn_name = &input_fn.sig.ident; + let fn_body: &Block = &input_fn.block; + + let mut extracted_spec: Option = None; + let mut arg_err = None; + let kept_attrs: Vec = input_fn + .attrs + .into_iter() + .filter_map(|attr| { + if attr.path().is_ident("args") { + let parsed_attr = match attr.parse_args::() { + Ok(p) => p, + Err(err) => { + arg_err = Some(err.to_compile_error()); + return None; + } + }; + extracted_spec = Some(parsed_attr.spec); + None + } else { + Some(attr) + } + }) + .collect(); + input_fn.attrs = kept_attrs; + if let Some(arg_err) = arg_err { + return arg_err.into(); + } + let spec = match extracted_spec { + Some(s) => s, + None => { + return syn::Error::new_spanned( + fn_name, + "Expected an attribute with integer or range: #[args(1)] #[args(0..2)], etc.", + ) + .to_compile_error() + .into() + } + }; + let arg_check = match spec { + ArgsSpec::Exact(exact_count) => { + quote! { + if argc != #exact_count { + log::error!( + "{} was called with {} arguments, expected exactly {}", + stringify!(#fn_name), + argc, + #exact_count + ); + return ::limbo_extension::Value::null(); + } + } + } + ArgsSpec::Range { + lower, + upper, + inclusive: true, + } => { + quote! { + if !(#lower..=#upper).contains(&argc) { + log::error!( + "{} was called with {} arguments, expected {}..={} range", + stringify!(#fn_name), + argc, + #lower, + #upper + ); + return ::limbo_extension::Value::null(); + } + } + } + ArgsSpec::Range { + lower, + upper, + inclusive: false, + } => { + quote! { + if !(#lower..#upper).contains(&argc) { + log::error!( + "{} was called with {} arguments, expected {}..{} (exclusive)", + stringify!(#fn_name), + argc, + #lower, + #upper + ); + return ::limbo_extension::Value::null(); + } + } + } + }; + let expanded = quote! { + #[export_name = stringify!(#fn_name)] + extern "C" fn #fn_name(argc: i32, argv: *const ::limbo_extension::Value) -> ::limbo_extension::Value { + #arg_check + + // from_raw_parts doesn't currently accept null ptr + if argc == 0 || argv.is_null() { + log::debug!("{} was called with no arguments", stringify!(#fn_name)); + let args: &[::limbo_extension::Value] = &[]; + #fn_body + } else { + let ptr_slice = unsafe { + std::slice::from_raw_parts(argv, argc as usize) + }; + let args: &[::limbo_extension::Value] = ptr_slice; + #fn_body + } + } + }; + TokenStream::from(expanded) +} From 23d9d09b70238ad44beac3e06a32b3bee1e23b9d Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Tue, 14 Jan 2025 10:09:03 -0500 Subject: [PATCH 77/97] Add load_extension function, resolve shared lib extensions --- COMPAT.md | 2 +- cli/app.rs | 5 ++++- core/function.rs | 6 ++++++ core/lib.rs | 31 +++++++++++++++++++++++++++++-- core/translate/expr.rs | 13 +++++++++++++ core/vdbe/mod.rs | 10 +++++++++- 6 files changed, 62 insertions(+), 5 deletions(-) diff --git a/COMPAT.md b/COMPAT.md index 5584899f4..5cd309bf2 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -120,7 +120,7 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html). | like(X,Y,Z) | Yes | | | likelihood(X,Y) | No | | | likely(X) | No | | -| load_extension(X) | No | | +| load_extension(X) | Yes | sqlite3 extensions not yet supported | | load_extension(X,Y) | No | | | lower(X) | Yes | | | ltrim(X) | Yes | | diff --git a/cli/app.rs b/cli/app.rs index a0a7b5d99..26a99c470 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -325,7 +325,10 @@ impl Limbo { #[cfg(not(target_family = "wasm"))] fn handle_load_extension(&mut self, path: &str) -> Result<(), String> { - self.conn.load_extension(path).map_err(|e| e.to_string()) + let ext_path = limbo_core::resolve_ext_path(path).map_err(|e| e.to_string())?; + self.conn + .load_extension(ext_path) + .map_err(|e| e.to_string()) } fn display_in_memory(&mut self) -> std::io::Result<()> { diff --git a/core/function.rs b/core/function.rs index 68b5ef005..75a90e798 100644 --- a/core/function.rs +++ b/core/function.rs @@ -136,6 +136,8 @@ pub enum ScalarFunc { ZeroBlob, LastInsertRowid, Replace, + #[cfg(not(target_family = "wasm"))] + LoadExtension, } impl Display for ScalarFunc { @@ -185,6 +187,8 @@ impl Display for ScalarFunc { Self::LastInsertRowid => "last_insert_rowid".to_string(), Self::Replace => "replace".to_string(), Self::DateTime => "datetime".to_string(), + #[cfg(not(target_family = "wasm"))] + Self::LoadExtension => "load_extension".to_string(), }; write!(f, "{}", str) } @@ -426,6 +430,8 @@ impl Func { "tan" => Ok(Self::Math(MathFunc::Tan)), "tanh" => Ok(Self::Math(MathFunc::Tanh)), "trunc" => Ok(Self::Math(MathFunc::Trunc)), + #[cfg(not(target_family = "wasm"))] + "load_extension" => Ok(Self::Scalar(ScalarFunc::LoadExtension)), _ => Err(()), } } diff --git a/core/lib.rs b/core/lib.rs index 8f8914b0f..a0bd6d98c 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -182,7 +182,7 @@ impl Database { } #[cfg(not(target_family = "wasm"))] - pub fn load_extension(&self, path: &str) -> Result<()> { + pub fn load_extension>(&self, path: P) -> Result<()> { let api = Box::new(self.build_limbo_extension()); let lib = unsafe { Library::new(path).map_err(|e| LimboError::ExtensionError(e.to_string()))? }; @@ -401,7 +401,7 @@ impl Connection { } #[cfg(not(target_family = "wasm"))] - pub fn load_extension(&self, path: &str) -> Result<()> { + pub fn load_extension>(&self, path: P) -> Result<()> { Database::load_extension(self.db.as_ref(), path) } @@ -515,6 +515,33 @@ impl std::fmt::Debug for SymbolTable { } } +fn is_shared_library(path: &std::path::Path) -> bool { + path.extension() + .map_or(false, |ext| ext == "so" || ext == "dylib" || ext == "dll") +} + +pub fn resolve_ext_path(extpath: &str) -> Result { + let path = std::path::Path::new(extpath); + if !path.exists() { + if is_shared_library(path) { + return Err(LimboError::ExtensionError(format!( + "Extension file not found: {}", + extpath + ))); + }; + let maybe = path.with_extension(std::env::consts::DLL_EXTENSION); + maybe + .exists() + .then_some(maybe) + .ok_or(LimboError::ExtensionError(format!( + "Extension file not found: {}", + extpath + ))) + } else { + Ok(path.to_path_buf()) + } +} + impl SymbolTable { pub fn new() -> Self { Self { diff --git a/core/translate/expr.rs b/core/translate/expr.rs index d13902433..b8de8e638 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -1092,6 +1092,19 @@ pub fn translate_expr( }); Ok(target_register) } + #[cfg(not(target_family = "wasm"))] + ScalarFunc::LoadExtension => { + let args = expect_arguments_exact!(args, 1, srf); + let reg = + translate_and_mark(program, referenced_tables, &args[0], resolver)?; + program.emit_insn(Insn::Function { + constant_mask: 0, + start_reg: reg, + dest: target_register, + func: func_ctx, + }); + Ok(target_register) + } ScalarFunc::Random => { if args.is_some() { crate::bail_parse_error!( diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 580834d39..9da225f44 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -41,7 +41,7 @@ use crate::{ json::json_arrow_extract, json::json_arrow_shift_extract, json::json_error_position, json::json_extract, json::json_type, }; -use crate::{Connection, Result, Rows, TransactionState, DATABASE_VERSION}; +use crate::{resolve_ext_path, Connection, Result, Rows, TransactionState, DATABASE_VERSION}; use datetime::{exec_date, exec_datetime_full, exec_julianday, exec_time, exec_unixepoch}; use insn::{ exec_add, exec_bit_and, exec_bit_not, exec_bit_or, exec_divide, exec_multiply, exec_remainder, @@ -1863,6 +1863,14 @@ impl Program { let replacement = &state.registers[*start_reg + 2]; state.registers[*dest] = exec_replace(source, pattern, replacement); } + #[cfg(not(target_family = "wasm"))] + ScalarFunc::LoadExtension => { + let extension = &state.registers[*start_reg]; + let ext = resolve_ext_path(&extension.to_string())?; + if let Some(conn) = self.connection.upgrade() { + conn.load_extension(ext)?; + } + } }, crate::function::Func::External(f) => { call_external_function! {f.func, *dest, state, arg_count, *start_reg }; From 3c118db20d905ab79ad83eae98591cca88cdf225 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 14 Jan 2025 19:15:14 +0200 Subject: [PATCH 78/97] simulator: Welcome banner --- simulator/main.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/simulator/main.rs b/simulator/main.rs index 3d9752b77..a710309e9 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -37,6 +37,8 @@ fn main() { let db_path = output_dir.join("simulator.db"); let plan_path = output_dir.join("simulator.plan"); + banner(); + // Print the seed, the locations of the database and the plan file log::info!("database path: {:?}", db_path); log::info!("simulator plan path: {:?}", plan_path); @@ -334,3 +336,30 @@ fn init_logger() { .format_target(false) .init(); } + +fn banner() { + println!("{}", BANNER); +} + +const BANNER: &str = r#" + ,_______________________________. + | ,___________________________. | + | | | | + | | >HELLO | | + | | | | + | | >A STRANGE GAME. | | + | | >THE ONLY WINNING MOVE IS | | + | | >NOT TO PLAY. | | + | |___________________________| | + | | + | | + `-------------------------------` + | | + |______________| + ,______________________. + / /====================\ \ + / /======================\ \ + /____________________________\ + \____________________________/ + +"#; From eed610d457ff7e4855f0f39cc78cd3ca31546db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Wed, 15 Jan 2025 09:08:45 +0900 Subject: [PATCH 79/97] Add JDBC4Statement.java --- .../tursodatabase/jdbc4/JDBC4Statement.java | 266 ++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java new file mode 100644 index 000000000..f3eb0f6ed --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java @@ -0,0 +1,266 @@ +package org.github.tursodatabase.jdbc4; + +import org.github.tursodatabase.annotations.SkipNullableCheck; + +import java.sql.*; + +/** + * Implementation of the {@link Statement} interface for JDBC 4. + */ +public class JDBC4Statement implements Statement { + @Override + @SkipNullableCheck + public ResultSet executeQuery(String sql) throws SQLException { + // TODO + return null; + } + + @Override + public int executeUpdate(String sql) throws SQLException { + // TODO + return 0; + } + + @Override + public void close() throws SQLException { + // TODO + } + + @Override + public int getMaxFieldSize() throws SQLException { + // TODO + return 0; + } + + @Override + public void setMaxFieldSize(int max) throws SQLException { + // TODO + } + + @Override + public int getMaxRows() throws SQLException { + // TODO + return 0; + } + + @Override + public void setMaxRows(int max) throws SQLException { + // TODO + } + + @Override + public void setEscapeProcessing(boolean enable) throws SQLException { + // TODO + } + + @Override + public int getQueryTimeout() throws SQLException { + // TODO + return 0; + } + + @Override + public void setQueryTimeout(int seconds) throws SQLException { + // TODO + } + + @Override + public void cancel() throws SQLException { + // TODO + } + + @Override + @SkipNullableCheck + public SQLWarning getWarnings() throws SQLException { + // TODO + return null; + } + + @Override + public void clearWarnings() throws SQLException { + // TODO + } + + @Override + public void setCursorName(String name) throws SQLException { + // TODO + } + + @Override + public boolean execute(String sql) throws SQLException { + // TODO + return false; + } + + @Override + @SkipNullableCheck + public ResultSet getResultSet() throws SQLException { + // TODO + return null; + } + + @Override + public int getUpdateCount() throws SQLException { + // TODO + return 0; + } + + @Override + public boolean getMoreResults() throws SQLException { + // TODO + return false; + } + + @Override + public void setFetchDirection(int direction) throws SQLException { + // TODO + } + + @Override + public int getFetchDirection() throws SQLException { + // TODO + return 0; + } + + @Override + public void setFetchSize(int rows) throws SQLException { + // TODO + } + + @Override + public int getFetchSize() throws SQLException { + // TODO + return 0; + } + + @Override + public int getResultSetConcurrency() throws SQLException { + // TODO + return 0; + } + + @Override + public int getResultSetType() throws SQLException { + // TODO + return 0; + } + + @Override + public void addBatch(String sql) throws SQLException { + // TODO + } + + @Override + public void clearBatch() throws SQLException { + // TODO + } + + @Override + public int[] executeBatch() throws SQLException { + // TODO + return new int[0]; + } + + @Override + @SkipNullableCheck + public Connection getConnection() throws SQLException { + // TODO + return null; + } + + @Override + public boolean getMoreResults(int current) throws SQLException { + // TODO + return false; + } + + @Override + @SkipNullableCheck + public ResultSet getGeneratedKeys() throws SQLException { + // TODO + return null; + } + + @Override + public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { + // TODO + return 0; + } + + @Override + public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { + // TODO + return 0; + } + + @Override + public int executeUpdate(String sql, String[] columnNames) throws SQLException { + // TODO + return 0; + } + + @Override + public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { + // TODO + return false; + } + + @Override + public boolean execute(String sql, int[] columnIndexes) throws SQLException { + // TODO + return false; + } + + @Override + public boolean execute(String sql, String[] columnNames) throws SQLException { + // TODO + return false; + } + + @Override + public int getResultSetHoldability() throws SQLException { + // TODO + return 0; + } + + @Override + public boolean isClosed() throws SQLException { + // TODO + return false; + } + + @Override + public void setPoolable(boolean poolable) throws SQLException { + // TODO + } + + @Override + public boolean isPoolable() throws SQLException { + // TODO + return false; + } + + @Override + public void closeOnCompletion() throws SQLException { + // TODO + } + + @Override + public boolean isCloseOnCompletion() throws SQLException { + // TODO + return false; + } + + @Override + @SkipNullableCheck + public T unwrap(Class iface) throws SQLException { + // TODO + return null; + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + // TODO + return false; + } +} From e5bf3c2644c5fe134f10098c57f83c89c5c56166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Wed, 15 Jan 2025 09:11:51 +0900 Subject: [PATCH 80/97] Add Codes.java --- .../org/github/tursodatabase/core/Codes.java | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 bindings/java/src/main/java/org/github/tursodatabase/core/Codes.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/Codes.java b/bindings/java/src/main/java/org/github/tursodatabase/core/Codes.java new file mode 100644 index 000000000..0f8a3c402 --- /dev/null +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/Codes.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2007 David Crawshaw + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +package org.github.tursodatabase.core; + +public class Codes { + /** Successful result */ + public static final int SQLITE_OK = 0; + + /** SQL error or missing database */ + public static final int SQLITE_ERROR = 1; + + /** An internal logic error in SQLite */ + public static final int SQLITE_INTERNAL = 2; + + /** Access permission denied */ + public static final int SQLITE_PERM = 3; + + /** Callback routine requested an abort */ + public static final int SQLITE_ABORT = 4; + + /** The database file is locked */ + public static final int SQLITE_BUSY = 5; + + /** A table in the database is locked */ + public static final int SQLITE_LOCKED = 6; + + /** A malloc() failed */ + public static final int SQLITE_NOMEM = 7; + + /** Attempt to write a readonly database */ + public static final int SQLITE_READONLY = 8; + + /** Operation terminated by sqlite_interrupt() */ + public static final int SQLITE_INTERRUPT = 9; + + /** Some kind of disk I/O error occurred */ + public static final int SQLITE_IOERR = 10; + + /** The database disk image is malformed */ + public static final int SQLITE_CORRUPT = 11; + + /** (Internal Only) Table or record not found */ + public static final int SQLITE_NOTFOUND = 12; + + /** Insertion failed because database is full */ + public static final int SQLITE_FULL = 13; + + /** Unable to open the database file */ + public static final int SQLITE_CANTOPEN = 14; + + /** Database lock protocol error */ + public static final int SQLITE_PROTOCOL = 15; + + /** (Internal Only) Database table is empty */ + public static final int SQLITE_EMPTY = 16; + + /** The database schema changed */ + public static final int SQLITE_SCHEMA = 17; + + /** Too much data for one row of a table */ + public static final int SQLITE_TOOBIG = 18; + + /** Abort due to constraint violation */ + public static final int SQLITE_CONSTRAINT = 19; + + /** Data type mismatch */ + public static final int SQLITE_MISMATCH = 20; + + /** Library used incorrectly */ + public static final int SQLITE_MISUSE = 21; + + /** Uses OS features not supported on host */ + public static final int SQLITE_NOLFS = 22; + + /** Authorization denied */ + public static final int SQLITE_AUTH = 23; + + /** sqlite_step() has another row ready */ + public static final int SQLITE_ROW = 100; + + /** sqlite_step() has finished executing */ + public static final int SQLITE_DONE = 101; + + // types returned by sqlite3_column_type() + + public static final int SQLITE_INTEGER = 1; + public static final int SQLITE_FLOAT = 2; + public static final int SQLITE_TEXT = 3; + public static final int SQLITE_BLOB = 4; + public static final int SQLITE_NULL = 5; +} From 7104a290e44231cdee710a92d064940101b0ee77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Wed, 15 Jan 2025 09:20:44 +0900 Subject: [PATCH 81/97] Basic support for close method --- .../tursodatabase/core/CoreStatement.java | 25 ++++++++++++++++-- .../tursodatabase/jdbc4/JDBC4Statement.java | 26 +++++++++++++++---- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/CoreStatement.java b/bindings/java/src/main/java/org/github/tursodatabase/core/CoreStatement.java index f71827d07..98dd89ab3 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/CoreStatement.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/CoreStatement.java @@ -1,5 +1,26 @@ package org.github.tursodatabase.core; -// TODO: add fields and methods -public class CoreStatement { +import org.github.tursodatabase.LimboConnection; + +import java.sql.SQLException; + +public abstract class CoreStatement { + + private final LimboConnection connection; + + protected CoreStatement(LimboConnection connection) { + this.connection = connection; + } + + protected void internalClose() throws SQLException { + // TODO + } + + protected void clearGeneratedKeys() throws SQLException { + // TODO + } + + protected void updateGeneratedKeys() throws SQLException { + // TODO + } } diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java index f3eb0f6ed..d8ea925b4 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java @@ -1,13 +1,23 @@ package org.github.tursodatabase.jdbc4; +import org.github.tursodatabase.LimboConnection; import org.github.tursodatabase.annotations.SkipNullableCheck; +import org.github.tursodatabase.core.CoreStatement; import java.sql.*; /** * Implementation of the {@link Statement} interface for JDBC 4. */ -public class JDBC4Statement implements Statement { +public class JDBC4Statement extends CoreStatement implements Statement { + + private boolean closed; + private boolean closeOnCompletion; + + public JDBC4Statement(LimboConnection connection) { + super(connection); + } + @Override @SkipNullableCheck public ResultSet executeQuery(String sql) throws SQLException { @@ -23,7 +33,9 @@ public class JDBC4Statement implements Statement { @Override public void close() throws SQLException { - // TODO + clearGeneratedKeys(); + internalClose(); + closed = true; } @Override @@ -242,13 +254,17 @@ public class JDBC4Statement implements Statement { @Override public void closeOnCompletion() throws SQLException { - // TODO + if (closed) throw new SQLException("statement is closed"); + closeOnCompletion = true; } + /** + * Indicates whether the statement should be closed automatically when all its dependent result sets are closed. + */ @Override public boolean isCloseOnCompletion() throws SQLException { - // TODO - return false; + if (closed) throw new SQLException("statement is closed"); + return closeOnCompletion; } @Override From d151824f668beee28ef5d224bbd29bf5761ca3b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Wed, 15 Jan 2025 09:46:42 +0900 Subject: [PATCH 82/97] Update JDBC4Statement to include resultSetType, resultSetConcurrency, resultSetHoldability --- .../github/tursodatabase/LimboConnection.java | 40 +++++++++++++ .../tursodatabase/jdbc4/JDBC4Connection.java | 35 +++++------ .../tursodatabase/jdbc4/JDBC4Statement.java | 26 ++++++--- .../jdbc4/JDBC4ConnectionTest.java | 58 +++++++++++++++++++ 4 files changed, 133 insertions(+), 26 deletions(-) create mode 100644 bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java b/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java index 98f0ad04b..de1a5228e 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/LimboConnection.java @@ -4,6 +4,7 @@ import org.github.tursodatabase.core.AbstractDB; import org.github.tursodatabase.core.LimboDB; import java.sql.Connection; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.Properties; @@ -61,4 +62,43 @@ public abstract class LimboConnection implements Connection { database.open(0); return database; } + + protected void checkOpen() throws SQLException { + if (isClosed()) throw new SQLException("database connection closed"); + } + + @Override + public void close() throws SQLException { + if (isClosed()) return; + database.close(); + } + + @Override + public boolean isClosed() throws SQLException { + return database.isClosed(); + } + + // TODO: check whether this is still valid for limbo + /** + * Checks whether the type, concurrency, and holdability settings for a {@link ResultSet} are + * supported by the SQLite interface. Supported settings are: + * + *
    + *
  • type: {@link ResultSet#TYPE_FORWARD_ONLY} + *
  • concurrency: {@link ResultSet#CONCUR_READ_ONLY}) + *
  • holdability: {@link ResultSet#CLOSE_CURSORS_AT_COMMIT} + *
+ * + * @param resultSetType the type setting. + * @param resultSetConcurrency the concurrency setting. + * @param resultSetHoldability the holdability setting. + */ + protected void checkCursor(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) + throw new SQLException("SQLite only supports TYPE_FORWARD_ONLY cursors"); + if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) + throw new SQLException("SQLite only supports CONCUR_READ_ONLY cursors"); + if (resultSetHoldability != ResultSet.CLOSE_CURSORS_AT_COMMIT) + throw new SQLException("SQLite only supports closing cursors at commit"); + } } diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index 9e67ae501..5883f7487 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -16,10 +16,25 @@ public class JDBC4Connection extends LimboConnection { } @Override - @SkipNullableCheck public Statement createStatement() throws SQLException { - // TODO - return null; + return createStatement( + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY, + ResultSet.CLOSE_CURSORS_AT_COMMIT + ); + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + return createStatement(resultSetType, resultSetConcurrency, ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + checkOpen(); + checkCursor(resultSetType, resultSetConcurrency, resultSetHoldability); + + return new JDBC4Statement(this); } @Override @@ -127,13 +142,6 @@ public class JDBC4Connection extends LimboConnection { // TODO } - @Override - @SkipNullableCheck - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - // TODO - return null; - } - @Override @SkipNullableCheck public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { @@ -193,13 +201,6 @@ public class JDBC4Connection extends LimboConnection { // TODO } - @Override - @SkipNullableCheck - public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - // TODO - return null; - } - @Override @SkipNullableCheck public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java index d8ea925b4..f1fb14221 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java @@ -14,8 +14,19 @@ public class JDBC4Statement extends CoreStatement implements Statement { private boolean closed; private boolean closeOnCompletion; + private final int resultSetType; + private final int resultSetConcurrency; + private final int resultSetHoldability; + public JDBC4Statement(LimboConnection connection) { + this(connection, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + public JDBC4Statement(LimboConnection connection, int resultSetType, int resultSetConcurrency, int resultSetHoldability) { super(connection); + this.resultSetType = resultSetType; + this.resultSetConcurrency = resultSetConcurrency; + this.resultSetHoldability = resultSetHoldability; } @Override @@ -146,15 +157,13 @@ public class JDBC4Statement extends CoreStatement implements Statement { } @Override - public int getResultSetConcurrency() throws SQLException { - // TODO - return 0; + public int getResultSetConcurrency() { + return resultSetConcurrency; } @Override - public int getResultSetType() throws SQLException { - // TODO - return 0; + public int getResultSetType() { + return resultSetType; } @Override @@ -230,9 +239,8 @@ public class JDBC4Statement extends CoreStatement implements Statement { } @Override - public int getResultSetHoldability() throws SQLException { - // TODO - return 0; + public int getResultSetHoldability() { + return resultSetHoldability; } @Override diff --git a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java new file mode 100644 index 000000000..bf2a20b88 --- /dev/null +++ b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java @@ -0,0 +1,58 @@ +package org.github.tursodatabase.jdbc4; + +import org.github.tursodatabase.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class JDBC4ConnectionTest { + + private JDBC4Connection connection; + + @BeforeEach + void setUp() throws Exception { + String fileUrl = TestUtils.createTempFile(); + String url = "jdbc:sqlite:" + fileUrl; + connection = new JDBC4Connection(url, fileUrl, new Properties()); + } + + @Test + void test_create_statement_valid() throws SQLException { + Statement stmt = connection.createStatement(); + assertNotNull(stmt); + assertEquals(ResultSet.TYPE_FORWARD_ONLY, stmt.getResultSetType()); + assertEquals(ResultSet.CONCUR_READ_ONLY, stmt.getResultSetConcurrency()); + assertEquals(ResultSet.CLOSE_CURSORS_AT_COMMIT, stmt.getResultSetHoldability()); + } + + @Test + void test_create_statement_with_type_and_concurrency_valid() throws SQLException { + Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + assertNotNull(stmt); + assertEquals(ResultSet.TYPE_FORWARD_ONLY, stmt.getResultSetType()); + assertEquals(ResultSet.CONCUR_READ_ONLY, stmt.getResultSetConcurrency()); + } + + @Test + void test_create_statement_with_all_params_valid() throws SQLException { + Statement stmt = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, ResultSet.CLOSE_CURSORS_AT_COMMIT); + assertNotNull(stmt); + assertEquals(ResultSet.TYPE_FORWARD_ONLY, stmt.getResultSetType()); + assertEquals(ResultSet.CONCUR_READ_ONLY, stmt.getResultSetConcurrency()); + assertEquals(ResultSet.CLOSE_CURSORS_AT_COMMIT, stmt.getResultSetHoldability()); + } + + @Test + void test_create_statement_invalid() { + assertThrows(SQLException.class, () -> { + connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, -1); + }); + } +} From c446e29a5087f4efaffa2c9d8d6c16188d3c2e77 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Wed, 15 Jan 2025 11:42:48 +0300 Subject: [PATCH 83/97] add missed updates from the merge --- simulator/generation/plan.rs | 38 ++++++++++++++++++-------------- simulator/generation/property.rs | 2 +- simulator/shrink/plan.rs | 5 +++++ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 96719f036..01c594648 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -12,7 +12,10 @@ use crate::{ use crate::generation::{frequency, Arbitrary, ArbitraryFrom}; -use super::{pick, property::Property}; +use super::{ + pick, + property::{remaining, Property}, +}; pub(crate) type ResultSet = Result>>; @@ -470,38 +473,39 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { rng: &mut R, (env, stats): (&SimulatorEnv, InteractionStats), ) -> Self { - let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) - - (stats.read_count as f64)) - .max(0.0); - let remaining_write = ((env.opts.max_interactions as f64 * env.opts.write_percent / 100.0) - - (stats.write_count as f64)) - .max(0.0); - let remaining_create = ((env.opts.max_interactions as f64 * env.opts.create_percent - / 100.0) - - (stats.create_count as f64)) - .max(0.0); - + let remaining_ = remaining(env, &stats); + println!( + "remaining: {} {} {}", + remaining_.read, remaining_.write, remaining_.create + ); frequency( vec![ ( - f64::min(remaining_read, remaining_write) + remaining_create, + f64::min(remaining_.read, remaining_.write) + remaining_.create, Box::new(|rng: &mut R| { Interactions::Property(Property::arbitrary_from(rng, (env, &stats))) }), ), ( - remaining_read, + remaining_.read, Box::new(|rng: &mut R| random_read(rng, env)), ), ( - remaining_write, + remaining_.write, Box::new(|rng: &mut R| random_write(rng, env)), ), ( - remaining_create, + remaining_.create, Box::new(|rng: &mut R| create_table(rng, env)), ), - (1.0, Box::new(|rng: &mut R| random_fault(rng, env))), + ( + remaining_ + .read + .min(remaining_.write) + .min(remaining_.create) + .max(1.0), + Box::new(|rng: &mut R| random_fault(rng, env)), + ), ], rng, ) diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index cc02e6233..dd92a7af8 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -174,7 +174,7 @@ pub(crate) struct Remaining { pub(crate) create: f64, } -fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaining { +pub(crate) fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaining { let remaining_read = ((env.opts.max_interactions as f64 * env.opts.read_percent / 100.0) - (stats.read_count as f64)) .max(0.0); diff --git a/simulator/shrink/plan.rs b/simulator/shrink/plan.rs index c97503f65..92867d82e 100644 --- a/simulator/shrink/plan.rs +++ b/simulator/shrink/plan.rs @@ -1,5 +1,6 @@ use crate::{ generation::plan::{InteractionPlan, Interactions}, + model::query::Query, runner::execution::Execution, }; @@ -35,6 +36,10 @@ impl InteractionPlan { } } } + + plan.plan + .retain(|p| !matches!(p, Interactions::Query(Query::Select(_)))); + let after = plan.plan.len(); log::info!( From ea6ad8d414a0afe9515de9745c6606856a6e59a6 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Wed, 15 Jan 2025 12:44:43 +0300 Subject: [PATCH 84/97] remove debug print --- simulator/generation/plan.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 01c594648..028da2a30 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -474,10 +474,6 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions { (env, stats): (&SimulatorEnv, InteractionStats), ) -> Self { let remaining_ = remaining(env, &stats); - println!( - "remaining: {} {} {}", - remaining_.read, remaining_.write, remaining_.create - ); frequency( vec![ ( From f8b3b06163715a858a3952cb68a060b04b57d9b8 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Wed, 15 Jan 2025 14:12:08 +0200 Subject: [PATCH 85/97] Expr: fix recursive binary operation logic --- core/translate/expr.rs | 82 +++++++++---------------------------- core/translate/group_by.rs | 1 - core/translate/main_loop.rs | 4 -- testing/where.test | 18 ++++++++ 4 files changed, 38 insertions(+), 67 deletions(-) diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 4a0677d8d..28e21e81f 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -16,7 +16,6 @@ pub struct ConditionMetadata { pub jump_if_condition_is_true: bool, pub jump_target_when_true: BranchOffset, pub jump_target_when_false: BranchOffset, - pub parent_op: Option, } fn emit_cond_jump(program: &mut ProgramBuilder, cond_meta: ConditionMetadata, reg: usize) { @@ -137,87 +136,46 @@ pub fn translate_condition_expr( match expr { ast::Expr::Between { .. } => todo!(), ast::Expr::Binary(lhs, ast::Operator::And, rhs) => { - // In a binary AND, never jump to the 'jump_target_when_true' label on the first condition, because - // the second condition must also be true. + // In a binary AND, never jump to the parent 'jump_target_when_true' label on the first condition, because + // the second condition must also be true. Instead we instruct the child expression to jump to a local + // true label. + let jump_target_when_true = program.allocate_label(); let _ = translate_condition_expr( program, referenced_tables, lhs, ConditionMetadata { - jump_if_condition_is_true: false, - // Mark that the parent op for sub-expressions is AND - parent_op: Some(ast::Operator::And), + jump_target_when_true, ..condition_metadata }, resolver, ); + program.resolve_label(jump_target_when_true, program.offset()); let _ = translate_condition_expr( program, referenced_tables, rhs, - ConditionMetadata { - parent_op: Some(ast::Operator::And), - ..condition_metadata - }, + condition_metadata, resolver, ); } ast::Expr::Binary(lhs, ast::Operator::Or, rhs) => { - if matches!(condition_metadata.parent_op, Some(ast::Operator::And)) { - // we are inside a bigger AND expression, so we do NOT jump to parent's 'true' if LHS or RHS is true. - // we only short-circuit the parent's false label if LHS and RHS are both false. - let local_true_label = program.allocate_label(); - let local_false_label = program.allocate_label(); + let jump_target_when_false = program.allocate_label(); - // evaluate LHS in normal OR fashion, short-circuit local if true - let lhs_metadata = ConditionMetadata { - jump_if_condition_is_true: true, - jump_target_when_true: local_true_label, - jump_target_when_false: local_false_label, - parent_op: Some(ast::Operator::Or), - }; - translate_condition_expr(program, referenced_tables, lhs, lhs_metadata, resolver)?; + let lhs_metadata = ConditionMetadata { + jump_if_condition_is_true: true, + jump_target_when_false, + ..condition_metadata + }; - // if lhs was false, we land here: - program.resolve_label(local_false_label, program.offset()); + translate_condition_expr(program, referenced_tables, lhs, lhs_metadata, resolver)?; - // evaluate rhs with normal OR: short-circuit if true, go to local_true - let rhs_metadata = ConditionMetadata { - jump_if_condition_is_true: true, - jump_target_when_true: local_true_label, - jump_target_when_false: condition_metadata.jump_target_when_false, - // if rhs is also false => parent's false - parent_op: Some(ast::Operator::Or), - }; - translate_condition_expr(program, referenced_tables, rhs, rhs_metadata, resolver)?; - - // if we get here, both lhs+rhs are false: explicit jump to parent's false - program.emit_insn(Insn::Goto { - target_pc: condition_metadata.jump_target_when_false, - }); - // local_true: we do not jump to parent's "true" label because the parent is AND, - // so we want to keep evaluating the rest - program.resolve_label(local_true_label, program.offset()); - } else { - let jump_target_when_false = program.allocate_label(); - - let lhs_metadata = ConditionMetadata { - jump_if_condition_is_true: true, - jump_target_when_false, - parent_op: Some(ast::Operator::Or), - ..condition_metadata - }; - - translate_condition_expr(program, referenced_tables, lhs, lhs_metadata, resolver)?; - - // if LHS was false, we land here: - program.resolve_label(jump_target_when_false, program.offset()); - let rhs_metadata = ConditionMetadata { - parent_op: Some(ast::Operator::Or), - ..condition_metadata - }; - translate_condition_expr(program, referenced_tables, rhs, rhs_metadata, resolver)?; - } + // if LHS was false, we land here: + program.resolve_label(jump_target_when_false, program.offset()); + let rhs_metadata = ConditionMetadata { + ..condition_metadata + }; + translate_condition_expr(program, referenced_tables, rhs, rhs_metadata, resolver)?; } ast::Expr::Binary(lhs, op, rhs) => { let lhs_reg = translate_and_mark(program, Some(referenced_tables), lhs, resolver)?; diff --git a/core/translate/group_by.rs b/core/translate/group_by.rs index 2b6e7afb3..5855caa06 100644 --- a/core/translate/group_by.rs +++ b/core/translate/group_by.rs @@ -386,7 +386,6 @@ pub fn emit_group_by<'a>( jump_if_condition_is_true: false, jump_target_when_false: group_by_end_without_emitting_row_label, jump_target_when_true: BranchOffset::Placeholder, // not used. FIXME: this is a bug. HAVING can have e.g. HAVING a OR b. - parent_op: None, }, &t_ctx.resolver, )?; diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs index 33f76a84e..a19fe66f7 100644 --- a/core/translate/main_loop.rs +++ b/core/translate/main_loop.rs @@ -230,7 +230,6 @@ pub fn open_loop( jump_if_condition_is_true: false, jump_target_when_true, jump_target_when_false: next, - parent_op: None, }; translate_condition_expr( program, @@ -279,7 +278,6 @@ pub fn open_loop( jump_if_condition_is_true: false, jump_target_when_true, jump_target_when_false, - parent_op: None, }; for predicate in predicates.iter() { translate_condition_expr( @@ -352,7 +350,6 @@ pub fn open_loop( jump_if_condition_is_true: false, jump_target_when_true, jump_target_when_false: next, - parent_op: None, }; translate_condition_expr( program, @@ -537,7 +534,6 @@ pub fn open_loop( jump_if_condition_is_true: false, jump_target_when_true, jump_target_when_false: next, - parent_op: None, }; translate_condition_expr( program, diff --git a/testing/where.test b/testing/where.test index c613f784b..b3154cd7c 100755 --- a/testing/where.test +++ b/testing/where.test @@ -365,3 +365,21 @@ do_execsql_test nested-parens-conditionals-and-double-or { 8171|Andrea|Lee|dgarrison@example.com|001-594-430-0646|452 Anthony Stravenue|Sandraville|CA|28572|12 9110|Anthony|Barrett|steven05@example.net|(562)928-9177x8454|86166 Foster Inlet Apt. 284|North Jeffreyburgh|CA|80147|97 9279|Annette|Lynn|joanne37@example.com|(272)700-7181|2676 Laura Points Apt. 683|Tristanville|NY|48646|91}} + +# Regression test for nested parens + OR + AND. This returned 0 rows before the fix. +# It should always return 1 row because it is true for id = 6. +do_execsql_test nested-parens-and-inside-or-regression-test { + SELECT count(1) FROM users + WHERE ( + ( + ( + (id != 5) + AND + (id = 5 OR TRUE) + ) + OR FALSE + ) + AND + (id = 6 OR FALSE) + ); +} {1} \ No newline at end of file From 84ef8a8951b847429dbf9ec559a8a733de6dc0a5 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Wed, 15 Jan 2025 14:24:40 +0200 Subject: [PATCH 86/97] translate_condition_expr(): unify how the AND and OR cases look --- core/translate/expr.rs | 45 ++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 28e21e81f..8886f704a 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -137,10 +137,10 @@ pub fn translate_condition_expr( ast::Expr::Between { .. } => todo!(), ast::Expr::Binary(lhs, ast::Operator::And, rhs) => { // In a binary AND, never jump to the parent 'jump_target_when_true' label on the first condition, because - // the second condition must also be true. Instead we instruct the child expression to jump to a local + // the second condition MUST also be true. Instead we instruct the child expression to jump to a local // true label. let jump_target_when_true = program.allocate_label(); - let _ = translate_condition_expr( + translate_condition_expr( program, referenced_tables, lhs, @@ -149,33 +149,40 @@ pub fn translate_condition_expr( ..condition_metadata }, resolver, - ); + )?; program.resolve_label(jump_target_when_true, program.offset()); - let _ = translate_condition_expr( + translate_condition_expr( program, referenced_tables, rhs, condition_metadata, resolver, - ); + )?; } ast::Expr::Binary(lhs, ast::Operator::Or, rhs) => { + // In a binary OR, never jump to the parent 'jump_target_when_false' label on the first condition, because + // the second condition CAN also be true. Instead we instruct the child expression to jump to a local + // false label. let jump_target_when_false = program.allocate_label(); - - let lhs_metadata = ConditionMetadata { - jump_if_condition_is_true: true, - jump_target_when_false, - ..condition_metadata - }; - - translate_condition_expr(program, referenced_tables, lhs, lhs_metadata, resolver)?; - - // if LHS was false, we land here: + translate_condition_expr( + program, + referenced_tables, + lhs, + ConditionMetadata { + jump_if_condition_is_true: true, + jump_target_when_false, + ..condition_metadata + }, + resolver, + )?; program.resolve_label(jump_target_when_false, program.offset()); - let rhs_metadata = ConditionMetadata { - ..condition_metadata - }; - translate_condition_expr(program, referenced_tables, rhs, rhs_metadata, resolver)?; + translate_condition_expr( + program, + referenced_tables, + rhs, + condition_metadata, + resolver, + )?; } ast::Expr::Binary(lhs, op, rhs) => { let lhs_reg = translate_and_mark(program, Some(referenced_tables), lhs, resolver)?; From 9cc9577c91edb1ab807f6c5fb6c2deb826abfcd1 Mon Sep 17 00:00:00 2001 From: psvri Date: Wed, 8 Jan 2025 22:17:36 +0530 Subject: [PATCH 87/97] Run all statements from sql argument in cli --- cli/app.rs | 41 ++++++++------ core/lib.rs | 126 ++++++++++++++++++++++++++++---------------- testing/insert.test | 8 ++- 3 files changed, 112 insertions(+), 63 deletions(-) diff --git a/cli/app.rs b/cli/app.rs index 26a99c470..03ee75ffa 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -3,7 +3,7 @@ use crate::{ opcodes_dictionary::OPCODE_DESCRIPTIONS, }; use cli_table::{Cell, Table}; -use limbo_core::{Database, LimboError, StepResult, Value}; +use limbo_core::{Database, LimboError, Rows, StepResult, Value}; use clap::{Parser, ValueEnum}; use std::{ @@ -304,8 +304,14 @@ impl Limbo { fn handle_first_input(&mut self, cmd: &str) { if cmd.trim().starts_with('.') { self.handle_dot_command(cmd); - } else if let Err(e) = self.query(cmd) { - eprintln!("{}", e); + } else { + let conn = self.conn.clone(); + let runner = conn.query_runner(cmd.as_bytes()); + for output in runner { + if let Err(e) = self.print_query_result(cmd, output) { + let _ = self.writeln(e.to_string()); + } + } } std::process::exit(0); } @@ -443,17 +449,16 @@ impl Limbo { self.buffer_input(line); let buff = self.input_buff.clone(); let echo = self.opts.echo; - buff.split(';') - .map(str::trim) - .filter(|s| !s.is_empty()) - .for_each(|stmt| { - if echo { - let _ = self.writeln(stmt); - } - if let Err(e) = self.query(stmt) { - let _ = self.writeln(e.to_string()); - } - }); + if echo { + let _ = self.writeln(&buff); + } + let conn = self.conn.clone(); + let runner = conn.query_runner(buff.as_bytes()); + for output in runner { + if let Err(e) = self.print_query_result(&buff, output) { + let _ = self.writeln(e.to_string()); + } + } self.reset_input(); } else { self.buffer_input(line); @@ -570,8 +575,12 @@ impl Limbo { } } - pub fn query(&mut self, sql: &str) -> anyhow::Result<()> { - match self.conn.query(sql) { + fn print_query_result( + &mut self, + sql: &str, + mut output: Result, LimboError>, + ) -> anyhow::Result<()> { + match output { Ok(Some(ref mut rows)) => match self.opts.output_mode { OutputMode::Raw => loop { if self.interrupt_count.load(Ordering::SeqCst) > 0 { diff --git a/core/lib.rs b/core/lib.rs index a0bd6d98c..906a1675f 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -298,57 +298,65 @@ impl Connection { pub fn query(self: &Rc, sql: impl Into) -> Result> { let sql = sql.into(); trace!("Querying: {}", sql); - let db = self.db.clone(); - let syms: &SymbolTable = &db.syms.borrow(); let mut parser = Parser::new(sql.as_bytes()); let cmd = parser.next()?; - if let Some(cmd) = cmd { - match cmd { - Cmd::Stmt(stmt) => { - let program = Rc::new(translate::translate( - &self.schema.borrow(), - stmt, - self.header.clone(), - self.pager.clone(), - Rc::downgrade(self), - syms, - )?); - let stmt = Statement::new(program, self.pager.clone()); - Ok(Some(Rows { stmt })) - } - Cmd::Explain(stmt) => { - let program = translate::translate( - &self.schema.borrow(), - stmt, - self.header.clone(), - self.pager.clone(), - Rc::downgrade(self), - syms, - )?; - program.explain(); - Ok(None) - } - Cmd::ExplainQueryPlan(stmt) => { - match stmt { - ast::Stmt::Select(select) => { - let mut plan = prepare_select_plan( - &self.schema.borrow(), - *select, - &self.db.syms.borrow(), - )?; - optimize_plan(&mut plan)?; - println!("{}", plan); - } - _ => todo!(), - } - Ok(None) - } - } - } else { - Ok(None) + match cmd { + Some(cmd) => self.run_cmd(cmd), + None => Ok(None), } } + pub(crate) fn run_cmd(self: &Rc, cmd: Cmd) -> Result> { + let db = self.db.clone(); + let syms: &SymbolTable = &db.syms.borrow(); + + match cmd { + Cmd::Stmt(stmt) => { + let program = Rc::new(translate::translate( + &self.schema.borrow(), + stmt, + self.header.clone(), + self.pager.clone(), + Rc::downgrade(self), + syms, + )?); + let stmt = Statement::new(program, self.pager.clone()); + Ok(Some(Rows { stmt })) + } + Cmd::Explain(stmt) => { + let program = translate::translate( + &self.schema.borrow(), + stmt, + self.header.clone(), + self.pager.clone(), + Rc::downgrade(self), + syms, + )?; + program.explain(); + Ok(None) + } + Cmd::ExplainQueryPlan(stmt) => { + match stmt { + ast::Stmt::Select(select) => { + let mut plan = prepare_select_plan( + &self.schema.borrow(), + *select, + &self.db.syms.borrow(), + )?; + optimize_plan(&mut plan)?; + println!("{}", plan); + } + _ => todo!(), + } + Ok(None) + } + } + } + + pub fn query_runner<'a>(self: &'a Rc, sql: &'a [u8]) -> QueryRunner<'a> { + QueryRunner::new(self, sql) + } + pub fn execute(self: &Rc, sql: impl Into) -> Result<()> { let sql = sql.into(); let db = self.db.clone(); @@ -560,3 +568,29 @@ impl SymbolTable { self.functions.get(name).cloned() } } + +pub struct QueryRunner<'a> { + parser: Parser<'a>, + conn: &'a Rc, +} + +impl<'a> QueryRunner<'a> { + pub(crate) fn new(conn: &'a Rc, statements: &'a [u8]) -> Self { + Self { + parser: Parser::new(statements), + conn, + } + } +} + +impl Iterator for QueryRunner<'_> { + type Item = Result>; + + fn next(&mut self) -> Option { + match self.parser.next() { + Ok(Some(cmd)) => Some(self.conn.run_cmd(cmd)), + Ok(None) => None, + Err(err) => Some(Result::Err(LimboError::from(err))), + } + } +} diff --git a/testing/insert.test b/testing/insert.test index e7a03ad81..731ccfd6c 100755 --- a/testing/insert.test +++ b/testing/insert.test @@ -1,3 +1,9 @@ #!/usr/bin/env tclsh set testdir [file dirname $argv0] -source $testdir/tester.tcl \ No newline at end of file +source $testdir/tester.tcl + +do_execsql_test_on_specific_db {:memory:} basic-insert { + create table temp (t1 integer, primary key (t1)); + insert into temp values (1); + select * from temp; +} {1} \ No newline at end of file From 5b4d82abbf7a479b82812c76b499e25e9b9ce76d Mon Sep 17 00:00:00 2001 From: psvri Date: Wed, 15 Jan 2025 00:37:54 +0530 Subject: [PATCH 88/97] Implement ShiftLeft --- COMPAT.md | 2 +- core/translate/expr.rs | 7 ++++++ core/vdbe/explain.rs | 9 +++++++ core/vdbe/insn.rs | 54 ++++++++++++++++++++++++++++++++++++++++ core/vdbe/mod.rs | 7 +++++- testing/math.test | 56 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 2 deletions(-) diff --git a/COMPAT.md b/COMPAT.md index 5cd309bf2..0faceccd9 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -445,7 +445,7 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html). | SeekRowid | Yes | | Sequence | No | | SetCookie | No | -| ShiftLeft | No | +| ShiftLeft | Yes | | ShiftRight | No | | SoftNull | Yes | | Sort | No | diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 4a0677d8d..a011b68ca 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -613,6 +613,13 @@ pub fn translate_expr( dest: target_register, }); } + ast::Operator::LeftShift => { + program.emit_insn(Insn::ShiftLeft { + lhs: e1_reg, + rhs: e2_reg, + dest: target_register, + }); + } #[cfg(feature = "json")] op @ (ast::Operator::ArrowRight | ast::Operator::ArrowRightShift) => { let json_func = match op { diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index 40fda1a28..a8a61d657 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -1044,6 +1044,15 @@ pub fn insn_to_str( 0, "".to_string(), ), + Insn::ShiftLeft { lhs, rhs, dest } => ( + "ShiftLeft", + *rhs as i32, + *lhs as i32, + *dest as i32, + OwnedValue::build_text(Rc::new("".to_string())), + 0, + format!("r[{}]=r[{}] << r[{}]", dest, lhs, rhs), + ), }; format!( "{:<4} {:<17} {:<4} {:<4} {:<4} {:<13} {:<2} {}", diff --git a/core/vdbe/insn.rs b/core/vdbe/insn.rs index 6a42ba21a..95c3214a0 100644 --- a/core/vdbe/insn.rs +++ b/core/vdbe/insn.rs @@ -487,6 +487,12 @@ pub enum Insn { db: usize, where_clause: String, }, + // Place the result of lhs >> rhs in dest register. + ShiftLeft { + lhs: usize, + rhs: usize, + dest: usize, + }, } fn cast_text_to_numerical(value: &str) -> OwnedValue { @@ -720,3 +726,51 @@ pub fn exec_bit_not(mut reg: &OwnedValue) -> OwnedValue { _ => todo!(), } } + +pub fn exec_shift_left(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue { + if let OwnedValue::Agg(agg) = lhs { + lhs = agg.final_value(); + } + if let OwnedValue::Agg(agg) = rhs { + rhs = agg.final_value(); + } + match (lhs, rhs) { + (OwnedValue::Null, _) | (_, OwnedValue::Null) => OwnedValue::Null, + (OwnedValue::Integer(lh), OwnedValue::Integer(rh)) => { + OwnedValue::Integer(compute_shl(*lh, *rh)) + } + (OwnedValue::Float(lh), OwnedValue::Integer(rh)) => { + OwnedValue::Integer(compute_shl(*lh as i64, *rh)) + } + (OwnedValue::Integer(lh), OwnedValue::Float(rh)) => { + OwnedValue::Integer(compute_shl(*lh, *rh as i64)) + } + (OwnedValue::Float(lh), OwnedValue::Float(rh)) => { + OwnedValue::Integer(compute_shl(*lh as i64, *rh as i64)) + } + (OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_shift_left( + &cast_text_to_numerical(&lhs.value), + &cast_text_to_numerical(&rhs.value), + ), + (OwnedValue::Text(text), other) => { + exec_shift_left(&cast_text_to_numerical(&text.value), other) + } + (other, OwnedValue::Text(text)) => { + exec_shift_left(other, &cast_text_to_numerical(&text.value)) + } + _ => todo!(), + } +} + +fn compute_shl(lhs: i64, rhs: i64) -> i64 { + if rhs == 0 { + lhs + } else if rhs >= 64 || rhs <= -64 { + 0 + } else if rhs < 0 { + // if negative do right shift + lhs >> (-rhs) + } else { + lhs << rhs + } +} diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 9da225f44..f96617b0b 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -45,7 +45,7 @@ use crate::{resolve_ext_path, Connection, Result, Rows, TransactionState, DATABA use datetime::{exec_date, exec_datetime_full, exec_julianday, exec_time, exec_unixepoch}; use insn::{ exec_add, exec_bit_and, exec_bit_not, exec_bit_or, exec_divide, exec_multiply, exec_remainder, - exec_subtract, + exec_shift_left, exec_subtract, }; use likeop::{construct_like_escape_arg, exec_glob, exec_like_with_escape}; use rand::distributions::{Distribution, Uniform}; @@ -2158,6 +2158,11 @@ impl Program { parse_schema_rows(Some(rows), &mut schema, conn.pager.io.clone())?; state.pc += 1; } + Insn::ShiftLeft { lhs, rhs, dest } => { + state.registers[*dest] = + exec_shift_left(&state.registers[*lhs], &state.registers[*rhs]); + state.pc += 1; + } } } } diff --git a/testing/math.test b/testing/math.test index 6188c5a71..94d41f468 100644 --- a/testing/math.test +++ b/testing/math.test @@ -459,6 +459,62 @@ do_execsql_test bitwise-and-int-agg-int-agg { } {66} +foreach {testname lhs rhs ans} { + int-int 1 2 4 + int-neg_int 8 -2 2 + int-float 1 4.0 16 + int-text 1 'a' 1 + int-text_float 1 '3.0' 8 + int-text_int 1 '1' 2 + int-null 1 NULL {} + int-int-overflow 1 64 0 + int-int-underflow 1 -64 0 + int-float-overflow 1 64.0 0 + int-float-underflow 1 -64.0 0 +} { + do_execsql_test shift-left-$testname "SELECT $lhs << $rhs" $::ans +} + +foreach {testname lhs rhs ans} { + float-int 1.0 2 4 + float-neg_int 8.0 -2 2 + float-float 1.0 4.0 16 + float-text 1.0 'a' 1 + float-text_float 1.0 '3.0' 8 + float-text_int 1.0 '1' 2 + float-null 1.0 NULL {} + float-int-overflow 1.0 64 0 + float-int-underflow 1.0 -64 0 + float-float-overflow 1.0 64.0 0 + float-float-underflow 1.0 -64.0 0 +} { + do_execsql_test shift-left-$testname "SELECT $lhs << $rhs" $::ans +} + +foreach {testname lhs rhs ans} { + text-int 'a' 2 0 + text-float 'a' 4.0 0 + text-text 'a' 'a' 0 + text_int-text_int '1' '1' 2 + text_int-text_float '1' '3.0' 8 + text_int-text '1' 'a' 1 + text_float-text_int '1.0' '1' 2 + text_float-text_float '1.0' '3.0' 8 + text_float-text '1.0' 'a' 1 + text-null '1' NULL {} +} { + do_execsql_test shift-left-$testname "SELECT $lhs << $rhs" $::ans +} + +foreach {testname lhs rhs ans} { + null-int NULL 2 {} + null-float NULL 4.0 {} + null-text NULL 'a' {} + null-null NULL NULL {} +} { + do_execsql_test shift-left-$testname "SELECT $lhs << $rhs" $::ans +} + do_execsql_test bitwise-not-null { SELECT ~NULL } {} From d3f28c51f439612adaa2d015c2cdadf597540239 Mon Sep 17 00:00:00 2001 From: psvri Date: Wed, 15 Jan 2025 21:21:51 +0530 Subject: [PATCH 89/97] Implement ShiftRight --- COMPAT.md | 2 +- core/translate/expr.rs | 7 ++++++ core/vdbe/explain.rs | 9 +++++++ core/vdbe/insn.rs | 54 ++++++++++++++++++++++++++++++++++++++++ core/vdbe/mod.rs | 7 +++++- testing/math.test | 56 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 2 deletions(-) diff --git a/COMPAT.md b/COMPAT.md index 0faceccd9..6dabca9f0 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -446,7 +446,7 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html). | Sequence | No | | SetCookie | No | | ShiftLeft | Yes | -| ShiftRight | No | +| ShiftRight | Yes | | SoftNull | Yes | | Sort | No | | SorterCompare | No | diff --git a/core/translate/expr.rs b/core/translate/expr.rs index a011b68ca..6e35c3da8 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -613,6 +613,13 @@ pub fn translate_expr( dest: target_register, }); } + ast::Operator::RightShift => { + program.emit_insn(Insn::ShiftRight { + lhs: e1_reg, + rhs: e2_reg, + dest: target_register, + }); + } ast::Operator::LeftShift => { program.emit_insn(Insn::ShiftLeft { lhs: e1_reg, diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index a8a61d657..7c3de8364 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -1044,6 +1044,15 @@ pub fn insn_to_str( 0, "".to_string(), ), + Insn::ShiftRight { lhs, rhs, dest } => ( + "ShiftRight", + *rhs as i32, + *lhs as i32, + *dest as i32, + OwnedValue::build_text(Rc::new("".to_string())), + 0, + format!("r[{}]=r[{}] >> r[{}]", dest, lhs, rhs), + ), Insn::ShiftLeft { lhs, rhs, dest } => ( "ShiftLeft", *rhs as i32, diff --git a/core/vdbe/insn.rs b/core/vdbe/insn.rs index 95c3214a0..225a9edaf 100644 --- a/core/vdbe/insn.rs +++ b/core/vdbe/insn.rs @@ -488,6 +488,12 @@ pub enum Insn { where_clause: String, }, // Place the result of lhs >> rhs in dest register. + ShiftRight { + lhs: usize, + rhs: usize, + dest: usize, + }, + // Place the result of lhs << rhs in dest register. ShiftLeft { lhs: usize, rhs: usize, @@ -774,3 +780,51 @@ fn compute_shl(lhs: i64, rhs: i64) -> i64 { lhs << rhs } } + +pub fn exec_shift_right(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue { + if let OwnedValue::Agg(agg) = lhs { + lhs = agg.final_value(); + } + if let OwnedValue::Agg(agg) = rhs { + rhs = agg.final_value(); + } + match (lhs, rhs) { + (OwnedValue::Null, _) | (_, OwnedValue::Null) => OwnedValue::Null, + (OwnedValue::Integer(lh), OwnedValue::Integer(rh)) => { + OwnedValue::Integer(compute_shr(*lh, *rh)) + } + (OwnedValue::Float(lh), OwnedValue::Integer(rh)) => { + OwnedValue::Integer(compute_shr(*lh as i64, *rh)) + } + (OwnedValue::Integer(lh), OwnedValue::Float(rh)) => { + OwnedValue::Integer(compute_shr(*lh, *rh as i64)) + } + (OwnedValue::Float(lh), OwnedValue::Float(rh)) => { + OwnedValue::Integer(compute_shr(*lh as i64, *rh as i64)) + } + (OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_shift_right( + &cast_text_to_numerical(&lhs.value), + &cast_text_to_numerical(&rhs.value), + ), + (OwnedValue::Text(text), other) => { + exec_shift_right(&cast_text_to_numerical(&text.value), other) + } + (other, OwnedValue::Text(text)) => { + exec_shift_right(other, &cast_text_to_numerical(&text.value)) + } + _ => todo!(), + } +} + +fn compute_shr(lhs: i64, rhs: i64) -> i64 { + if rhs == 0 { + lhs + } else if rhs >= 64 || rhs <= -64 { + 0 + } else if rhs < 0 { + // if negative do left shift + lhs << (-rhs) + } else { + lhs >> rhs + } +} diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index f96617b0b..3791b994e 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -45,7 +45,7 @@ use crate::{resolve_ext_path, Connection, Result, Rows, TransactionState, DATABA use datetime::{exec_date, exec_datetime_full, exec_julianday, exec_time, exec_unixepoch}; use insn::{ exec_add, exec_bit_and, exec_bit_not, exec_bit_or, exec_divide, exec_multiply, exec_remainder, - exec_shift_left, exec_subtract, + exec_shift_left, exec_shift_right, exec_subtract, }; use likeop::{construct_like_escape_arg, exec_glob, exec_like_with_escape}; use rand::distributions::{Distribution, Uniform}; @@ -2158,6 +2158,11 @@ impl Program { parse_schema_rows(Some(rows), &mut schema, conn.pager.io.clone())?; state.pc += 1; } + Insn::ShiftRight { lhs, rhs, dest } => { + state.registers[*dest] = + exec_shift_right(&state.registers[*lhs], &state.registers[*rhs]); + state.pc += 1; + } Insn::ShiftLeft { lhs, rhs, dest } => { state.registers[*dest] = exec_shift_left(&state.registers[*lhs], &state.registers[*rhs]); diff --git a/testing/math.test b/testing/math.test index 94d41f468..f404d9901 100644 --- a/testing/math.test +++ b/testing/math.test @@ -515,6 +515,62 @@ foreach {testname lhs rhs ans} { do_execsql_test shift-left-$testname "SELECT $lhs << $rhs" $::ans } +foreach {testname lhs rhs ans} { + int-int 8 2 2 + int-neg_int 8 -2 32 + int-float 8 1.0 4 + int-text 8 'a' 8 + int-text_float 8 '3.0' 1 + int-text_int 8 '1' 4 + int-null 8 NULL {} + int-int-overflow 8 64 0 + int-int-underflow 8 -64 0 + int-float-overflow 8 64.0 0 + int-float-underflow 8 -64.0 0 +} { + do_execsql_test shift-right-$testname "SELECT $lhs >> $rhs" $::ans +} + +foreach {testname lhs rhs ans} { + float-int 8.0 2 2 + float-neg_int 8.0 -2 32 + float-float 8.0 1.0 4 + float-text 8.0 'a' 8 + float-text_float 8.0 '3.0' 1 + float-text_int 8.0 '1' 4 + float-null 8.0 NULL {} + float-int-overflow 8.0 64 0 + float-int-underflow 8.0 -64 0 + float-float-overflow 8.0 64.0 0 + float-float-underflow 8.0 -64.0 0 +} { + do_execsql_test shift-right-$testname "SELECT $lhs >> $rhs" $::ans +} + +foreach {testname lhs rhs ans} { + text-int 'a' 2 0 + text-float 'a' 4.0 0 + text-text 'a' 'a' 0 + text_int-text_int '8' '1' 4 + text_int-text_float '8' '3.0' 1 + text_int-text '8' 'a' 8 + text_float-text_int '8.0' '1' 4 + text_float-text_float '8.0' '3.0' 1 + text_float-text '8.0' 'a' 8 + text-null '8' NULL {} +} { + do_execsql_test shift-right-$testname "SELECT $lhs >> $rhs" $::ans +} + +foreach {testname lhs rhs ans} { + null-int NULL 2 {} + null-float NULL 4.0 {} + null-text NULL 'a' {} + null-null NULL NULL {} +} { + do_execsql_test shift-right-$testname "SELECT $lhs >> $rhs" $::ans +} + do_execsql_test bitwise-not-null { SELECT ~NULL } {} From 4fafaba60732919aecd264e932a689530dcedd20 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 15 Jan 2025 18:31:48 +0200 Subject: [PATCH 90/97] simulator: Reduce generated sequence size defaults ...otherwise the simulator runs forever... --- simulator/runner/cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index 2ad69d4ea..b4a6d94f1 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -18,14 +18,14 @@ pub struct SimulatorCLI { short = 'n', long, help = "change the maximum size of the randomly generated sequence of interactions", - default_value_t = 20000 + default_value_t = 5000 )] pub maximum_size: usize, #[clap( short = 'k', long, help = "change the minimum size of the randomly generated sequence of interactions", - default_value_t = 10000 + default_value_t = 1000 )] pub minimum_size: usize, #[clap( From 0f4cc8f0cc97c92ec4d29abb60e1d57b350607de Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Wed, 15 Jan 2025 19:22:05 +0200 Subject: [PATCH 91/97] Simulator: expose inner error in assertion failure, if any --- simulator/generation/plan.rs | 28 +++++++++++++++++++--------- simulator/generation/property.rs | 16 ++++++++-------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 028da2a30..9acef25ad 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -167,7 +167,7 @@ impl Display for Interaction { } } -type AssertionFunc = dyn Fn(&Vec, &SimulatorEnv) -> bool; +type AssertionFunc = dyn Fn(&Vec, &SimulatorEnv) -> Result; enum AssertionAST { Pick(), @@ -375,12 +375,17 @@ impl Interaction { unreachable!("unexpected: this function should only be called on assertions") } Self::Assertion(assertion) => { - if !assertion.func.as_ref()(stack, env) { - return Err(limbo_core::LimboError::InternalError( + let result = assertion.func.as_ref()(stack, env); + match result { + Ok(true) => Ok(()), + Ok(false) => Err(limbo_core::LimboError::InternalError( assertion.message.clone(), - )); + )), + Err(err) => Err(limbo_core::LimboError::InternalError(format!( + "{}. Inner error: {}", + assertion.message, err + ))), } - Ok(()) } Self::Assumption(_) => { unreachable!("unexpected: this function should only be called on assertions") @@ -404,12 +409,17 @@ impl Interaction { unreachable!("unexpected: this function should only be called on assumptions") } Self::Assumption(assumption) => { - if !assumption.func.as_ref()(stack, env) { - return Err(limbo_core::LimboError::InternalError( + let result = assumption.func.as_ref()(stack, env); + match result { + Ok(true) => Ok(()), + Ok(false) => Err(limbo_core::LimboError::InternalError( assumption.message.clone(), - )); + )), + Err(err) => Err(limbo_core::LimboError::InternalError(format!( + "{}. Inner error: {}", + assumption.message, err + ))), } - Ok(()) } Self::Fault(_) => { unreachable!("unexpected: this function should only be called on assumptions") diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index dd92a7af8..cae2a4145 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -1,3 +1,5 @@ +use limbo_core::LimboError; + use crate::{ model::{ query::{Create, Delete, Insert, Predicate, Query, Select}, @@ -94,7 +96,7 @@ impl Property { func: Box::new({ let table_name = insert.table.clone(); move |_: &Vec, env: &SimulatorEnv| { - env.tables.iter().any(|t| t.name == table_name) + Ok(env.tables.iter().any(|t| t.name == table_name)) } }), }); @@ -109,8 +111,8 @@ impl Property { func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { let rows = stack.last().unwrap(); match rows { - Ok(rows) => rows.iter().any(|r| r == &row), - Err(_) => false, + Ok(rows) => Ok(rows.iter().any(|r| r == &row)), + Err(err) => Err(LimboError::InternalError(err.to_string())), } }), }); @@ -131,7 +133,7 @@ impl Property { message: "Double-Create-Failure should not be called on an existing table" .to_string(), func: Box::new(move |_: &Vec, env: &SimulatorEnv| { - !env.tables.iter().any(|t| t.name == table_name) + Ok(!env.tables.iter().any(|t| t.name == table_name)) }), }); @@ -147,10 +149,8 @@ impl Property { func: Box::new(move |stack: &Vec, _: &SimulatorEnv| { let last = stack.last().unwrap(); match last { - Ok(_) => false, - Err(e) => e - .to_string() - .contains(&format!("Table {table_name} already exists")), + Ok(_) => Ok(false), + Err(e) => Ok(e.to_string().contains(&format!("Table {table_name} already exists"))), } }), }); From 845de125dbbb29b30fdde849568791bed403a74b Mon Sep 17 00:00:00 2001 From: psvri Date: Wed, 8 Jan 2025 22:22:04 +0530 Subject: [PATCH 92/97] Align MustBeInt logic with sqlite --- core/vdbe/mod.rs | 51 +++++++++++++++++++++++++++++++++++---------- testing/insert.test | 11 +++++++++- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 3791b994e..01ddd55df 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -2038,11 +2038,25 @@ impl Program { state.pc += 1; } Insn::MustBeInt { reg } => { - match state.registers[*reg] { + match &state.registers[*reg] { OwnedValue::Integer(_) => {} + OwnedValue::Float(f) => match cast_real_to_integer(*f) { + Ok(i) => state.registers[*reg] = OwnedValue::Integer(i), + Err(_) => crate::bail_parse_error!( + "MustBeInt: the value in register cannot be cast to integer" + ), + }, + OwnedValue::Text(text) => match checked_cast_text_to_numeric(&text.value) { + Ok(OwnedValue::Integer(i)) => { + state.registers[*reg] = OwnedValue::Integer(i) + } + _ => crate::bail_parse_error!( + "MustBeInt: the value in register cannot be cast to integer" + ), + }, _ => { crate::bail_parse_error!( - "MustBeInt: the value in the register is not an integer" + "MustBeInt: the value in register cannot be cast to integer" ); } }; @@ -3079,23 +3093,38 @@ fn cast_text_to_real(text: &str) -> OwnedValue { /// IEEE 754 64-bit float and thus provides a 1-bit of margin for the text-to-float conversion operation.) /// Any text input that describes a value outside the range of a 64-bit signed integer yields a REAL result. /// Casting a REAL or INTEGER value to NUMERIC is a no-op, even if a real value could be losslessly converted to an integer. -fn cast_text_to_numeric(text: &str) -> OwnedValue { +fn checked_cast_text_to_numeric(text: &str) -> std::result::Result { if !text.contains('.') && !text.contains('e') && !text.contains('E') { // Looks like an integer if let Ok(i) = text.parse::() { - return OwnedValue::Integer(i); + return Ok(OwnedValue::Integer(i)); } } // Try as float if let Ok(f) = text.parse::() { - // Check if can be losslessly converted to 51-bit integer - let i = f as i64; - if f == i as f64 && i.abs() < (1i64 << 51) { - return OwnedValue::Integer(i); - } - return OwnedValue::Float(f); + return match cast_real_to_integer(f) { + Ok(i) => Ok(OwnedValue::Integer(i)), + Err(_) => Ok(OwnedValue::Float(f)), + }; } - OwnedValue::Integer(0) + Err(()) +} + +// try casting to numeric if not possible return integer 0 +fn cast_text_to_numeric(text: &str) -> OwnedValue { + match checked_cast_text_to_numeric(text) { + Ok(value) => value, + Err(_) => OwnedValue::Integer(0), + } +} + +// Check if float can be losslessly converted to 51-bit integer +fn cast_real_to_integer(float: f64) -> std::result::Result { + let i = float as i64; + if float == i as f64 && i.abs() < (1i64 << 51) { + return Ok(i); + } + Err(()) } fn execute_sqlite_version(version_integer: i64) -> String { diff --git a/testing/insert.test b/testing/insert.test index 731ccfd6c..5a37fd692 100755 --- a/testing/insert.test +++ b/testing/insert.test @@ -6,4 +6,13 @@ do_execsql_test_on_specific_db {:memory:} basic-insert { create table temp (t1 integer, primary key (t1)); insert into temp values (1); select * from temp; -} {1} \ No newline at end of file +} {1} + +do_execsql_test_on_specific_db {:memory:} must-be-int-insert { + create table temp (t1 integer, primary key (t1)); + insert into temp values (1),(2.0),('3'),('4.0'); + select * from temp; +} {1 +2 +3 +4} \ No newline at end of file From 08c8c655e91e2e87fdf40f9dc7200abf4217c313 Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Tue, 14 Jan 2025 06:35:02 -0300 Subject: [PATCH 93/97] feat: initial implementation of `Statement::bind` --- core/error.rs | 4 ++ core/lib.rs | 10 +++- core/parameters.rs | 111 ++++++++++++++++++++++++++++++++++++++ core/translate/expr.rs | 9 +++- core/translate/mod.rs | 3 +- core/translate/planner.rs | 2 +- core/types.rs | 12 +++++ core/vdbe/builder.rs | 28 ++++++++++ core/vdbe/explain.rs | 9 ++++ core/vdbe/insn.rs | 10 ++++ core/vdbe/mod.rs | 22 ++++++++ test/src/lib.rs | 27 +++++++++- 12 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 core/parameters.rs diff --git a/core/error.rs b/core/error.rs index e3e176b79..ca495eb99 100644 --- a/core/error.rs +++ b/core/error.rs @@ -1,3 +1,5 @@ +use std::num::NonZero; + use thiserror::Error; #[derive(Debug, Error, miette::Diagnostic)] @@ -41,6 +43,8 @@ pub enum LimboError { Constraint(String), #[error("Extension error: {0}")] ExtensionError(String), + #[error("Unbound parameter at index {0}")] + Unbound(NonZero), } #[macro_export] diff --git a/core/lib.rs b/core/lib.rs index 906a1675f..1020c495e 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -386,6 +386,7 @@ impl Connection { Rc::downgrade(self), syms, )?; + let mut state = vdbe::ProgramState::new(program.max_registers); program.step(&mut state, self.pager.clone())?; } @@ -473,7 +474,14 @@ impl Statement { Ok(Rows::new(stmt)) } - pub fn reset(&self) {} + pub fn reset(&self) { + self.state.reset(); + } + + pub fn bind(&mut self, value: Value) { + self.state.bind(value.into()); + } + } pub enum StepResult<'a> { diff --git a/core/parameters.rs b/core/parameters.rs new file mode 100644 index 000000000..9bfdf7f63 --- /dev/null +++ b/core/parameters.rs @@ -0,0 +1,111 @@ +use std::num::NonZero; + +#[derive(Clone, Debug)] +pub enum Parameter { + Anonymous(NonZero), + Indexed(NonZero), + Named(String, NonZero), +} + +impl PartialEq for Parameter { + fn eq(&self, other: &Self) -> bool { + self.index() == other.index() + } +} + +impl Parameter { + pub fn index(&self) -> NonZero { + match self { + Parameter::Anonymous(index) => *index, + Parameter::Indexed(index) => *index, + Parameter::Named(_, index) => *index, + } + } +} + +#[derive(Debug)] +pub struct Parameters { + index: NonZero, + pub list: Vec, +} + +impl Parameters { + pub fn new() -> Self { + Self { + index: 1.try_into().unwrap(), + list: vec![], + } + } + + pub fn count(&self) -> usize { + let mut params = self.list.clone(); + params.dedup(); + params.len() + } + + pub fn name(&self, index: NonZero) -> Option { + self.list.iter().find_map(|p| match p { + Parameter::Anonymous(i) if *i == index => Some("?".to_string()), + Parameter::Indexed(i) if *i == index => Some(format!("?{i}")), + Parameter::Named(name, i) if *i == index => Some(name.to_owned()), + _ => None, + }) + } + + pub fn index(&self, name: impl AsRef) -> Option> { + self.list + .iter() + .find_map(|p| match p { + Parameter::Named(n, index) if n == name.as_ref() => Some(index), + _ => None, + }) + .copied() + } + + pub fn next_index(&mut self) -> NonZero { + let index = self.index; + self.index = self.index.checked_add(1).unwrap(); + index + } + + pub fn push(&mut self, name: impl AsRef) -> NonZero { + match name.as_ref() { + "" => { + let index = self.next_index(); + self.list.push(Parameter::Anonymous(index)); + log::trace!("anonymous parameter at {index}"); + index + } + name if name.starts_with(&['$', ':', '@', '#']) => { + match self + .list + .iter() + .find(|p| matches!(p, Parameter::Named(n, _) if name == n)) + { + Some(t) => { + let index = t.index(); + self.list.push(t.clone()); + log::trace!("named parameter at {index} as {name}"); + index + } + None => { + let index = self.next_index(); + self.list.push(Parameter::Named(name.to_owned(), index)); + log::trace!("named parameter at {index} as {name}"); + index + } + } + } + index => { + // SAFETY: Garanteed from parser that the index is bigger that 0. + let index: NonZero = index.parse().unwrap(); + if index > self.index { + self.index = index.checked_add(1).unwrap(); + } + self.list.push(Parameter::Indexed(index)); + log::trace!("indexed parameter at {index}"); + index + } + } + } +} diff --git a/core/translate/expr.rs b/core/translate/expr.rs index d5cd7983e..bde068a20 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -1710,7 +1710,14 @@ pub fn translate_expr( } _ => todo!(), }, - ast::Expr::Variable(_) => todo!(), + ast::Expr::Variable(name) => { + let index = program.get_parameter_index(name); + program.emit_insn(Insn::Variable { + index, + dest: target_register, + }); + Ok(target_register) + } } } diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 9a990eb36..37470cd56 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -32,8 +32,7 @@ use crate::vdbe::{builder::ProgramBuilder, insn::Insn, Program}; use crate::{bail_parse_error, Connection, LimboError, Result, SymbolTable}; use insert::translate_insert; use select::translate_select; -use sqlite3_parser::ast::fmt::ToTokens; -use sqlite3_parser::ast::{self, PragmaName}; +use sqlite3_parser::ast::{self, fmt::ToTokens, PragmaName}; use std::cell::RefCell; use std::fmt::Display; use std::rc::{Rc, Weak}; diff --git a/core/translate/planner.rs b/core/translate/planner.rs index d15a2fab6..8e2c10146 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -269,7 +269,7 @@ pub fn bind_column_references( bind_column_references(expr, referenced_tables)?; Ok(()) } - ast::Expr::Variable(_) => todo!(), + ast::Expr::Variable(_) => Ok(()), } } diff --git a/core/types.rs b/core/types.rs index 81d9d5328..c2bb0be9a 100644 --- a/core/types.rs +++ b/core/types.rs @@ -336,6 +336,18 @@ impl std::ops::DivAssign for OwnedValue { } } +impl From> for OwnedValue { + fn from(value: Value<'_>) -> Self { + match value { + Value::Null => OwnedValue::Null, + Value::Integer(i) => OwnedValue::Integer(i), + Value::Float(f) => OwnedValue::Float(f), + Value::Text(s) => OwnedValue::Text(LimboText::new(Rc::new(s.to_owned()))), + Value::Blob(b) => OwnedValue::Blob(Rc::new(b.to_owned())), + } + } +} + pub fn to_value(value: &OwnedValue) -> Value<'_> { match value { OwnedValue::Null => Value::Null, diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs index 7acc4be6f..c796ee5f9 100644 --- a/core/vdbe/builder.rs +++ b/core/vdbe/builder.rs @@ -1,6 +1,7 @@ use std::{ cell::RefCell, collections::HashMap, + num::NonZero, rc::{Rc, Weak}, }; @@ -29,6 +30,8 @@ pub struct ProgramBuilder { seekrowid_emitted_bitmask: u64, // map of instruction index to manual comment (used in EXPLAIN) comments: HashMap, + named_parameters: HashMap>, + next_free_parameter_index: NonZero, } #[derive(Debug, Clone)] @@ -51,6 +54,7 @@ impl ProgramBuilder { next_free_register: 1, next_free_label: 0, next_free_cursor_id: 0, + next_free_parameter_index: 1.into(), insns: Vec::new(), next_insn_label: None, cursor_ref: Vec::new(), @@ -58,6 +62,7 @@ impl ProgramBuilder { label_to_resolved_offset: HashMap::new(), seekrowid_emitted_bitmask: 0, comments: HashMap::new(), + named_parameters: HashMap::new(), } } @@ -341,4 +346,27 @@ impl ProgramBuilder { auto_commit: true, } } + + fn next_parameter(&mut self) -> NonZero { + let index = self.next_free_parameter_index; + self.next_free_parameter_index.checked_add(1).unwrap(); + index + } + + pub fn get_parameter_index(&mut self, name: impl AsRef) -> NonZero { + let name = name.as_ref(); + + if name == "" { + return self.next_parameter(); + } + + match self.named_parameters.get(name) { + Some(index) => *index, + None => { + let index = self.next_parameter(); + self.named_parameters.insert(name.to_owned(), index); + index + } + } + } } diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index 7c3de8364..22b154809 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -1062,6 +1062,15 @@ pub fn insn_to_str( 0, format!("r[{}]=r[{}] << r[{}]", dest, lhs, rhs), ), + Insn::Variable { index, dest } => ( + "Variable", + usize::from(*index) as i32, + *dest as i32, + 0, + OwnedValue::build_text(Rc::new("".to_string())), + 0, + format!("r[{}]=parameter({})", *dest, *index), + ), }; format!( "{:<4} {:<17} {:<4} {:<4} {:<4} {:<13} {:<2} {}", diff --git a/core/vdbe/insn.rs b/core/vdbe/insn.rs index 225a9edaf..3066a399c 100644 --- a/core/vdbe/insn.rs +++ b/core/vdbe/insn.rs @@ -1,3 +1,5 @@ +use std::num::NonZero; + use super::{AggFunc, BranchOffset, CursorID, FuncCtx, PageIdx}; use crate::types::{OwnedRecord, OwnedValue}; use limbo_macros::Description; @@ -487,18 +489,26 @@ pub enum Insn { db: usize, where_clause: String, }, + // Place the result of lhs >> rhs in dest register. ShiftRight { lhs: usize, rhs: usize, dest: usize, }, + // Place the result of lhs << rhs in dest register. ShiftLeft { lhs: usize, rhs: usize, dest: usize, }, + + /// Get parameter variable. + Variable { + index: NonZero, + dest: usize, + }, } fn cast_text_to_numerical(value: &str) -> OwnedValue { diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 01ddd55df..80732cf9d 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -55,6 +55,7 @@ use sorter::Sorter; use std::borrow::BorrowMut; use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; +use std::num::NonZero; use std::rc::{Rc, Weak}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -200,6 +201,7 @@ pub struct ProgramState { ended_coroutine: HashMap, // flag to indicate that a coroutine has ended (key is the yield register) regex_cache: RegexCache, interrupted: bool, + parameters: Vec, } impl ProgramState { @@ -222,6 +224,7 @@ impl ProgramState { ended_coroutine: HashMap::new(), regex_cache: RegexCache::new(), interrupted: false, + parameters: Vec::new(), } } @@ -240,6 +243,18 @@ impl ProgramState { pub fn is_interrupted(&self) -> bool { self.interrupted } + + pub fn bind(&mut self, value: OwnedValue) { + self.parameters.push(value); + } + + pub fn get_parameter(&self, index: NonZero) -> Option<&OwnedValue> { + self.parameters.get(usize::from(index) - 1) + } + + pub fn reset(&self) { + self.parameters.clear(); + } } macro_rules! must_be_btree_cursor { @@ -2182,6 +2197,13 @@ impl Program { exec_shift_left(&state.registers[*lhs], &state.registers[*rhs]); state.pc += 1; } + Insn::Variable { index, dest } => { + state.registers[*dest] = state + .get_parameter(*index) + .ok_or(LimboError::Unbound(*index))? + .clone(); + state.pc += 1; + } } } } diff --git a/test/src/lib.rs b/test/src/lib.rs index 9aa9116d5..1fa681ff9 100644 --- a/test/src/lib.rs +++ b/test/src/lib.rs @@ -40,7 +40,7 @@ impl TempDatabase { #[cfg(test)] mod tests { use super::*; - use limbo_core::{CheckpointStatus, Connection, StepResult, Value}; + use limbo_core::{CheckpointStatus, Connection, Rows, StepResult, Value}; use log::debug; #[ignore] @@ -572,4 +572,29 @@ mod tests { do_flush(&conn, &tmp_db)?; Ok(()) } + + #[test] + fn test_statement_bind() -> anyhow::Result<()> { + let _ = env_logger::try_init(); + let tmp_db = TempDatabase::new("CREATE TABLE test (x INTEGER PRIMARY KEY);"); + let conn = tmp_db.connect_limbo(); + let mut stmt = conn.prepare("select ?")?; + stmt.bind(Value::Text(&"hello".to_string())); + loop { + match stmt.step()? { + StepResult::Row(row) => { + if let Value::Text(s) = row.values[0] { + assert_eq!(s, "hello") + } + } + StepResult::IO => { + tmp_db.io.run_once()?; + } + StepResult::Interrupt => break, + StepResult::Done => break, + StepResult::Busy => panic!("Database is busy"), + }; + } + Ok(()) + } } From 6e0ce3dd01498ff3beac32851684e1f2c2e37c7d Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Tue, 14 Jan 2025 06:39:11 -0300 Subject: [PATCH 94/97] chore: cargo fmt --- core/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/lib.rs b/core/lib.rs index 1020c495e..a032dea1e 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -481,7 +481,6 @@ impl Statement { pub fn bind(&mut self, value: Value) { self.state.bind(value.into()); } - } pub enum StepResult<'a> { From d3582a382f6334413b42fb0b8db6f643e5a85b1a Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Tue, 14 Jan 2025 06:49:17 -0300 Subject: [PATCH 95/97] fix: small bugs --- core/lib.rs | 2 +- core/vdbe/builder.rs | 2 +- core/vdbe/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index a032dea1e..a851e7281 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -474,7 +474,7 @@ impl Statement { Ok(Rows::new(stmt)) } - pub fn reset(&self) { + pub fn reset(&mut self) { self.state.reset(); } diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs index c796ee5f9..796782640 100644 --- a/core/vdbe/builder.rs +++ b/core/vdbe/builder.rs @@ -54,7 +54,7 @@ impl ProgramBuilder { next_free_register: 1, next_free_label: 0, next_free_cursor_id: 0, - next_free_parameter_index: 1.into(), + next_free_parameter_index: 1.try_into().unwrap(), insns: Vec::new(), next_insn_label: None, cursor_ref: Vec::new(), diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 80732cf9d..50db5538a 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -252,7 +252,7 @@ impl ProgramState { self.parameters.get(usize::from(index) - 1) } - pub fn reset(&self) { + pub fn reset(&mut self) { self.parameters.clear(); } } From 5de2694834a55c5fa0f4f898e9680296cfa1c513 Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Tue, 14 Jan 2025 22:15:24 -0300 Subject: [PATCH 96/97] feat: more parameter support add `Statement::{parameter_index, parameter_name, parameter_count, bind_at}`. some refactoring is still needed, this is quite a rough iteration --- core/lib.rs | 21 ++++++-- core/translate/expr.rs | 4 +- core/translate/mod.rs | 119 ++++++++++++++++++++++++++++++++++++++++- core/vdbe/builder.rs | 33 ++++-------- core/vdbe/mod.rs | 43 +++++++++++++-- test/src/lib.rs | 30 +++++++++-- 6 files changed, 212 insertions(+), 38 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index a851e7281..ab35d7711 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -28,6 +28,7 @@ use sqlite3_parser::ast; use sqlite3_parser::{ast::Cmd, lexer::sql::Parser}; use std::cell::Cell; use std::collections::HashMap; +use std::num::NonZero; use std::sync::{Arc, OnceLock, RwLock}; use std::{cell::RefCell, rc::Rc}; use storage::btree::btree_init_page; @@ -474,12 +475,24 @@ impl Statement { Ok(Rows::new(stmt)) } - pub fn reset(&mut self) { - self.state.reset(); + pub fn parameter_count(&mut self) -> usize { + self.program.parameter_count() } - pub fn bind(&mut self, value: Value) { - self.state.bind(value.into()); + pub fn parameter_name(&self, index: NonZero) -> Option { + self.program.parameter_name(index) + } + + pub fn parameter_index(&self, name: impl AsRef) -> Option> { + self.program.parameter_index(name) + } + + pub fn bind_at(&mut self, index: NonZero, value: Value) { + self.state.bind_at(index, value.into()); + } + + pub fn reset(&mut self) { + self.state.reset(); } } diff --git a/core/translate/expr.rs b/core/translate/expr.rs index bde068a20..3bafa1b1c 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -1710,8 +1710,8 @@ pub fn translate_expr( } _ => todo!(), }, - ast::Expr::Variable(name) => { - let index = program.get_parameter_index(name); + ast::Expr::Variable(_) => { + let index = program.pop_index(); program.emit_insn(Insn::Variable { index, dest: target_register, diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 37470cd56..1f2dc2737 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -32,12 +32,121 @@ use crate::vdbe::{builder::ProgramBuilder, insn::Insn, Program}; use crate::{bail_parse_error, Connection, LimboError, Result, SymbolTable}; use insert::translate_insert; use select::translate_select; +use sqlite3_parser::ast::fmt::TokenStream; use sqlite3_parser::ast::{self, fmt::ToTokens, PragmaName}; +use sqlite3_parser::dialect::TokenType; use std::cell::RefCell; use std::fmt::Display; +use std::num::NonZero; use std::rc::{Rc, Weak}; use std::str::FromStr; +#[derive(Clone, Debug)] +pub enum Parameter { + Anonymous(NonZero), + Indexed(NonZero), + Named(String, NonZero), +} + +impl PartialEq for Parameter { + fn eq(&self, other: &Self) -> bool { + self.index() == other.index() + } +} + +impl Parameter { + pub fn index(&self) -> NonZero { + match self { + Parameter::Anonymous(index) => *index, + Parameter::Indexed(index) => *index, + Parameter::Named(_, index) => *index, + } + } +} + +/// `?` or `$` Prepared statement arg placeholder(s) +#[derive(Debug)] +pub struct Parameters { + index: NonZero, + pub list: Vec, +} + +impl Parameters { + pub fn new() -> Self { + Self { + index: 1.try_into().unwrap(), + list: vec![], + } + } + + pub fn push(&mut self, value: Parameter) { + self.list.push(value); + } + + pub fn next_index(&mut self) -> NonZero { + let index = self.index; + self.index = self.index.checked_add(1).unwrap(); + index + } + + pub fn get(&mut self, index: usize) -> Option<&Parameter> { + self.list.get(index) + } +} + +// https://sqlite.org/lang_expr.html#parameters +impl TokenStream for Parameters { + type Error = std::convert::Infallible; + + fn append( + &mut self, + ty: TokenType, + value: Option<&str>, + ) -> std::result::Result<(), Self::Error> { + if ty == TokenType::TK_VARIABLE { + if let Some(variable) = value { + match variable.split_at(1) { + ("?", "") => { + let index = self.next_index(); + self.push(Parameter::Anonymous(index.try_into().unwrap())); + log::trace!("anonymous parameter at {index}"); + } + ("?", index) => { + let index: NonZero = index.parse().unwrap(); + if index > self.index { + self.index = index.checked_add(1).unwrap(); + } + self.push(Parameter::Indexed(index.try_into().unwrap())); + log::trace!("indexed parameter at {index}"); + } + (_, _) => { + match self.list.iter().find(|p| { + let Parameter::Named(name, _) = p else { + return false; + }; + name == variable + }) { + Some(t) => { + log::trace!("named parameter at {} as {}", t.index(), variable); + self.push(t.clone()); + } + None => { + let index = self.next_index(); + self.push(Parameter::Named( + variable.to_owned(), + index.try_into().unwrap(), + )); + log::trace!("named parameter at {index} as {variable}"); + } + } + } + } + } + } + Ok(()) + } +} + /// Translate SQL statement into bytecode program. pub fn translate( schema: &Schema, @@ -47,7 +156,14 @@ pub fn translate( connection: Weak, syms: &SymbolTable, ) -> Result { - let mut program = ProgramBuilder::new(); + let mut parameters = Parameters::new(); + stmt.to_tokens(&mut parameters).unwrap(); + + // dbg!(¶meters); + // dbg!(¶meters.list.clone().dedup()); + + let mut program = ProgramBuilder::new(parameters); + match stmt { ast::Stmt::AlterTable(_, _) => bail_parse_error!("ALTER TABLE not supported yet"), ast::Stmt::Analyze(_) => bail_parse_error!("ANALYZE not supported yet"), @@ -118,6 +234,7 @@ pub fn translate( )?; } } + Ok(program.build(database_header, connection)) } diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs index 796782640..711b506a0 100644 --- a/core/vdbe/builder.rs +++ b/core/vdbe/builder.rs @@ -32,6 +32,8 @@ pub struct ProgramBuilder { comments: HashMap, named_parameters: HashMap>, next_free_parameter_index: NonZero, + parameters: crate::translate::Parameters, + parameter_index: usize, } #[derive(Debug, Clone)] @@ -49,7 +51,7 @@ impl CursorType { } impl ProgramBuilder { - pub fn new() -> Self { + pub fn new(parameters: crate::translate::Parameters) -> Self { Self { next_free_register: 1, next_free_label: 0, @@ -63,6 +65,8 @@ impl ProgramBuilder { seekrowid_emitted_bitmask: 0, comments: HashMap::new(), named_parameters: HashMap::new(), + parameters, + parameter_index: 0, } } @@ -336,6 +340,7 @@ impl ProgramBuilder { self.constant_insns.is_empty(), "constant_insns is not empty when build() is called, did you forget to call emit_constant_insns()?" ); + self.parameters.list.dedup(); Program { max_registers: self.next_free_register, insns: self.insns, @@ -344,29 +349,13 @@ impl ProgramBuilder { comments: self.comments, connection, auto_commit: true, + parameters: self.parameters.list, } } - fn next_parameter(&mut self) -> NonZero { - let index = self.next_free_parameter_index; - self.next_free_parameter_index.checked_add(1).unwrap(); - index - } - - pub fn get_parameter_index(&mut self, name: impl AsRef) -> NonZero { - let name = name.as_ref(); - - if name == "" { - return self.next_parameter(); - } - - match self.named_parameters.get(name) { - Some(index) => *index, - None => { - let index = self.next_parameter(); - self.named_parameters.insert(name.to_owned(), index); - index - } - } + pub fn pop_index(&mut self) -> NonZero { + let parameter = self.parameters.get(self.parameter_index).unwrap(); + self.parameter_index += 1; + return parameter.index(); } } diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 50db5538a..5188a2329 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -201,7 +201,7 @@ pub struct ProgramState { ended_coroutine: HashMap, // flag to indicate that a coroutine has ended (key is the yield register) regex_cache: RegexCache, interrupted: bool, - parameters: Vec, + parameters: HashMap, OwnedValue>, } impl ProgramState { @@ -224,7 +224,7 @@ impl ProgramState { ended_coroutine: HashMap::new(), regex_cache: RegexCache::new(), interrupted: false, - parameters: Vec::new(), + parameters: HashMap::new(), } } @@ -244,12 +244,12 @@ impl ProgramState { self.interrupted } - pub fn bind(&mut self, value: OwnedValue) { - self.parameters.push(value); + pub fn bind_at(&mut self, index: NonZero, value: OwnedValue) { + self.parameters.insert(index, value); } pub fn get_parameter(&self, index: NonZero) -> Option<&OwnedValue> { - self.parameters.get(usize::from(index) - 1) + self.parameters.get(&index) } pub fn reset(&mut self) { @@ -277,6 +277,7 @@ pub struct Program { pub cursor_ref: Vec<(Option, CursorType)>, pub database_header: Rc>, pub comments: HashMap, + pub parameters: Vec, pub connection: Weak, pub auto_commit: bool, } @@ -300,6 +301,38 @@ impl Program { } } + pub fn parameter_count(&self) -> usize { + self.parameters.len() + } + + pub fn parameter_name(&self, index: NonZero) -> Option { + use crate::translate::Parameter; + self.parameters.iter().find_map(|p| match p { + Parameter::Anonymous(i) if *i == index => Some("?".to_string()), + Parameter::Indexed(i) if *i == index => Some(format!("?{i}")), + Parameter::Named(name, i) if *i == index => Some(name.to_owned()), + _ => None, + }) + } + + pub fn parameter_index(&self, name: impl AsRef) -> Option> { + use crate::translate::Parameter; + self.parameters + .iter() + .find_map(|p| { + let Parameter::Named(parameter_name, index) = p else { + return None; + }; + + if name.as_ref() == parameter_name { + return Some(index); + } + + None + }) + .copied() + } + pub fn step<'a>( &self, state: &'a mut ProgramState, diff --git a/test/src/lib.rs b/test/src/lib.rs index 1fa681ff9..68b4b10e5 100644 --- a/test/src/lib.rs +++ b/test/src/lib.rs @@ -40,7 +40,7 @@ impl TempDatabase { #[cfg(test)] mod tests { use super::*; - use limbo_core::{CheckpointStatus, Connection, Rows, StepResult, Value}; + use limbo_core::{CheckpointStatus, Connection, StepResult, Value}; use log::debug; #[ignore] @@ -576,16 +576,38 @@ mod tests { #[test] fn test_statement_bind() -> anyhow::Result<()> { let _ = env_logger::try_init(); - let tmp_db = TempDatabase::new("CREATE TABLE test (x INTEGER PRIMARY KEY);"); + let tmp_db = TempDatabase::new("create table test (i integer);"); let conn = tmp_db.connect_limbo(); - let mut stmt = conn.prepare("select ?")?; - stmt.bind(Value::Text(&"hello".to_string())); + + let mut stmt = conn.prepare("select ?, ?1, :named, ?4")?; + + stmt.bind_at(1.try_into().unwrap(), Value::Text(&"hello".to_string())); + + let i = stmt.parameter_index(":named").unwrap(); + stmt.bind_at(i, Value::Integer(42)); + + stmt.bind_at(4.try_into().unwrap(), Value::Float(0.5)); + + assert_eq!(stmt.parameter_count(), 3); + loop { match stmt.step()? { StepResult::Row(row) => { if let Value::Text(s) = row.values[0] { assert_eq!(s, "hello") } + + if let Value::Text(s) = row.values[1] { + assert_eq!(s, "hello") + } + + if let Value::Integer(s) = row.values[2] { + assert_eq!(s, 42) + } + + if let Value::Float(s) = row.values[3] { + assert_eq!(s, 0.5) + } } StepResult::IO => { tmp_db.io.run_once()?; From 9b8722f38e7c88d507d957f5908e8b035c706af4 Mon Sep 17 00:00:00 2001 From: "Levy A." Date: Wed, 15 Jan 2025 16:33:33 -0300 Subject: [PATCH 97/97] refactor: more well rounded implementation `?0` parameters are now handled by the parser. --- core/lib.rs | 18 +-- core/translate/expr.rs | 4 +- core/translate/mod.rs | 117 +------------------ core/vdbe/builder.rs | 23 +--- core/vdbe/mod.rs | 34 +----- test/src/lib.rs | 95 +++++++++++++-- vendored/sqlite3-parser/src/lexer/sql/mod.rs | 7 +- 7 files changed, 109 insertions(+), 189 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index ab35d7711..04caf1c71 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -4,6 +4,7 @@ mod function; mod io; #[cfg(feature = "json")] mod json; +mod parameters; mod pseudo; mod result; mod schema; @@ -44,7 +45,7 @@ use util::parse_schema_rows; pub use error::LimboError; use translate::select::prepare_select_plan; -pub type Result = std::result::Result; +pub type Result = std::result::Result; use crate::translate::optimizer::optimize_plan; pub use io::OpenFlags; @@ -475,16 +476,8 @@ impl Statement { Ok(Rows::new(stmt)) } - pub fn parameter_count(&mut self) -> usize { - self.program.parameter_count() - } - - pub fn parameter_name(&self, index: NonZero) -> Option { - self.program.parameter_name(index) - } - - pub fn parameter_index(&self, name: impl AsRef) -> Option> { - self.program.parameter_index(name) + pub fn parameters(&self) -> ¶meters::Parameters { + &self.program.parameters } pub fn bind_at(&mut self, index: NonZero, value: Value) { @@ -492,7 +485,8 @@ impl Statement { } pub fn reset(&mut self) { - self.state.reset(); + let state = vdbe::ProgramState::new(self.program.max_registers); + self.state = state } } diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 3bafa1b1c..cd9b6b28c 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -1710,8 +1710,8 @@ pub fn translate_expr( } _ => todo!(), }, - ast::Expr::Variable(_) => { - let index = program.pop_index(); + ast::Expr::Variable(name) => { + let index = program.parameters.push(name); program.emit_insn(Insn::Variable { index, dest: target_register, diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 1f2dc2737..20a514e5d 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -32,121 +32,12 @@ use crate::vdbe::{builder::ProgramBuilder, insn::Insn, Program}; use crate::{bail_parse_error, Connection, LimboError, Result, SymbolTable}; use insert::translate_insert; use select::translate_select; -use sqlite3_parser::ast::fmt::TokenStream; use sqlite3_parser::ast::{self, fmt::ToTokens, PragmaName}; -use sqlite3_parser::dialect::TokenType; use std::cell::RefCell; use std::fmt::Display; -use std::num::NonZero; use std::rc::{Rc, Weak}; use std::str::FromStr; -#[derive(Clone, Debug)] -pub enum Parameter { - Anonymous(NonZero), - Indexed(NonZero), - Named(String, NonZero), -} - -impl PartialEq for Parameter { - fn eq(&self, other: &Self) -> bool { - self.index() == other.index() - } -} - -impl Parameter { - pub fn index(&self) -> NonZero { - match self { - Parameter::Anonymous(index) => *index, - Parameter::Indexed(index) => *index, - Parameter::Named(_, index) => *index, - } - } -} - -/// `?` or `$` Prepared statement arg placeholder(s) -#[derive(Debug)] -pub struct Parameters { - index: NonZero, - pub list: Vec, -} - -impl Parameters { - pub fn new() -> Self { - Self { - index: 1.try_into().unwrap(), - list: vec![], - } - } - - pub fn push(&mut self, value: Parameter) { - self.list.push(value); - } - - pub fn next_index(&mut self) -> NonZero { - let index = self.index; - self.index = self.index.checked_add(1).unwrap(); - index - } - - pub fn get(&mut self, index: usize) -> Option<&Parameter> { - self.list.get(index) - } -} - -// https://sqlite.org/lang_expr.html#parameters -impl TokenStream for Parameters { - type Error = std::convert::Infallible; - - fn append( - &mut self, - ty: TokenType, - value: Option<&str>, - ) -> std::result::Result<(), Self::Error> { - if ty == TokenType::TK_VARIABLE { - if let Some(variable) = value { - match variable.split_at(1) { - ("?", "") => { - let index = self.next_index(); - self.push(Parameter::Anonymous(index.try_into().unwrap())); - log::trace!("anonymous parameter at {index}"); - } - ("?", index) => { - let index: NonZero = index.parse().unwrap(); - if index > self.index { - self.index = index.checked_add(1).unwrap(); - } - self.push(Parameter::Indexed(index.try_into().unwrap())); - log::trace!("indexed parameter at {index}"); - } - (_, _) => { - match self.list.iter().find(|p| { - let Parameter::Named(name, _) = p else { - return false; - }; - name == variable - }) { - Some(t) => { - log::trace!("named parameter at {} as {}", t.index(), variable); - self.push(t.clone()); - } - None => { - let index = self.next_index(); - self.push(Parameter::Named( - variable.to_owned(), - index.try_into().unwrap(), - )); - log::trace!("named parameter at {index} as {variable}"); - } - } - } - } - } - } - Ok(()) - } -} - /// Translate SQL statement into bytecode program. pub fn translate( schema: &Schema, @@ -156,13 +47,7 @@ pub fn translate( connection: Weak, syms: &SymbolTable, ) -> Result { - let mut parameters = Parameters::new(); - stmt.to_tokens(&mut parameters).unwrap(); - - // dbg!(¶meters); - // dbg!(¶meters.list.clone().dedup()); - - let mut program = ProgramBuilder::new(parameters); + let mut program = ProgramBuilder::new(); match stmt { ast::Stmt::AlterTable(_, _) => bail_parse_error!("ALTER TABLE not supported yet"), diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs index 711b506a0..ab643c105 100644 --- a/core/vdbe/builder.rs +++ b/core/vdbe/builder.rs @@ -1,18 +1,17 @@ use std::{ cell::RefCell, collections::HashMap, - num::NonZero, rc::{Rc, Weak}, }; use crate::{ + parameters::Parameters, schema::{BTreeTable, Index, PseudoTable}, storage::sqlite3_ondisk::DatabaseHeader, Connection, }; use super::{BranchOffset, CursorID, Insn, InsnReference, Program}; - #[allow(dead_code)] pub struct ProgramBuilder { next_free_register: usize, @@ -30,10 +29,7 @@ pub struct ProgramBuilder { seekrowid_emitted_bitmask: u64, // map of instruction index to manual comment (used in EXPLAIN) comments: HashMap, - named_parameters: HashMap>, - next_free_parameter_index: NonZero, - parameters: crate::translate::Parameters, - parameter_index: usize, + pub parameters: Parameters, } #[derive(Debug, Clone)] @@ -51,12 +47,11 @@ impl CursorType { } impl ProgramBuilder { - pub fn new(parameters: crate::translate::Parameters) -> Self { + pub fn new() -> Self { Self { next_free_register: 1, next_free_label: 0, next_free_cursor_id: 0, - next_free_parameter_index: 1.try_into().unwrap(), insns: Vec::new(), next_insn_label: None, cursor_ref: Vec::new(), @@ -64,9 +59,7 @@ impl ProgramBuilder { label_to_resolved_offset: HashMap::new(), seekrowid_emitted_bitmask: 0, comments: HashMap::new(), - named_parameters: HashMap::new(), - parameters, - parameter_index: 0, + parameters: Parameters::new(), } } @@ -349,13 +342,7 @@ impl ProgramBuilder { comments: self.comments, connection, auto_commit: true, - parameters: self.parameters.list, + parameters: self.parameters, } } - - pub fn pop_index(&mut self) -> NonZero { - let parameter = self.parameters.get(self.parameter_index).unwrap(); - self.parameter_index += 1; - return parameter.index(); - } } diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 5188a2329..43160eb40 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -277,7 +277,7 @@ pub struct Program { pub cursor_ref: Vec<(Option, CursorType)>, pub database_header: Rc>, pub comments: HashMap, - pub parameters: Vec, + pub parameters: crate::parameters::Parameters, pub connection: Weak, pub auto_commit: bool, } @@ -301,38 +301,6 @@ impl Program { } } - pub fn parameter_count(&self) -> usize { - self.parameters.len() - } - - pub fn parameter_name(&self, index: NonZero) -> Option { - use crate::translate::Parameter; - self.parameters.iter().find_map(|p| match p { - Parameter::Anonymous(i) if *i == index => Some("?".to_string()), - Parameter::Indexed(i) if *i == index => Some(format!("?{i}")), - Parameter::Named(name, i) if *i == index => Some(name.to_owned()), - _ => None, - }) - } - - pub fn parameter_index(&self, name: impl AsRef) -> Option> { - use crate::translate::Parameter; - self.parameters - .iter() - .find_map(|p| { - let Parameter::Named(parameter_name, index) = p else { - return None; - }; - - if name.as_ref() == parameter_name { - return Some(index); - } - - None - }) - .copied() - } - pub fn step<'a>( &self, state: &'a mut ProgramState, diff --git a/test/src/lib.rs b/test/src/lib.rs index 68b4b10e5..e9a368e07 100644 --- a/test/src/lib.rs +++ b/test/src/lib.rs @@ -573,22 +573,99 @@ mod tests { Ok(()) } + #[test] + fn test_statement_reset() -> anyhow::Result<()> { + let _ = env_logger::try_init(); + let tmp_db = TempDatabase::new("create table test (i integer);"); + let conn = tmp_db.connect_limbo(); + + conn.execute("insert into test values (1)")?; + conn.execute("insert into test values (2)")?; + + let mut stmt = conn.prepare("select * from test")?; + + loop { + match stmt.step()? { + StepResult::Row(row) => { + assert_eq!(row.values[0], Value::Integer(1)); + break; + } + StepResult::IO => tmp_db.io.run_once()?, + _ => break, + } + } + + stmt.reset(); + + loop { + match stmt.step()? { + StepResult::Row(row) => { + assert_eq!(row.values[0], Value::Integer(1)); + break; + } + StepResult::IO => tmp_db.io.run_once()?, + _ => break, + } + } + + Ok(()) + } + + #[test] + fn test_statement_reset_bind() -> anyhow::Result<()> { + let _ = env_logger::try_init(); + let tmp_db = TempDatabase::new("create table test (i integer);"); + let conn = tmp_db.connect_limbo(); + + let mut stmt = conn.prepare("select ?")?; + + stmt.bind_at(1.try_into().unwrap(), Value::Integer(1)); + + loop { + match stmt.step()? { + StepResult::Row(row) => { + assert_eq!(row.values[0], Value::Integer(1)); + } + StepResult::IO => tmp_db.io.run_once()?, + _ => break, + } + } + + stmt.reset(); + + stmt.bind_at(1.try_into().unwrap(), Value::Integer(2)); + + loop { + match stmt.step()? { + StepResult::Row(row) => { + assert_eq!(row.values[0], Value::Integer(2)); + } + StepResult::IO => tmp_db.io.run_once()?, + _ => break, + } + } + + Ok(()) + } + #[test] fn test_statement_bind() -> anyhow::Result<()> { let _ = env_logger::try_init(); let tmp_db = TempDatabase::new("create table test (i integer);"); let conn = tmp_db.connect_limbo(); - let mut stmt = conn.prepare("select ?, ?1, :named, ?4")?; + let mut stmt = conn.prepare("select ?, ?1, :named, ?3, ?4")?; stmt.bind_at(1.try_into().unwrap(), Value::Text(&"hello".to_string())); - let i = stmt.parameter_index(":named").unwrap(); + let i = stmt.parameters().index(":named").unwrap(); stmt.bind_at(i, Value::Integer(42)); + stmt.bind_at(3.try_into().unwrap(), Value::Blob(&vec![0x1, 0x2, 0x3])); + stmt.bind_at(4.try_into().unwrap(), Value::Float(0.5)); - assert_eq!(stmt.parameter_count(), 3); + assert_eq!(stmt.parameters().count(), 4); loop { match stmt.step()? { @@ -601,12 +678,16 @@ mod tests { assert_eq!(s, "hello") } - if let Value::Integer(s) = row.values[2] { - assert_eq!(s, 42) + if let Value::Integer(i) = row.values[2] { + assert_eq!(i, 42) } - if let Value::Float(s) = row.values[3] { - assert_eq!(s, 0.5) + if let Value::Blob(v) = row.values[3] { + assert_eq!(v, &vec![0x1 as u8, 0x2, 0x3]) + } + + if let Value::Float(f) = row.values[4] { + assert_eq!(f, 0.5) } } StepResult::IO => { diff --git a/vendored/sqlite3-parser/src/lexer/sql/mod.rs b/vendored/sqlite3-parser/src/lexer/sql/mod.rs index 72cca0b97..fa98282cc 100644 --- a/vendored/sqlite3-parser/src/lexer/sql/mod.rs +++ b/vendored/sqlite3-parser/src/lexer/sql/mod.rs @@ -441,7 +441,12 @@ impl Splitter for Tokenizer { // do not include the '?' in the token Ok((Some((&data[1..=i], TK_VARIABLE)), i + 1)) } - None => Ok((Some((&data[1..], TK_VARIABLE)), data.len())), + None => { + if !data[1..].is_empty() && data[1..].iter().all(|ch| *ch == b'0') { + return Err(Error::BadVariableName(None, None)); + } + Ok((Some((&data[1..], TK_VARIABLE)), data.len())) + } } } b'$' | b'@' | b'#' | b':' => {