From 98e9d334787199c2e912662eefcab9388f6a49e0 Mon Sep 17 00:00:00 2001 From: Jonathan Webb Date: Thu, 6 Feb 2025 11:10:27 -0500 Subject: [PATCH 001/115] Add read implementation of user_version pragma with ReadCookie opcode --- COMPAT.md | 4 +-- core/storage/sqlite3_ondisk.rs | 4 +-- core/translate/pragma.rs | 15 +++++++++++- core/vdbe/explain.rs | 9 +++++++ core/vdbe/insn.rs | 23 ++++++++++++++++++ core/vdbe/mod.rs | 14 ++++++++++- testing/pragma.test | 8 ++++++ testing/testing_user_version_10.db | Bin 0 -> 4096 bytes vendored/sqlite3-parser/src/parser/ast/mod.rs | 2 ++ 9 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 testing/testing_user_version_10.db diff --git a/COMPAT.md b/COMPAT.md index 1c2b9b227..045d4205f 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -160,7 +160,7 @@ The current status of Limbo is: | PRAGMA temp_store_directory | Not Needed | deprecated in SQLite | | PRAGMA threads | No | | | PRAGMA trusted_schema | No | | -| PRAGMA user_version | No | | +| PRAGMA user_version | Partial | Only read implemented | | PRAGMA vdbe_addoptrace | No | | | PRAGMA vdbe_debug | No | | | PRAGMA vdbe_listing | No | | @@ -514,7 +514,7 @@ Modifiers: | PrevAsync | Yes | | | PrevAwait | Yes | | | Program | No | | -| ReadCookie | No | | +| ReadCookie | Partial| no temp databases, only user_version supported | | Real | Yes | | | RealAffinity | Yes | | | Remainder | Yes | | diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 56bc959f9..c0b889f91 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -128,7 +128,7 @@ pub struct DatabaseHeader { text_encoding: u32, /// The "user version" as read and set by the user_version pragma. - user_version: u32, + pub user_version: u32, /// True (non-zero) for incremental-vacuum mode. False (zero) otherwise. incremental_vacuum_enabled: u32, @@ -232,7 +232,7 @@ impl Default for DatabaseHeader { default_page_cache_size: 500, // pages vacuum_mode_largest_root_page: 0, text_encoding: 1, // utf-8 - user_version: 1, + user_version: 0, incremental_vacuum_enabled: 0, application_id: 0, reserved_for_expansion: [0; 20], diff --git a/core/translate/pragma.rs b/core/translate/pragma.rs index ebc854442..e31e732ce 100644 --- a/core/translate/pragma.rs +++ b/core/translate/pragma.rs @@ -11,7 +11,7 @@ use crate::storage::sqlite3_ondisk::{DatabaseHeader, MIN_PAGE_CACHE_SIZE}; use crate::storage::wal::CheckpointMode; use crate::util::normalize_ident; use crate::vdbe::builder::{ProgramBuilder, ProgramBuilderOpts, QueryMode}; -use crate::vdbe::insn::Insn; +use crate::vdbe::insn::{Cookie, Insn}; use crate::vdbe::BranchOffset; use crate::{bail_parse_error, Pager}; use std::str::FromStr; @@ -148,6 +148,10 @@ fn update_pragma( query_pragma(PragmaName::PageCount, schema, None, header, program)?; Ok(()) } + PragmaName::UserVersion => { + // TODO: Implement updating user_version + todo!("updating user_version not yet implemented") + } PragmaName::TableInfo => { // because we need control over the write parameter for the transaction, // this should be unreachable. We have to force-call query_pragma before @@ -241,6 +245,15 @@ fn query_pragma( } } } + PragmaName::UserVersion => { + program.emit_transaction(false); + program.emit_insn(Insn::ReadCookie { + db: 0, + dest: register, + cookie: Cookie::UserVersion, + }); + program.emit_result_row(register, 1); + } } Ok(()) diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index a0bb63023..8ec75a187 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -1232,6 +1232,15 @@ pub fn insn_to_str( 0, "".to_string(), ), + Insn::ReadCookie { db, dest, cookie } => ( + "ReadCookie", + *db as i32, + *dest as i32, + *cookie as i32, + OwnedValue::build_text(Rc::new("".to_string())), + 0, + "".to_string(), + ), }; format!( "{:<4} {:<17} {:<4} {:<4} {:<4} {:<13} {:<2} {}", diff --git a/core/vdbe/insn.rs b/core/vdbe/insn.rs index 223f321aa..0766b02d4 100644 --- a/core/vdbe/insn.rs +++ b/core/vdbe/insn.rs @@ -639,6 +639,29 @@ pub enum Insn { db: usize, dest: usize, }, + /// Read cookie number P3 from database P1 and write it into register P2 + ReadCookie { + db: usize, + dest: usize, + cookie: Cookie, + }, +} + +// TODO: Add remaining cookies. +#[derive(Description, Debug, Clone, Copy)] +pub enum Cookie { + /// The schema cookie. + SchemaVersion = 1, + /// The schema format number. Supported schema formats are 1, 2, 3, and 4. + DatabaseFormat = 2, + /// Default page cache size. + DefaultPageCacheSize = 3, + /// The page number of the largest root b-tree page when in auto-vacuum or incremental-vacuum modes, or zero otherwise. + LargestRootPageNumber = 4, + /// The database text encoding. A value of 1 means UTF-8. A value of 2 means UTF-16le. A value of 3 means UTF-16be. + DatabaseTextEncoding = 5, + /// The "user version" as read and set by the user_version pragma. + UserVersion = 6, } fn cast_text_to_numerical(value: &str) -> OwnedValue { diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 5003b72c6..81427fb00 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -55,7 +55,7 @@ use crate::{resolve_ext_path, Connection, Result, TransactionState, DATABASE_VER use insn::{ exec_add, exec_and, exec_bit_and, exec_bit_not, exec_bit_or, exec_boolean_not, exec_concat, exec_divide, exec_multiply, exec_or, exec_remainder, exec_shift_left, exec_shift_right, - exec_subtract, + exec_subtract, Cookie, }; use likeop::{construct_like_escape_arg, exec_glob, exec_like_with_escape}; use rand::distributions::{Distribution, Uniform}; @@ -2662,6 +2662,18 @@ impl Program { parse_schema_rows(Some(stmt), &mut schema, conn.pager.io.clone())?; state.pc += 1; } + Insn::ReadCookie { db, dest, cookie } => { + if *db > 0 { + // TODO: implement temp databases + todo!("temp databases not implemented yet"); + } + let cookie_value = match cookie { + Cookie::UserVersion => pager.db_header.borrow().user_version.into(), + cookie => todo!("{cookie:?} is not yet implement for ReadCookie"), + }; + state.registers[*dest] = OwnedValue::Integer(cookie_value); + state.pc += 1; + } Insn::ShiftRight { lhs, rhs, dest } => { state.registers[*dest] = exec_shift_right(&state.registers[*lhs], &state.registers[*rhs]); diff --git a/testing/pragma.test b/testing/pragma.test index 436ad48e1..c478c032c 100755 --- a/testing/pragma.test +++ b/testing/pragma.test @@ -41,3 +41,11 @@ do_execsql_test_on_specific_db ":memory:" pragma-page-count-table { CREATE TABLE foo(bar); PRAGMA page_count } {2} + +do_execsql_test_on_specific_db "testing/testing_user_version_10.db" pragma-user-version-user-set { + PRAGMA user_version +} {10} + +do_execsql_test_on_specific_db ":memory:" pragma-user-version-default { + PRAGMA user_version +} {0} \ No newline at end of file diff --git a/testing/testing_user_version_10.db b/testing/testing_user_version_10.db new file mode 100644 index 0000000000000000000000000000000000000000..d51d013f1a4fba55c4eb1060af8cd0adcd3229a7 GIT binary patch literal 4096 zcmWFz^vNtqRY=P(%1ta$FlG>7U}R))P*7lCU|@t|AcYff;go?$Ff!=XMe%~z0t8f! rN{@!XXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD&@2Q1bP5LX literal 0 HcmV?d00001 diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index f149322b3..e42897029 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -1598,6 +1598,8 @@ pub enum PragmaName { PageCount, /// returns information about the columns of a table TableInfo, + /// Returns the user version of the database file. + UserVersion, /// trigger a checkpoint to run on database(s) if WAL is enabled WalCheckpoint, } From 55dd108878c67ee76ceed9522dd0b988889b9114 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 12:38:32 +0400 Subject: [PATCH 002/115] setup simple insertion fuzz test --- Cargo.lock | 1 + core/Cargo.toml | 3 ++- core/storage/btree.rs | 57 +++++++++++++++++++++++++++++++++++++++++++ core/types.rs | 1 + 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 819311184..f25ad611d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1641,6 +1641,7 @@ dependencies = [ "quickcheck", "quickcheck_macros", "rand 0.8.5", + "rand_chacha 0.9.0", "regex", "regex-syntax", "rstest", diff --git a/core/Cargo.toml b/core/Cargo.toml index 687f4ff19..58a29c475 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -94,7 +94,8 @@ rusqlite = "0.29.0" tempfile = "3.8.0" quickcheck = { version = "1.0", default-features = false } quickcheck_macros = { version = "1.0", default-features = false } -rand = "0.8" # Required for quickcheck +rand = "0.8.5" # Required for quickcheck +rand_chacha = "0.9.0" [[bench]] name = "benchmark" diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 4a963581b..280f46eac 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2362,6 +2362,10 @@ fn to_static_buf(buf: &[u8]) -> &'static [u8] { #[cfg(test)] mod tests { + use rand_chacha::rand_core::RngCore; + use rand_chacha::rand_core::SeedableRng; + use rand_chacha::ChaCha8Rng; + use super::*; use crate::io::{Buffer, Completion, MemoryIO, OpenFlags, IO}; use crate::storage::database::FileStorage; @@ -2371,6 +2375,59 @@ mod tests { use std::cell::RefCell; use std::sync::Arc; + fn empty_btree() -> (Rc, usize) { + let db_header = DatabaseHeader::default(); + let page_size = db_header.page_size as usize; + + let io: Arc = Arc::new(MemoryIO::new().unwrap()); + let io_file = io.open_file("test.db", OpenFlags::Create, false).unwrap(); + let page_io = Rc::new(FileStorage::new(io_file)); + + let buffer_pool = Rc::new(BufferPool::new(db_header.page_size as usize)); + let wal_shared = WalFileShared::open_shared(&io, "test.wal", db_header.page_size).unwrap(); + let wal_file = WalFile::new(io.clone(), page_size, wal_shared, buffer_pool.clone()); + let wal = Rc::new(RefCell::new(wal_file)); + + let page_cache = Arc::new(parking_lot::RwLock::new(DumbLruPageCache::new(10))); + let pager = { + let db_header = Rc::new(RefCell::new(db_header.clone())); + Pager::finish_open(db_header, page_io, wal, io, page_cache, buffer_pool).unwrap() + }; + let pager = Rc::new(pager); + let page1 = pager.allocate_page().unwrap(); + btree_init_page(&page1, PageType::TableLeaf, &db_header, 0); + (pager, page1.get().id) + } + + #[test] + pub fn btree_insert_fuzz() { + let (pager, root_page) = empty_btree(); + let mut cursor = BTreeCursor::new(pager, root_page); + let mut keys = Vec::new(); + let mut rng = ChaCha8Rng::seed_from_u64(0); + for _ in 0..16 { + let size = (rng.next_u64() % 4096) as usize; + let key = (rng.next_u64() % (1 << 30)) as i64; + keys.push(key); + println!("INSERT INTO t VALUES ({}, randomblob({}));", key, size); + let key = OwnedValue::Integer(key); + let value = Record::new(vec![OwnedValue::Blob(Rc::new(vec![0; size]))]); + cursor.insert(&key, &value, false).unwrap(); + } + + for key in keys { + let seek_key = SeekKey::TableRowId(key as u64); + assert!( + matches!( + cursor.seek(seek_key, SeekOp::EQ).unwrap(), + CursorResult::Ok(true) + ), + "key {} is not found", + key + ); + } + } + #[allow(clippy::arc_with_non_send_sync)] fn setup_test_env(database_size: u32) -> (Rc, Rc>) { let page_size = 512; diff --git a/core/types.rs b/core/types.rs index ae31314c1..d72ddc284 100644 --- a/core/types.rs +++ b/core/types.rs @@ -727,6 +727,7 @@ impl Cursor { } } +#[derive(Debug)] pub enum CursorResult { Ok(T), IO, From 75e2f01ec4db7d4cee5051315ba57ad2579c9ea1 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 13:03:38 +0400 Subject: [PATCH 003/115] print btree for debugging --- Cargo.lock | 5 ++ core/Cargo.toml | 1 + core/storage/btree.rs | 90 +++++++++++++++++++++++++++++----- core/storage/sqlite3_ondisk.rs | 2 +- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f25ad611d..e64dc3c1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -736,6 +736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" dependencies = [ "log", + "regex", ] [[package]] @@ -767,7 +768,10 @@ version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" dependencies = [ + "anstream", + "anstyle", "env_filter", + "humantime", "log", ] @@ -1612,6 +1616,7 @@ dependencies = [ "chrono", "criterion", "crossbeam-skiplist", + "env_logger 0.11.6", "fallible-iterator 0.3.0", "getrandom 0.2.15", "hex", diff --git a/core/Cargo.toml b/core/Cargo.toml index 58a29c475..6799fe29b 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -96,6 +96,7 @@ quickcheck = { version = "1.0", default-features = false } quickcheck_macros = { version = "1.0", default-features = false } rand = "0.8.5" # Required for quickcheck rand_chacha = "0.9.0" +env_logger = "0.11.6" [[bench]] name = "benchmark" diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 280f46eac..efdcb47ae 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2375,6 +2375,65 @@ mod tests { use std::cell::RefCell; use std::sync::Arc; + fn format_btree(pager: Rc, page_idx: usize, depth: usize) -> String { + let cursor = BTreeCursor::new(pager.clone(), page_idx); + let page = pager.read_page(page_idx).unwrap(); + let page = page.get(); + let contents = page.contents.as_ref().unwrap(); + let page_type = contents.page_type(); + let mut current = Vec::new(); + let mut child = Vec::new(); + for cell_idx in 0..contents.cell_count() { + let cell = contents + .cell_get( + cell_idx, + pager.clone(), + cursor.payload_overflow_threshold_max(page_type), + cursor.payload_overflow_threshold_min(page_type), + cursor.usable_space(), + ) + .unwrap(); + match cell { + BTreeCell::TableInteriorCell(cell) => { + current.push(format!( + "node[rowid:{}, ptr(<=):{}]", + cell._rowid, cell._left_child_page + )); + child.push(format_btree( + pager.clone(), + cell._left_child_page as usize, + depth + 2, + )); + } + BTreeCell::TableLeafCell(cell) => { + current.push(format!( + "leaf[rowid:{}, len(payload):{}, overflow:{}]", + cell._rowid, + cell._payload.len(), + cell.first_overflow_page.is_some() + )); + } + _ => panic!("unsupported btree cell: {:?}", cell), + } + } + if let Some(rightmost) = contents.rightmost_pointer() { + child.push(format_btree(pager.clone(), rightmost as usize, depth + 2)); + } + let current = format!( + "{}-page:{}, ptr(right):{}\n{}+cells:{}", + " ".repeat(depth), + page_idx, + contents.rightmost_pointer().unwrap_or(0), + " ".repeat(depth), + current.join(", ") + ); + if child.is_empty() { + current + } else { + current + "\n" + &child.join("\n") + } + } + fn empty_btree() -> (Rc, usize) { let db_header = DatabaseHeader::default(); let page_size = db_header.page_size as usize; @@ -2401,30 +2460,35 @@ mod tests { #[test] pub fn btree_insert_fuzz() { + let _ = env_logger::init(); let (pager, root_page) = empty_btree(); - let mut cursor = BTreeCursor::new(pager, root_page); + let mut cursor = BTreeCursor::new(pager.clone(), root_page); let mut keys = Vec::new(); let mut rng = ChaCha8Rng::seed_from_u64(0); for _ in 0..16 { let size = (rng.next_u64() % 4096) as usize; let key = (rng.next_u64() % (1 << 30)) as i64; keys.push(key); - println!("INSERT INTO t VALUES ({}, randomblob({}));", key, size); + log::info!("INSERT INTO t VALUES ({}, randomblob({}));", key, size); let key = OwnedValue::Integer(key); let value = Record::new(vec![OwnedValue::Blob(Rc::new(vec![0; size]))]); cursor.insert(&key, &value, false).unwrap(); - } - - for key in keys { - let seek_key = SeekKey::TableRowId(key as u64); - assert!( - matches!( - cursor.seek(seek_key, SeekOp::EQ).unwrap(), - CursorResult::Ok(true) - ), - "key {} is not found", - key + log::info!( + "=========== btree ===========\n{}\n\n", + format_btree(pager.clone(), root_page, 0) ); + + for key in keys.iter() { + let seek_key = SeekKey::TableRowId(*key as u64); + assert!( + matches!( + cursor.seek(seek_key, SeekOp::EQ).unwrap(), + CursorResult::Ok(true) + ), + "key {} is not found", + key + ); + } } } diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 4963fa669..fbdac7dca 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -362,7 +362,7 @@ pub fn write_header_to_buf(buf: &mut [u8], header: &DatabaseHeader) { } #[repr(u8)] -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Copy)] pub enum PageType { IndexInterior = 2, TableInterior = 5, From 0d27ae9402bc7d6b6cc9ba6c1157c9ce4d81ef28 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 14:49:01 +0200 Subject: [PATCH 004/115] Add failing tests for CTE functionality --- testing/subquery.test | 221 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 220 insertions(+), 1 deletion(-) diff --git a/testing/subquery.test b/testing/subquery.test index 4754d1563..3eb2e3326 100644 --- a/testing/subquery.test +++ b/testing/subquery.test @@ -10,6 +10,14 @@ do_execsql_test subquery-inner-filter { ) sub; } {hat!!!} +do_execsql_test subquery-inner-filter-cte { + with sub as ( + select concat(name, '!!!') as loud_hat + from products where name = 'hat' + ) + select sub.loud_hat from sub; +} {hat!!!} + do_execsql_test subquery-outer-filter { select sub.loud_hat from ( select concat(name, '!!!') as loud_hat @@ -17,6 +25,14 @@ do_execsql_test subquery-outer-filter { ) sub where sub.loud_hat = 'hat!!!' } {hat!!!} +do_execsql_test subquery-outer-filter-cte { + with sub as ( + select concat(name, '!!!') as loud_hat + from products + ) + select sub.loud_hat from sub where sub.loud_hat = 'hat!!!' +} {hat!!!} + do_execsql_test subquery-without-alias { select loud_hat from ( select concat(name, '!!!') as loud_hat @@ -24,30 +40,66 @@ do_execsql_test subquery-without-alias { ); } {hat!!!} +do_execsql_test subquery-without-alias-cte { + with cte as ( + select concat(name, '!!!') as loud_hat + from products where name = 'hat' + ) + select loud_hat from cte; +} {hat!!!} + do_execsql_test subquery-no-alias-on-col { select price from ( select * from products where name = 'hat' ) } {79.0} +do_execsql_test subquery-no-alias-on-col-cte { + with cte as ( + select * from products where name = 'hat' + ) + select price from cte +} {79.0} + do_execsql_test subquery-no-alias-on-col-named { select price from ( select price from products where name = 'hat' ) } {79.0} +do_execsql_test subquery-no-alias-on-col-named-cte { + with cte as ( + select price from products where name = 'hat' + ) + select price from cte +} {79.0} + do_execsql_test subquery-select-star { select * from ( select price, price + 1.0, name from products where name = 'hat' ) } {79.0|80.0|hat} +do_execsql_test subquery-select-star-cte { + with cte as ( + select price, price + 1.0, name from products where name = 'hat' + ) + select * from cte +} {79.0|80.0|hat} + do_execsql_test subquery-select-table-star { select sub.* from ( select price, price + 1.0, name from products where name = 'hat' ) sub } {79.0|80.0|hat} +do_execsql_test subquery-select-table-star-cte { + with sub as ( + select price, price + 1.0, name from products where name = 'hat' + ) + select sub.* from sub +} {79.0|80.0|hat} + do_execsql_test nested-subquery { select sub.loudest_hat from ( select upper(nested_sub.loud_hat) as loudest_hat from ( @@ -57,6 +109,17 @@ do_execsql_test nested-subquery { ) sub; } {HAT!!!} +do_execsql_test nested-subquery-cte { + with nested_sub as ( + select concat(name, '!!!') as loud_hat + from products where name = 'hat' + ), + sub as ( + select upper(nested_sub.loud_hat) as loudest_hat from nested_sub + ) + select sub.loudest_hat from sub; +} {HAT!!!} + do_execsql_test subquery-orderby-limit { select upper(sub.loud_name) as loudest_name from ( @@ -69,6 +132,18 @@ do_execsql_test subquery-orderby-limit { BOOTS!!! CAP!!!} +do_execsql_test subquery-orderby-limit-cte { + with sub as ( + select concat(name, '!!!') as loud_name + from products + order by name + limit 3 + ) + select upper(sub.loud_name) as loudest_name from sub; +} {ACCESSORIES!!! +BOOTS!!! +CAP!!!} + do_execsql_test table-join-subquery { select sub.product_name, p.name from products p join ( @@ -77,6 +152,16 @@ do_execsql_test table-join-subquery { ) sub on p.name = sub.product_name where p.name = 'hat' } {hat|hat} +do_execsql_test table-join-subquery-cte { + with sub as ( + select name as product_name + from products + ) + select sub.product_name, p.name + from products p join sub on p.name = sub.product_name + where p.name = 'hat' +} {hat|hat} + do_execsql_test subquery-join-table { select sub.product_name, p.name from ( @@ -85,6 +170,16 @@ do_execsql_test subquery-join-table { ) sub join products p on sub.product_name = p.name where sub.product_name = 'hat' } {hat|hat} +do_execsql_test subquery-join-table-cte { + with sub as ( + select name as product_name + from products + ) + select sub.product_name, p.name + from sub join products p on sub.product_name = p.name + where sub.product_name = 'hat' +} {hat|hat} + do_execsql_test subquery-join-subquery { select sub1.sus_name, sub2.truthful_name from ( @@ -98,6 +193,21 @@ do_execsql_test subquery-join-subquery { ) sub2; } {"cap|no cap"} +do_execsql_test subquery-join-subquery-cte { + with sub1 as ( + select name as sus_name + from products + where name = 'cap' + ), + sub2 as ( + select concat('no ', name) as truthful_name + from products + where name = 'cap' + ) + select sub1.sus_name, sub2.truthful_name + from sub1 join sub2; +} {"cap|no cap"} + do_execsql_test select-star-table-subquery { select * from products p join ( @@ -107,6 +217,16 @@ do_execsql_test select-star-table-subquery { ) sub on p.name = sub.name; } {1|hat|79.0|hat|79.0} +do_execsql_test select-star-table-subquery-cte { + with sub as ( + select name, price + from products + where name = 'hat' + ) + select * + from products p join sub on p.name = sub.name; +} {1|hat|79.0|hat|79.0} + do_execsql_test select-star-subquery-table { select * from ( @@ -116,6 +236,16 @@ do_execsql_test select-star-subquery-table { ) sub join products p on sub.name = p.name; } {hat|79.0|1|hat|79.0} +do_execsql_test select-star-subquery-table-cte { + with sub as ( + select name, price + from products + where name = 'hat' + ) + select * + from sub join products p on sub.name = p.name; +} {hat|79.0|1|hat|79.0} + do_execsql_test select-star-subquery-subquery { select * from ( @@ -129,6 +259,20 @@ do_execsql_test select-star-subquery-subquery { ) sub2 on sub1.price = sub2.price; } {hat|79.0|79.0} +do_execsql_test select-star-subquery-subquery-cte { + with sub1 as ( + select name, price + from products + where name = 'hat' + ), + sub2 as ( + select price + from products + where name = 'hat' + ) + select * + from sub1 join sub2 on sub1.price = sub2.price; +} {hat|79.0|79.0} do_execsql_test subquery-inner-grouping { select is_jennifer, person_count @@ -139,6 +283,16 @@ do_execsql_test subquery-inner-grouping { } {1|151 0|9849} +do_execsql_test subquery-inner-grouping-cte { + with cte as ( + select first_name = 'Jennifer' as is_jennifer, count(1) as person_count from users + group by first_name = 'Jennifer' + ) + select is_jennifer, person_count + from cte order by person_count asc +} {1|151 +0|9849} + do_execsql_test subquery-outer-grouping { select is_jennifer, count(1) as person_count from ( @@ -147,6 +301,15 @@ do_execsql_test subquery-outer-grouping { } {1|151 0|9849} +do_execsql_test subquery-outer-grouping-cte { + with cte as ( + select first_name = 'Jennifer' as is_jennifer from users + ) + select is_jennifer, count(1) as person_count + from cte group by is_jennifer order by count(1) asc +} {1|151 +0|9849} + do_execsql_test subquery-join-using-with-outer-limit { SELECT p.name, sub.funny_name FROM products p @@ -159,6 +322,19 @@ do_execsql_test subquery-join-using-with-outer-limit { cap|cap-lol shirt|shirt-lol"} +do_execsql_test subquery-join-using-with-outer-limit-cte { + WITH sub AS ( + select id, concat(name, '-lol') as funny_name + from products + ) + SELECT p.name, sub.funny_name + FROM products p + JOIN sub USING (id) + LIMIT 3; +} {"hat|hat-lol +cap|cap-lol +shirt|shirt-lol"} + do_execsql_test subquery-join-using-with-inner-limit { SELECT p.name, sub.funny_name FROM products p @@ -171,6 +347,19 @@ do_execsql_test subquery-join-using-with-inner-limit { cap|cap-lol shirt|shirt-lol"} +do_execsql_test subquery-join-using-with-inner-limit-cte { + WITH sub AS ( + select id, concat(name, '-lol') as funny_name + from products + limit 3 + ) + SELECT p.name, sub.funny_name + FROM products p + JOIN sub USING (id); +} {"hat|hat-lol +cap|cap-lol +shirt|shirt-lol"} + do_execsql_test subquery-join-using-with-both-limits { SELECT p.name, sub.funny_name FROM products p @@ -183,6 +372,19 @@ do_execsql_test subquery-join-using-with-both-limits { } {"hat|hat-lol cap|cap-lol"} +do_execsql_test subquery-join-using-with-both-limits-cte { + WITH sub AS ( + select id, concat(name, '-lol') as funny_name + from products + limit 3 + ) + SELECT p.name, sub.funny_name + FROM products p + JOIN sub USING (id) + LIMIT 2; +} {"hat|hat-lol +cap|cap-lol"} + do_execsql_test subquery-containing-join { select foo, bar from ( @@ -191,4 +393,21 @@ do_execsql_test subquery-containing-join { ) limit 3; } {hat|Jamie cap|Cindy -shirt|Tommy} \ No newline at end of file +shirt|Tommy} + +do_execsql_test subquery-containing-join-cte { + with cte as ( + select p.name as foo, u.first_name as bar + from products p join users u using (id) + ) + select foo, bar + from cte limit 3; +} {hat|Jamie +cap|Cindy +shirt|Tommy} + +do_execsql_test subquery-ignore-unused-cte { + with unused as (select last_name from users), + sub as (select first_name from users where first_name = 'Jamie' limit 1) + select * from sub; +} {Jamie} From 338c27dad60e4b02e178d6229aa0c2dbdc9cecee Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 14:49:46 +0200 Subject: [PATCH 005/115] introduce Scope and Cte structs --- core/translate/planner.rs | 40 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/core/translate/planner.rs b/core/translate/planner.rs index 311458f9f..2677c2176 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -352,7 +352,45 @@ fn parse_from_clause_table( } } -pub fn parse_from( +/// A scope is a list of tables that are visible to the current query. +/// It is used to resolve table references in the FROM clause. +/// To resolve table references that are potentially ambiguous, the resolution +/// first looks at schema tables and tables in the current scope (which currently just means CTEs in the current query), +/// and only after that looks at whether a table from an outer (upper) query level matches. +/// +/// For example: +/// +/// WITH nested AS (SELECT foo FROM bar) +/// WITH sub AS (SELECT foo FROM bar) +/// SELECT * FROM sub +/// +/// 'sub' would preferentially refer to the 'foo' column from the 'bar' table in the catalog. +/// With an explicit reference like: +/// +/// SELECT nested.foo FROM sub +/// +/// 'nested.foo' would refer to the 'foo' column from the 'nested' CTE. +/// +/// TODO: we should probably use Scope in all of our identifier resolution, because it allows for e.g. +/// WITH users AS (SELECT * FROM products) SELECT * FROM users <-- returns products, even if there is a table named 'users' in the catalog! +/// +/// Currently we are treating Schema as a first-class object in identifier resolution, when in reality +/// be part of the 'Scope' struct. +pub struct Scope<'a> { + /// The tables that are explicitly present in the current query, including catalog tables and CTEs. + tables: Vec, + ctes: Vec, + /// The parent scope, if any. For example, a second CTE has access to the first CTE via the parent scope. + parent: Option<&'a Scope<'a>>, +} + +pub struct Cte { + /// The name of the CTE. + name: String, + /// The query plan for the CTE. + /// Currently we only support SELECT queries in CTEs. + plan: SelectPlan, +} schema: &Schema, mut from: Option, syms: &SymbolTable, From 9e70e8fe02000363e7c261a2d2085baa1d1997e5 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 14:50:05 +0200 Subject: [PATCH 006/115] Add basic CTE support --- core/lib.rs | 1 + core/translate/planner.rs | 193 +++++++++++++++++++++++++++++--------- core/translate/select.rs | 11 ++- 3 files changed, 159 insertions(+), 46 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index 94853aa80..b280703f5 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -345,6 +345,7 @@ impl Connection { &self.schema.borrow(), *select, &self.db.syms.borrow(), + None, )?; optimize_plan(&mut plan, &self.schema.borrow())?; println!("{}", plan); diff --git a/core/translate/planner.rs b/core/translate/planner.rs index 2677c2176..5d09bbadd 100644 --- a/core/translate/planner.rs +++ b/core/translate/planner.rs @@ -1,7 +1,7 @@ use super::{ plan::{ - Aggregate, JoinInfo, Operation, Plan, ResultSetColumn, SelectQueryType, TableReference, - WhereTerm, + Aggregate, JoinInfo, Operation, Plan, ResultSetColumn, SelectPlan, SelectQueryType, + TableReference, WhereTerm, }, select::prepare_select_plan, SymbolTable, @@ -13,7 +13,9 @@ use crate::{ vdbe::BranchOffset, Result, VirtualTable, }; -use sqlite3_parser::ast::{self, Expr, FromClause, JoinType, Limit, UnaryOperator}; +use sqlite3_parser::ast::{ + self, Expr, FromClause, JoinType, Limit, Materialized, UnaryOperator, With, +}; pub const ROWID: &str = "rowid"; @@ -278,46 +280,92 @@ pub fn bind_column_references( } } -fn parse_from_clause_table( +fn parse_from_clause_table<'a>( schema: &Schema, table: ast::SelectTable, - cur_table_index: usize, + scope: &mut Scope<'a>, syms: &SymbolTable, -) -> Result { +) -> Result<()> { match table { ast::SelectTable::Table(qualified_name, maybe_alias, _) => { let normalized_qualified_name = normalize_ident(qualified_name.name.0.as_str()); - let Some(table) = schema.get_table(&normalized_qualified_name) else { - crate::bail_parse_error!("Table {} not found", normalized_qualified_name); + // Check if the FROM clause table is referring to a CTE in the current scope. + if let Some(cte) = scope + .ctes + .iter() + .find(|cte| cte.name == normalized_qualified_name) + { + // CTE can be rewritten as a subquery. + // TODO: find a way not to clone the CTE plan here. + let cte_table = + TableReference::new_subquery(cte.name.clone(), cte.plan.clone(), None); + scope.tables.push(cte_table); + return Ok(()); }; - let alias = maybe_alias - .map(|a| match a { - ast::As::As(id) => id, - ast::As::Elided(id) => id, - }) - .map(|a| a.0); - Ok(TableReference { - op: Operation::Scan { iter_dir: None }, - table: Table::BTree(table.clone()), - identifier: alias.unwrap_or(normalized_qualified_name), - join_info: None, - }) + // Check if our top level schema has this table. + if let Some(table) = schema.get_table(&normalized_qualified_name) { + let alias = maybe_alias + .map(|a| match a { + ast::As::As(id) => id, + ast::As::Elided(id) => id, + }) + .map(|a| a.0); + scope.tables.push(TableReference { + op: Operation::Scan { iter_dir: None }, + table: Table::BTree(table.clone()), + identifier: alias.unwrap_or(normalized_qualified_name), + join_info: None, + }); + return Ok(()); + }; + + // Check if the outer query scope has this table. + if let Some(outer_scope) = scope.parent { + if let Some(table_ref_idx) = outer_scope + .tables + .iter() + .position(|t| t.identifier == normalized_qualified_name) + { + // TODO: avoid cloning the table reference here. + scope.tables.push(outer_scope.tables[table_ref_idx].clone()); + return Ok(()); + } + if let Some(cte) = outer_scope + .ctes + .iter() + .find(|cte| cte.name == normalized_qualified_name) + { + // TODO: avoid cloning the CTE plan here. + let cte_table = + TableReference::new_subquery(cte.name.clone(), cte.plan.clone(), None); + scope.tables.push(cte_table); + return Ok(()); + } + } + + crate::bail_parse_error!("Table {} not found", normalized_qualified_name); } ast::SelectTable::Select(subselect, maybe_alias) => { - let Plan::Select(mut subplan) = prepare_select_plan(schema, *subselect, syms)? else { + let Plan::Select(mut subplan) = + prepare_select_plan(schema, *subselect, syms, Some(scope))? + else { unreachable!(); }; subplan.query_type = SelectQueryType::Subquery { yield_reg: usize::MAX, // will be set later in bytecode emission coroutine_implementation_start: BranchOffset::Placeholder, // will be set later in bytecode emission }; + let cur_table_index = scope.tables.len(); let identifier = maybe_alias .map(|a| match a { ast::As::As(id) => id.0.clone(), ast::As::Elided(id) => id.0.clone(), }) .unwrap_or(format!("subquery_{}", cur_table_index)); - Ok(TableReference::new_subquery(identifier, subplan, None)) + scope + .tables + .push(TableReference::new_subquery(identifier, subplan, None)); + Ok(()) } ast::SelectTable::TableCall(qualified_name, maybe_args, maybe_alias) => { let normalized_name = &normalize_ident(qualified_name.name.0.as_str()); @@ -332,7 +380,7 @@ fn parse_from_clause_table( }) .unwrap_or(normalized_name.to_string()); - Ok(TableReference { + scope.tables.push(TableReference { op: Operation::Scan { iter_dir: None }, join_info: None, table: Table::Virtual( @@ -346,7 +394,8 @@ fn parse_from_clause_table( ) .into(), identifier: alias.clone(), - }) + }); + Ok(()) } _ => todo!(), } @@ -391,25 +440,85 @@ pub struct Cte { /// Currently we only support SELECT queries in CTEs. plan: SelectPlan, } + +pub fn parse_from<'a>( schema: &Schema, mut from: Option, syms: &SymbolTable, + with: Option, out_where_clause: &mut Vec, + outer_scope: Option<&'a Scope<'a>>, ) -> Result> { if from.as_ref().and_then(|f| f.select.as_ref()).is_none() { return Ok(vec![]); } + let mut scope = Scope { + tables: vec![], + ctes: vec![], + parent: outer_scope, + }; + + if let Some(with) = with { + if with.recursive { + crate::bail_parse_error!("Recursive CTEs are not yet supported"); + } + for cte in with.ctes { + if cte.materialized == Materialized::Yes { + crate::bail_parse_error!("Materialized CTEs are not yet supported"); + } + if cte.columns.is_some() { + crate::bail_parse_error!("CTE columns are not yet supported"); + } + + // Check if normalized name conflicts with catalog tables or other CTEs + // TODO: sqlite actually allows overriding a catalog table with a CTE. + // We should carry over the 'Scope' struct to all of our identifier resolution. + let cte_name_normalized = normalize_ident(&cte.tbl_name.0); + if schema.get_table(&cte_name_normalized).is_some() { + crate::bail_parse_error!( + "CTE name {} conflicts with catalog table name", + cte.tbl_name.0 + ); + } + if scope + .tables + .iter() + .any(|t| t.identifier == cte_name_normalized) + { + crate::bail_parse_error!("CTE name {} conflicts with table name", cte.tbl_name.0); + } + if scope.ctes.iter().any(|c| c.name == cte_name_normalized) { + crate::bail_parse_error!("duplicate WITH table name {}", cte.tbl_name.0); + } + + // CTE can refer to other CTEs that came before it, plus any schema tables or tables in the outer scope. + let cte_plan = prepare_select_plan(schema, *cte.select, syms, Some(&scope))?; + let Plan::Select(mut cte_plan) = cte_plan else { + crate::bail_parse_error!("Only SELECT queries are currently supported in CTEs"); + }; + // CTE can be rewritten as a subquery. + cte_plan.query_type = SelectQueryType::Subquery { + yield_reg: usize::MAX, // will be set later in bytecode emission + coroutine_implementation_start: BranchOffset::Placeholder, // will be set later in bytecode emission + }; + scope.ctes.push(Cte { + name: cte_name_normalized, + plan: cte_plan, + }); + } + } + let mut from_owned = std::mem::take(&mut from).unwrap(); let select_owned = *std::mem::take(&mut from_owned.select).unwrap(); let joins_owned = std::mem::take(&mut from_owned.joins).unwrap_or_default(); - let mut tables = vec![parse_from_clause_table(schema, select_owned, 0, syms)?]; + parse_from_clause_table(schema, select_owned, &mut scope, syms)?; for join in joins_owned.into_iter() { - parse_join(schema, join, syms, &mut tables, out_where_clause)?; + parse_join(schema, join, syms, &mut scope, out_where_clause)?; } - Ok(tables) + Ok(scope.tables) } pub fn parse_where( @@ -489,11 +598,11 @@ fn get_rightmost_table_referenced_in_expr<'a>(predicate: &'a ast::Expr) -> Resul Ok(max_table_idx) } -fn parse_join( +fn parse_join<'a>( schema: &Schema, join: ast::JoinedSelectTable, syms: &SymbolTable, - tables: &mut Vec, + scope: &mut Scope<'a>, out_where_clause: &mut Vec, ) -> Result<()> { let ast::JoinedSelectTable { @@ -502,9 +611,7 @@ fn parse_join( constraint, } = join; - let cur_table_index = tables.len(); - let table = parse_from_clause_table(schema, table, cur_table_index, syms)?; - tables.push(table); + parse_from_clause_table(schema, table, scope, syms)?; let (outer, natural) = match join_operator { ast::JoinOperator::TypedJoin(Some(join_type)) => { @@ -522,15 +629,15 @@ fn parse_join( } let constraint = if natural { - assert!(tables.len() >= 2); - let rightmost_table = tables.last().unwrap(); + assert!(scope.tables.len() >= 2); + let rightmost_table = scope.tables.last().unwrap(); // NATURAL JOIN is first transformed into a USING join with the common columns let right_cols = rightmost_table.columns(); let mut distinct_names: Option = None; // TODO: O(n^2) maybe not great for large tables or big multiway joins for right_col in right_cols.iter() { let mut found_match = false; - for left_table in tables.iter().take(tables.len() - 1) { + for left_table in scope.tables.iter().take(scope.tables.len() - 1) { for left_col in left_table.columns().iter() { if left_col.name == right_col.name { if let Some(distinct_names) = distinct_names.as_mut() { @@ -568,10 +675,10 @@ fn parse_join( let mut preds = vec![]; break_predicate_at_and_boundaries(expr, &mut preds); for predicate in preds.iter_mut() { - bind_column_references(predicate, tables, None)?; + bind_column_references(predicate, &scope.tables, None)?; } for pred in preds { - let cur_table_idx = tables.len() - 1; + let cur_table_idx = scope.tables.len() - 1; let eval_at_loop = if outer { cur_table_idx } else { @@ -588,10 +695,10 @@ fn parse_join( // USING join is replaced with a list of equality predicates for distinct_name in distinct_names.iter() { let name_normalized = normalize_ident(distinct_name.0.as_str()); - let cur_table_idx = tables.len() - 1; - let left_tables = &tables[..cur_table_idx]; + let cur_table_idx = scope.tables.len() - 1; + let left_tables = &scope.tables[..cur_table_idx]; assert!(!left_tables.is_empty()); - let right_table = tables.last().unwrap(); + let right_table = scope.tables.last().unwrap(); let mut left_col = None; for (left_table_idx, left_table) in left_tables.iter().enumerate() { left_col = left_table @@ -658,9 +765,9 @@ fn parse_join( } } - assert!(tables.len() >= 2); - let last_idx = tables.len() - 1; - let rightmost_table = tables.get_mut(last_idx).unwrap(); + assert!(scope.tables.len() >= 2); + let last_idx = scope.tables.len() - 1; + let rightmost_table = scope.tables.get_mut(last_idx).unwrap(); rightmost_table.join_info = Some(JoinInfo { outer, using }); Ok(()) diff --git a/core/translate/select.rs b/core/translate/select.rs index 2a055afd2..c56d89d9e 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -1,5 +1,6 @@ use super::emitter::emit_program; use super::plan::{select_star, Operation, Search, SelectQueryType}; +use super::planner::Scope; use crate::function::{AggFunc, ExtFunc, Func}; use crate::translate::optimizer::optimize_plan; use crate::translate::plan::{Aggregate, Direction, GroupBy, Plan, ResultSetColumn, SelectPlan}; @@ -20,7 +21,7 @@ pub fn translate_select( select: ast::Select, syms: &SymbolTable, ) -> Result { - let mut select_plan = prepare_select_plan(schema, select, syms)?; + let mut select_plan = prepare_select_plan(schema, select, syms, None)?; optimize_plan(&mut select_plan, schema)?; let Plan::Select(ref select) = select_plan else { panic!("select_plan is not a SelectPlan"); @@ -36,10 +37,11 @@ pub fn translate_select( Ok(program) } -pub fn prepare_select_plan( +pub fn prepare_select_plan<'a>( schema: &Schema, select: ast::Select, syms: &SymbolTable, + outer_scope: Option<&'a Scope<'a>>, ) -> Result { match *select.body.select { ast::OneSelect::Select { @@ -56,8 +58,11 @@ pub fn prepare_select_plan( let mut where_predicates = vec![]; + let with = select.with; + // Parse the FROM clause into a vec of TableReferences. Fold all the join conditions expressions into the WHERE clause. - let table_references = parse_from(schema, from, syms, &mut where_predicates)?; + let table_references = + parse_from(schema, from, syms, with, &mut where_predicates, outer_scope)?; // Preallocate space for the result columns let result_columns = Vec::with_capacity( From 6a75266f14fc955ff17a313eb18b5587950d74fe Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 14:50:15 +0200 Subject: [PATCH 007/115] Update COMPAT.MD to include basic CTE support --- COMPAT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/COMPAT.md b/COMPAT.md index 1c2b9b227..cd666abff 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -86,7 +86,7 @@ The current status of Limbo is: | UPDATE | No | | | UPSERT | No | | | VACUUM | No | | -| WITH clause | No | | +| WITH clause | Partial | No RECURSIVE, no MATERIALIZED, only SELECT supported in CTEs | #### [PRAGMA](https://www.sqlite.org/pragma.html) From 75898027a02ebdef6ad07bf9921b05f578cf0731 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Thu, 6 Feb 2025 20:29:47 -0500 Subject: [PATCH 008/115] Remove unnecessary reference counting from completion io callbacks --- bindings/wasm/lib.rs | 24 ++++++++++++------------ core/io/generic.rs | 17 ++++++----------- core/io/io_uring.rs | 20 +++++++------------- core/io/memory.rs | 10 +++++----- core/io/mod.rs | 6 +++--- core/io/unix.rs | 23 +++++++++-------------- core/io/windows.rs | 15 +++++---------- core/lib.rs | 2 +- core/storage/btree.rs | 4 ++-- core/storage/database.rs | 22 +++++++++------------- core/storage/sqlite3_ondisk.rs | 24 ++++++++++++------------ core/storage/wal.rs | 2 +- simulator/runner/file.rs | 6 +++--- 13 files changed, 75 insertions(+), 100 deletions(-) diff --git a/bindings/wasm/lib.rs b/bindings/wasm/lib.rs index 26f877199..f25f762ed 100644 --- a/bindings/wasm/lib.rs +++ b/bindings/wasm/lib.rs @@ -228,9 +228,9 @@ impl limbo_core::File for File { Ok(()) } - fn pread(&self, pos: usize, c: Rc) -> Result<()> { - let r = match &*c { - limbo_core::Completion::Read(r) => r, + fn pread(&self, pos: usize, c: limbo_core::Completion) -> Result<()> { + let r = match &c { + limbo_core::Completion::Read(ref r) => r, _ => unreachable!(), }; { @@ -247,10 +247,10 @@ impl limbo_core::File for File { &self, pos: usize, buffer: Rc>, - c: Rc, + c: limbo_core::Completion, ) -> Result<()> { - let w = match &*c { - limbo_core::Completion::Write(w) => w, + let w = match &c { + limbo_core::Completion::Write(ref w) => w, _ => unreachable!(), }; let buf = buffer.borrow(); @@ -260,7 +260,7 @@ impl limbo_core::File for File { Ok(()) } - fn sync(&self, c: Rc) -> Result<()> { + fn sync(&self, c: limbo_core::Completion) -> Result<()> { self.vfs.sync(self.fd); c.complete(0); Ok(()) @@ -331,9 +331,9 @@ impl DatabaseStorage { } impl limbo_core::DatabaseStorage for DatabaseStorage { - fn read_page(&self, page_idx: usize, c: Rc) -> Result<()> { - let r = match c.as_ref() { - limbo_core::Completion::Read(r) => r, + fn read_page(&self, page_idx: usize, c: limbo_core::Completion) -> Result<()> { + let r = match c { + limbo_core::Completion::Read(ref r) => r, _ => unreachable!(), }; let size = r.buf().len(); @@ -350,7 +350,7 @@ impl limbo_core::DatabaseStorage for DatabaseStorage { &self, page_idx: usize, buffer: Rc>, - c: Rc, + c: limbo_core::Completion, ) -> Result<()> { let size = buffer.borrow().len(); let pos = (page_idx - 1) * size; @@ -358,7 +358,7 @@ impl limbo_core::DatabaseStorage for DatabaseStorage { Ok(()) } - fn sync(&self, _c: Rc) -> Result<()> { + fn sync(&self, _c: limbo_core::Completion) -> Result<()> { todo!() } } diff --git a/core/io/generic.rs b/core/io/generic.rs index 79bcde49c..a72b1837d 100644 --- a/core/io/generic.rs +++ b/core/io/generic.rs @@ -48,7 +48,7 @@ pub struct GenericFile { impl File for GenericFile { // Since we let the OS handle the locking, file locking is not supported on the generic IO implementation // No-op implementation allows compilation but provides no actual file locking. - fn lock_file(&self, exclusive: bool) -> Result<()> { + fn lock_file(&self, _exclusive: bool) -> Result<()> { Ok(()) } @@ -56,12 +56,12 @@ impl File for GenericFile { Ok(()) } - fn pread(&self, pos: usize, c: Rc) -> Result<()> { + fn pread(&self, pos: usize, c: Completion) -> Result<()> { let mut file = self.file.borrow_mut(); file.seek(std::io::SeekFrom::Start(pos as u64))?; { - let r = match c.as_ref() { - Completion::Read(r) => r, + let r = match c { + Completion::Read(ref r) => r, _ => unreachable!(), }; let mut buf = r.buf_mut(); @@ -72,12 +72,7 @@ impl File for GenericFile { Ok(()) } - fn pwrite( - &self, - pos: usize, - buffer: Rc>, - c: Rc, - ) -> Result<()> { + fn pwrite(&self, pos: usize, buffer: Rc>, c: Completion) -> Result<()> { let mut file = self.file.borrow_mut(); file.seek(std::io::SeekFrom::Start(pos as u64))?; let buf = buffer.borrow(); @@ -87,7 +82,7 @@ impl File for GenericFile { Ok(()) } - fn sync(&self, c: Rc) -> Result<()> { + fn sync(&self, c: Completion) -> Result<()> { let mut file = self.file.borrow_mut(); file.sync_all().map_err(|err| LimboError::IOError(err))?; c.complete(0); diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index eef5e523d..ebd1fcb61 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -19,7 +19,6 @@ enum UringIOError { IOUringCQError(i32), } -// Implement the Display trait to customize error messages impl fmt::Display for UringIOError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -39,7 +38,7 @@ pub struct UringIO { struct WrappedIOUring { ring: io_uring::IoUring, pending_ops: usize, - pub pending: [Option>; MAX_IOVECS as usize + 1], + pub pending: [Option; MAX_IOVECS as usize + 1], key: u64, } @@ -89,7 +88,7 @@ impl InnerUringIO { } impl WrappedIOUring { - fn submit_entry(&mut self, entry: &io_uring::squeue::Entry, c: Rc) { + fn submit_entry(&mut self, entry: &io_uring::squeue::Entry, c: Completion) { trace!("submit_entry({:?})", entry); self.pending[entry.get_user_data() as usize] = Some(c); unsafe { @@ -242,9 +241,9 @@ impl File for UringFile { Ok(()) } - fn pread(&self, pos: usize, c: Rc) -> Result<()> { - let r = match c.as_ref() { - Completion::Read(r) => r, + fn pread(&self, pos: usize, c: Completion) -> Result<()> { + let r = match c { + Completion::Read(ref r) => r, _ => unreachable!(), }; trace!("pread(pos = {}, length = {})", pos, r.buf().len()); @@ -264,12 +263,7 @@ impl File for UringFile { Ok(()) } - fn pwrite( - &self, - pos: usize, - buffer: Rc>, - c: Rc, - ) -> Result<()> { + fn pwrite(&self, pos: usize, buffer: Rc>, c: Completion) -> Result<()> { let mut io = self.io.borrow_mut(); let fd = io_uring::types::Fd(self.file.as_raw_fd()); let write = { @@ -285,7 +279,7 @@ impl File for UringFile { Ok(()) } - fn sync(&self, c: Rc) -> Result<()> { + fn sync(&self, c: Completion) -> Result<()> { let fd = io_uring::types::Fd(self.file.as_raw_fd()); let mut io = self.io.borrow_mut(); trace!("sync()"); diff --git a/core/io/memory.rs b/core/io/memory.rs index 18decf78a..164268d5e 100644 --- a/core/io/memory.rs +++ b/core/io/memory.rs @@ -78,9 +78,9 @@ impl File for MemoryFile { Ok(()) } - fn pread(&self, pos: usize, c: Rc) -> Result<()> { - let r = match &*c { - Completion::Read(r) => r, + fn pread(&self, pos: usize, c: Completion) -> Result<()> { + let r = match &c { + Completion::Read(ref r) => r, _ => unreachable!(), }; let buf_len = r.buf().len(); @@ -122,7 +122,7 @@ impl File for MemoryFile { Ok(()) } - fn pwrite(&self, pos: usize, buffer: Rc>, c: Rc) -> Result<()> { + fn pwrite(&self, pos: usize, buffer: Rc>, c: Completion) -> Result<()> { let buf = buffer.borrow(); let buf_len = buf.len(); if buf_len == 0 { @@ -159,7 +159,7 @@ impl File for MemoryFile { Ok(()) } - fn sync(&self, c: Rc) -> Result<()> { + fn sync(&self, c: Completion) -> Result<()> { // no-op c.complete(0); Ok(()) diff --git a/core/io/mod.rs b/core/io/mod.rs index f88a5d554..fa36e95be 100644 --- a/core/io/mod.rs +++ b/core/io/mod.rs @@ -12,9 +12,9 @@ use std::{ pub trait File { fn lock_file(&self, exclusive: bool) -> Result<()>; fn unlock_file(&self) -> Result<()>; - fn pread(&self, pos: usize, c: Rc) -> Result<()>; - fn pwrite(&self, pos: usize, buffer: Rc>, c: Rc) -> Result<()>; - fn sync(&self, c: Rc) -> Result<()>; + fn pread(&self, pos: usize, c: Completion) -> Result<()>; + fn pwrite(&self, pos: usize, buffer: Rc>, c: Completion) -> Result<()>; + fn sync(&self, c: Completion) -> Result<()>; fn size(&self) -> Result; } diff --git a/core/io/unix.rs b/core/io/unix.rs index effd94bf5..110e4546e 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -118,10 +118,10 @@ impl IO for UnixIO { } enum CompletionCallback { - Read(Rc>, Rc, usize), + Read(Rc>, Completion, usize), Write( Rc>, - Rc, + Completion, Rc>, usize, ), @@ -173,11 +173,11 @@ impl File for UnixFile { Ok(()) } - fn pread(&self, pos: usize, c: Rc) -> Result<()> { + fn pread(&self, pos: usize, c: Completion) -> Result<()> { let file = self.file.borrow(); let result = { - let r = match c.as_ref() { - Completion::Read(r) => r, + let r = match c { + Completion::Read(ref r) => r, _ => unreachable!(), }; let mut buf = r.buf_mut(); @@ -201,7 +201,7 @@ impl File for UnixFile { } self.callbacks.borrow_mut().insert( fd as usize, - CompletionCallback::Read(self.file.clone(), c.clone(), pos), + CompletionCallback::Read(self.file.clone(), c, pos), ); Ok(()) } @@ -209,12 +209,7 @@ impl File for UnixFile { } } - fn pwrite( - &self, - pos: usize, - buffer: Rc>, - c: Rc, - ) -> Result<()> { + fn pwrite(&self, pos: usize, buffer: Rc>, c: Completion) -> Result<()> { let file = self.file.borrow(); let result = { let buf = buffer.borrow(); @@ -238,7 +233,7 @@ impl File for UnixFile { } self.callbacks.borrow_mut().insert( fd as usize, - CompletionCallback::Write(self.file.clone(), c.clone(), buffer.clone(), pos), + CompletionCallback::Write(self.file.clone(), c, buffer.clone(), pos), ); Ok(()) } @@ -246,7 +241,7 @@ impl File for UnixFile { } } - fn sync(&self, c: Rc) -> Result<()> { + fn sync(&self, c: Completion) -> Result<()> { let file = self.file.borrow(); let result = fs::fsync(file.as_fd()); match result { diff --git a/core/io/windows.rs b/core/io/windows.rs index 50acfcb50..d359c4575 100644 --- a/core/io/windows.rs +++ b/core/io/windows.rs @@ -54,12 +54,12 @@ impl File for WindowsFile { unimplemented!() } - fn pread(&self, pos: usize, c: Rc) -> Result<()> { + fn pread(&self, pos: usize, c: Completion) -> Result<()> { let mut file = self.file.borrow_mut(); file.seek(std::io::SeekFrom::Start(pos as u64))?; { - let r = match c.as_ref() { - Completion::Read(r) => r, + let r = match c { + Completion::Read(ref r) => r, _ => unreachable!(), }; let mut buf = r.buf_mut(); @@ -70,12 +70,7 @@ impl File for WindowsFile { Ok(()) } - fn pwrite( - &self, - pos: usize, - buffer: Rc>, - c: Rc, - ) -> Result<()> { + fn pwrite(&self, pos: usize, buffer: Rc>, c: Completion) -> Result<()> { let mut file = self.file.borrow_mut(); file.seek(std::io::SeekFrom::Start(pos as u64))?; let buf = buffer.borrow(); @@ -85,7 +80,7 @@ impl File for WindowsFile { Ok(()) } - fn sync(&self, c: Rc) -> Result<()> { + fn sync(&self, c: Completion) -> Result<()> { let file = self.file.borrow_mut(); file.sync_all().map_err(LimboError::IOError)?; c.complete(0); diff --git a/core/lib.rs b/core/lib.rs index 94853aa80..3ff15f4fb 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -238,7 +238,7 @@ pub fn maybe_init_database_file(file: &Rc, io: &Arc) -> Result let completion = Completion::Write(WriteCompletion::new(Box::new(move |_| { *flag_complete.borrow_mut() = true; }))); - file.pwrite(0, contents.buffer.clone(), Rc::new(completion))?; + file.pwrite(0, contents.buffer.clone(), completion)?; } let mut limit = 100; loop { diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 4a963581b..3377d34cf 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2401,7 +2401,7 @@ mod tests { } let write_complete = Box::new(|_| {}); - let c = Rc::new(Completion::Write(WriteCompletion::new(write_complete))); + let c = Completion::Write(WriteCompletion::new(write_complete)); page_io.write_page(1, buf.clone(), c).unwrap(); let wal_shared = WalFileShared::open_shared(&io, "test.wal", page_size).unwrap(); @@ -2449,7 +2449,7 @@ mod tests { drop_fn, ))); let write_complete = Box::new(|_| {}); - let c = Rc::new(Completion::Write(WriteCompletion::new(write_complete))); + let c = Completion::Write(WriteCompletion::new(write_complete)); pager .page_io .write_page(current_page as usize, buf.clone(), c)?; diff --git a/core/storage/database.rs b/core/storage/database.rs index e59519f38..f9c20b972 100644 --- a/core/storage/database.rs +++ b/core/storage/database.rs @@ -7,14 +7,10 @@ use std::{cell::RefCell, rc::Rc}; /// the storage medium. A database can either be a file on disk, like in SQLite, /// or something like a remote page server service. pub trait DatabaseStorage { - fn read_page(&self, page_idx: usize, c: Rc) -> Result<()>; - fn write_page( - &self, - page_idx: usize, - buffer: Rc>, - c: Rc, - ) -> Result<()>; - fn sync(&self, c: Rc) -> Result<()>; + fn read_page(&self, page_idx: usize, c: Completion) -> Result<()>; + fn write_page(&self, page_idx: usize, buffer: Rc>, c: Completion) + -> Result<()>; + fn sync(&self, c: Completion) -> Result<()>; } #[cfg(feature = "fs")] @@ -24,9 +20,9 @@ pub struct FileStorage { #[cfg(feature = "fs")] impl DatabaseStorage for FileStorage { - fn read_page(&self, page_idx: usize, c: Rc) -> Result<()> { - let r = match c.as_ref() { - Completion::Read(r) => r, + fn read_page(&self, page_idx: usize, c: Completion) -> Result<()> { + let r = match c { + Completion::Read(ref r) => r, _ => unreachable!(), }; let size = r.buf().len(); @@ -43,7 +39,7 @@ impl DatabaseStorage for FileStorage { &self, page_idx: usize, buffer: Rc>, - c: Rc, + c: Completion, ) -> Result<()> { let buffer_size = buffer.borrow().len(); assert!(buffer_size >= 512); @@ -54,7 +50,7 @@ impl DatabaseStorage for FileStorage { Ok(()) } - fn sync(&self, c: Rc) -> Result<()> { + fn sync(&self, c: Completion) -> Result<()> { self.file.sync(c) } } diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 4963fa669..cb399f04f 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -253,8 +253,8 @@ pub fn begin_read_database_header( let header = header.clone(); finish_read_database_header(buf, header).unwrap(); }); - let c = Rc::new(Completion::Read(ReadCompletion::new(buf, complete))); - page_io.read_page(1, c.clone())?; + let c = Completion::Read(ReadCompletion::new(buf, complete)); + page_io.read_page(1, c)?; Ok(result) } @@ -313,7 +313,7 @@ pub fn begin_write_database_header(header: &DatabaseHeader, pager: &Pager) -> Re let drop_fn = Rc::new(|_buf| {}); let buf = Rc::new(RefCell::new(Buffer::allocate(512, drop_fn))); - let c = Rc::new(Completion::Read(ReadCompletion::new(buf, read_complete))); + let c = Completion::Read(ReadCompletion::new(buf, read_complete)); page_source.read_page(1, c)?; // run get header block pager.io.run_once()?; @@ -327,7 +327,7 @@ pub fn begin_write_database_header(header: &DatabaseHeader, pager: &Pager) -> Re // finish_read_database_header(buf, header).unwrap(); }); - let c = Rc::new(Completion::Write(WriteCompletion::new(write_complete))); + let c = Completion::Write(WriteCompletion::new(write_complete)); page_source.write_page(0, buffer_to_copy, c)?; Ok(()) @@ -675,8 +675,8 @@ pub fn begin_read_page( page.set_error(); } }); - let c = Rc::new(Completion::Read(ReadCompletion::new(buf, complete))); - page_io.read_page(page_idx, c.clone())?; + let c = Completion::Read(ReadCompletion::new(buf, complete)); + page_io.read_page(page_idx, c)?; Ok(()) } @@ -733,7 +733,7 @@ pub fn begin_write_btree_page( } }) }; - let c = Rc::new(Completion::Write(WriteCompletion::new(write_complete))); + let c = Completion::Write(WriteCompletion::new(write_complete)); page_source.write_page(page_id, buffer.clone(), c)?; Ok(()) } @@ -746,7 +746,7 @@ pub fn begin_sync(page_io: Rc, syncing: Rc>) *syncing.borrow_mut() = false; }), }); - page_io.sync(Rc::new(completion))?; + page_io.sync(completion)?; Ok(()) } @@ -1145,7 +1145,7 @@ pub fn begin_read_wal_header(io: &Rc) -> Result> let header = header.clone(); finish_read_wal_header(buf, header).unwrap(); }); - let c = Rc::new(Completion::Read(ReadCompletion::new(buf, complete))); + let c = Completion::Read(ReadCompletion::new(buf, complete)); io.pread(0, c)?; Ok(result) } @@ -1187,7 +1187,7 @@ pub fn begin_read_wal_frame( let frame = frame.clone(); finish_read_page(2, buf, frame).unwrap(); }); - let c = Rc::new(Completion::Read(ReadCompletion::new(buf, complete))); + let c = Completion::Read(ReadCompletion::new(buf, complete)); io.pread(offset, c)?; Ok(()) } @@ -1262,7 +1262,7 @@ pub fn begin_write_wal_frame( } }) }; - let c = Rc::new(Completion::Write(WriteCompletion::new(write_complete))); + let c = Completion::Write(WriteCompletion::new(write_complete)); io.pwrite(offset, buffer.clone(), c)?; Ok(checksums) } @@ -1295,7 +1295,7 @@ pub fn begin_write_wal_header(io: &Rc, header: &WalHeader) -> Result<( } }) }; - let c = Rc::new(Completion::Write(WriteCompletion::new(write_complete))); + let c = Completion::Write(WriteCompletion::new(write_complete)); io.pwrite(0, buffer.clone(), c)?; Ok(()) } diff --git a/core/storage/wal.rs b/core/storage/wal.rs index e42fde2b6..fc5f68845 100644 --- a/core/storage/wal.rs +++ b/core/storage/wal.rs @@ -630,7 +630,7 @@ impl Wal for WalFile { *syncing.borrow_mut() = false; }), }); - shared.file.sync(Rc::new(completion))?; + shared.file.sync(completion)?; } self.sync_state.replace(SyncState::Syncing); Ok(CheckpointStatus::IO) diff --git a/simulator/runner/file.rs b/simulator/runner/file.rs index 7a514ea1e..6d73af505 100644 --- a/simulator/runner/file.rs +++ b/simulator/runner/file.rs @@ -74,7 +74,7 @@ impl File for SimulatorFile { self.inner.unlock_file() } - fn pread(&self, pos: usize, c: Rc) -> Result<()> { + fn pread(&self, pos: usize, c: limbo_core::Completion) -> Result<()> { *self.nr_pread_calls.borrow_mut() += 1; if *self.fault.borrow() { *self.nr_pread_faults.borrow_mut() += 1; @@ -89,7 +89,7 @@ impl File for SimulatorFile { &self, pos: usize, buffer: Rc>, - c: Rc, + c: limbo_core::Completion, ) -> Result<()> { *self.nr_pwrite_calls.borrow_mut() += 1; if *self.fault.borrow() { @@ -101,7 +101,7 @@ impl File for SimulatorFile { self.inner.pwrite(pos, buffer, c) } - fn sync(&self, c: Rc) -> Result<()> { + fn sync(&self, c: limbo_core::Completion) -> Result<()> { *self.nr_sync_calls.borrow_mut() += 1; self.inner.sync(c) } From cd2d817c10cc90019e5d49ba9a93c7b446c01019 Mon Sep 17 00:00:00 2001 From: Gustavo Sverzut Barbieri Date: Sat, 8 Feb 2025 10:21:49 -0300 Subject: [PATCH 009/115] github.com/penberg/limbo was moved to github.com/tursodatabase/limbo Adjust all the references since they were broken --- CHANGELOG.md | 2 +- CONTRIBUTING.md | 2 +- bindings/python/pyproject.toml | 4 ++-- bindings/wasm/package.json | 2 +- core/benches/benchmark.rs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fee38e12c..847deec42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -303,7 +303,7 @@ - `ORDER BY` support for nullable sorting columns and qualified identifiers (Jussi Saurio) -- Fix `.schema` command crash in the CLI ([#212](https://github.com/penberg/limbo/issues/212) (Jussi Saurio) +- Fix `.schema` command crash in the CLI ([#212](https://github.com/tursodatabase/limbo/issues/212) (Jussi Saurio) ## 0.0.2 - 2024-07-24 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cad33a8db..ac4566c00 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ cargo bench --bench benchmark -- --profile-time=5 ## Finding things to work on -The issue tracker has issues tagged with [good first issue](https://github.com/penberg/limbo/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22), +The issue tracker has issues tagged with [good first issue](https://github.com/tursodatabase/limbo/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22), which are considered to be things to work on to get going. If you're interested in working on one of them, comment on the issue tracker, and we're happy to help you get going. ## Submitting your work diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 3dd269b0e..4ed6066ce 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -40,8 +40,8 @@ dev = [ ] [project.urls] -Homepage = "https://github.com/penberg/limbo" -Source = "https://github.com/penberg/limbo" +Homepage = "https://github.com/tursodatabase/limbo" +Source = "https://github.com/tursodatabase/limbo" [tool.maturin] bindings = 'pyo3' diff --git a/bindings/wasm/package.json b/bindings/wasm/package.json index 2265a7799..d7c799bb9 100644 --- a/bindings/wasm/package.json +++ b/bindings/wasm/package.json @@ -7,7 +7,7 @@ "license": "MIT", "repository": { "type": "git", - "url": "https://github.com/penberg/limbo" + "url": "https://github.com/tursodatabase/limbo" }, "type": "module", "main": "./node/dist/index.cjs", diff --git a/core/benches/benchmark.rs b/core/benches/benchmark.rs index 21b75424d..d2aef982b 100644 --- a/core/benches/benchmark.rs +++ b/core/benches/benchmark.rs @@ -12,7 +12,7 @@ fn rusqlite_open() -> rusqlite::Connection { } fn bench(criterion: &mut Criterion) { - // https://github.com/penberg/limbo/issues/174 + // https://github.com/tursodatabase/limbo/issues/174 // The rusqlite benchmark crashes on Mac M1 when using the flamegraph features let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err(); From 6308ce454413d226243a6c7fc79377c9e28c3b42 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sat, 8 Feb 2025 09:34:51 -0500 Subject: [PATCH 010/115] fix the shrinking file and poison errors --- simulator/main.rs | 10 +++++----- simulator/runner/execution.rs | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/simulator/main.rs b/simulator/main.rs index 82c39b809..b40934cfc 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -233,6 +233,11 @@ fn run_simulator( }) .collect::>(); + + // Write the shrunk plan to a file + let mut f = std::fs::File::create(&paths.shrunk_plan).unwrap(); + f.write_all(shrunk_plans[0].to_string().as_bytes()).unwrap(); + let last_execution = Arc::new(Mutex::new(*last_execution)); let shrunk = SandboxedResult::from( @@ -270,11 +275,6 @@ fn run_simulator( log::error!("shrinking failed, the error was not properly reproduced"); } } - - // Write the shrunk plan to a file - let shrunk_plan = std::fs::read(&paths.shrunk_plan).unwrap(); - let mut f = std::fs::File::create(&paths.shrunk_plan).unwrap(); - f.write_all(&shrunk_plan).unwrap(); } } } diff --git a/simulator/runner/execution.rs b/simulator/runner/execution.rs index 6544928a1..6342dff3a 100644 --- a/simulator/runner/execution.rs +++ b/simulator/runner/execution.rs @@ -62,6 +62,7 @@ pub(crate) fn execute_plans( ) -> ExecutionResult { let mut history = ExecutionHistory::new(); let now = std::time::Instant::now(); + env.clear_poison(); let mut env = env.lock().unwrap(); for _tick in 0..env.opts.ticks { // Pick the connection to interact with From 4362bc16a3876aed31e21056d31529e1fb7f6d8c Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sat, 8 Feb 2025 09:37:08 -0500 Subject: [PATCH 011/115] fix formatting --- simulator/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/simulator/main.rs b/simulator/main.rs index b40934cfc..2eb463529 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -233,7 +233,6 @@ fn run_simulator( }) .collect::>(); - // Write the shrunk plan to a file let mut f = std::fs::File::create(&paths.shrunk_plan).unwrap(); f.write_all(shrunk_plans[0].to_string().as_bytes()).unwrap(); From 3ae3e650aed41823767fdbd1f3924f2fcc3063f4 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sat, 8 Feb 2025 10:59:11 -0500 Subject: [PATCH 012/115] fix watch mode bug deleting the last interaction of a property --- simulator/generation/plan.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index d06cdfff3..b601a8975 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -57,8 +57,11 @@ impl InteractionPlan { i += 1; continue; } - - if interactions[i].contains(plan[j1][j2].to_string().as_str()) { + if plan[j1].len() == j2 { + i += 1; + j1 += 1; + j2 = 0; + } else if interactions[i].contains(plan[j1][j2].to_string().as_str()) { i += 1; if j2 + 1 < plan[j1].len() { j2 += 1; From e9046fef7818479f2c101ea6b1f2c5bde58d86ab Mon Sep 17 00:00:00 2001 From: wyhaya Date: Sun, 9 Feb 2025 00:01:07 +0800 Subject: [PATCH 013/115] cli: Improve pretty mode table --- Cargo.lock | 70 +++++++++++++++++++++++--------------- cli/Cargo.toml | 2 +- cli/app.rs | 91 ++++++++++++++++++-------------------------------- 3 files changed, 77 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 819311184..3d610591d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -399,29 +399,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" -[[package]] -name = "cli-table" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b53f9241f288a7b12c56565f04aaeaeeab6b8923d42d99255d4ca428b4d97f89" -dependencies = [ - "cli-table-derive", - "csv", - "termcolor", - "unicode-width", -] - -[[package]] -name = "cli-table-derive" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e83a93253aaae7c74eb7428ce4faa6e219ba94886908048888701819f82fb94" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "clipboard-win" version = "4.5.0" @@ -449,6 +426,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "comfy-table" +version = "7.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" +dependencies = [ + "crossterm", + "unicode-segmentation", + "unicode-width 0.2.0", +] + [[package]] name = "comma" version = "1.0.0" @@ -595,6 +583,28 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.8.0", + "crossterm_winapi", + "parking_lot", + "rustix", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -1564,7 +1574,7 @@ version = "0.0.14" dependencies = [ "anyhow", "clap", - "cli-table", + "comfy-table", "csv", "ctrlc", "dirs", @@ -1848,7 +1858,7 @@ dependencies = [ "terminal_size", "textwrap", "thiserror 1.0.69", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -2704,7 +2714,7 @@ dependencies = [ "radix_trie", "scopeguard", "unicode-segmentation", - "unicode-width", + "unicode-width 0.1.14", "utf8parse", "winapi", ] @@ -3033,7 +3043,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" dependencies = [ "unicode-linebreak", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -3201,6 +3211,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unindent" version = "0.2.3" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 1886627e4..03857adb2 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -21,7 +21,7 @@ path = "main.rs" [dependencies] anyhow = "1.0.75" clap = { version = "4.5", features = ["derive"] } -cli-table = "0.4.7" +comfy-table = "7.1.4" dirs = "5.0.1" env_logger = "0.10.1" limbo_core = { path = "../core" } diff --git a/cli/app.rs b/cli/app.rs index 7fa7b70f0..45dd14896 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -2,8 +2,7 @@ use crate::{ import::{ImportFile, IMPORT_HELP}, opcodes_dictionary::OPCODE_DESCRIPTIONS, }; -use cli_table::format::{Border, HorizontalLine, Separator, VerticalLine}; -use cli_table::{Cell, Style, Table}; +use comfy_table::{Attribute, Cell, CellAlignment, ContentArrangement, Row, Table}; use limbo_core::{Database, LimboError, Statement, StepResult, Value}; use clap::{Parser, ValueEnum}; @@ -670,35 +669,42 @@ impl Limbo { println!("Query interrupted."); return Ok(()); } - let mut table_rows: Vec> = vec![]; + let mut table = Table::new(); + table + .set_content_arrangement(ContentArrangement::Dynamic) + .set_truncation_indicator("…") + .apply_modifier("││──├─┼┤│─┼├┤┬┴┌┐└┘"); if rows.num_columns() > 0 { - let columns = (0..rows.num_columns()) + let header = (0..rows.num_columns()) .map(|i| { - rows.get_column_name(i) - .map(|name| name.cell().bold(true)) - .unwrap_or_else(|| " ".cell()) + let name = rows.get_column_name(i).cloned().unwrap_or_default(); + Cell::new(name).add_attribute(Attribute::Bold) }) .collect::>(); - table_rows.push(columns); + table.set_header(header); } loop { match rows.step() { Ok(StepResult::Row) => { - let row = rows.row().unwrap(); - table_rows.push( - row.values - .iter() - .map(|value| match value.to_value() { - Value::Null => self.opts.null_value.clone().cell(), - Value::Integer(i) => i.to_string().cell(), - Value::Float(f) => f.to_string().cell(), - Value::Text(s) => s.cell(), - Value::Blob(b) => { - format!("{}", String::from_utf8_lossy(b)).cell() - } - }) - .collect(), - ); + let record = rows.row().unwrap(); + let mut row = Row::new(); + row.max_height(1); + for value in &record.values { + let (content, alignment) = match value.to_value() { + Value::Null => { + (self.opts.null_value.clone(), CellAlignment::Left) + } + Value::Integer(i) => (i.to_string(), CellAlignment::Right), + Value::Float(f) => (f.to_string(), CellAlignment::Right), + Value::Text(s) => (s.to_string(), CellAlignment::Left), + Value::Blob(b) => ( + String::from_utf8_lossy(b).to_string(), + CellAlignment::Left, + ), + }; + row.add_cell(Cell::new(content).set_alignment(alignment)); + } + table.add_row(row); } Ok(StepResult::IO) => { self.io.run_once()?; @@ -718,7 +724,10 @@ impl Limbo { } } } - self.print_table(table_rows); + + if table.header().is_some() { + let _ = self.write_fmt(format_args!("{}", table)); + } } }, Ok(None) => {} @@ -734,40 +743,6 @@ impl Limbo { Ok(()) } - fn print_table(&mut self, table_rows: Vec>) { - if table_rows.is_empty() { - return; - } - - let horizontal_line = HorizontalLine::new('┌', '┐', '┬', '─'); - let horizontal_line_mid = HorizontalLine::new('├', '┤', '┼', '─'); - let horizontal_line_bottom = HorizontalLine::new('└', '┘', '┴', '─'); - let vertical_line = VerticalLine::new('│'); - - let border = Border::builder() - .top(horizontal_line) - .bottom(horizontal_line_bottom) - .left(vertical_line.clone()) - .right(vertical_line.clone()) - .build(); - - let separator = Separator::builder() - .column(Some(vertical_line)) - .row(Some(horizontal_line_mid)) - .build(); - - if let Ok(table) = table_rows - .table() - .border(border) - .separator(separator) - .display() - { - let _ = self.write_fmt(format_args!("{}", table)); - } else { - let _ = self.writeln("Error displaying table."); - } - } - fn display_schema(&mut self, table: Option<&str>) -> anyhow::Result<()> { let sql = match table { Some(table_name) => format!( From 670dac59399ed17006c77ceb54ca5c391ab12f35 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 18:04:39 +0200 Subject: [PATCH 014/115] sqlite3-parser: box the where clause in Delete --- core/translate/delete.rs | 11 ++++++++--- vendored/sqlite3-parser/src/parser/ast/mod.rs | 2 +- vendored/sqlite3-parser/src/parser/parse.y | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/core/translate/delete.rs b/core/translate/delete.rs index ffad33d73..b5ce85fdc 100644 --- a/core/translate/delete.rs +++ b/core/translate/delete.rs @@ -13,7 +13,7 @@ pub fn translate_delete( query_mode: QueryMode, schema: &Schema, tbl_name: &QualifiedName, - where_clause: Option, + where_clause: Option>, limit: Option>, syms: &SymbolTable, ) -> Result { @@ -35,7 +35,7 @@ pub fn translate_delete( pub fn prepare_delete_plan( schema: &Schema, tbl_name: &QualifiedName, - where_clause: Option, + where_clause: Option>, limit: Option>, ) -> Result { let table = match schema.get_table(tbl_name.name.0.as_str()) { @@ -53,7 +53,12 @@ pub fn prepare_delete_plan( let mut where_predicates = vec![]; // Parse the WHERE clause - parse_where(where_clause, &table_references, None, &mut where_predicates)?; + parse_where( + where_clause.map(|e| *e), + &table_references, + None, + &mut where_predicates, + )?; // Parse the LIMIT/OFFSET clause let (resolved_limit, resolved_offset) = limit.map_or(Ok((None, None)), |l| parse_limit(*l))?; diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index f149322b3..077f2b6ac 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -168,7 +168,7 @@ pub enum Stmt { /// `INDEXED` indexed: Option, /// `WHERE` clause - where_clause: Option, + where_clause: Option>, /// `RETURNING` returning: Option>, /// `ORDER BY` diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 226dca5a7..76c053964 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -761,13 +761,13 @@ limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). cmd ::= with(C) DELETE FROM xfullname(X) indexed_opt(I) where_opt_ret(W) orderby_opt(O) limit_opt(L). { let (where_clause, returning) = W; - self.ctx.stmt = Some(Stmt::Delete{ with: C, tbl_name: X, indexed: I, where_clause, returning, + self.ctx.stmt = Some(Stmt::Delete{ with: C, tbl_name: X, indexed: I, where_clause: where_clause.map(Box::new), returning, order_by: O, limit: L }); } %else cmd ::= with(C) DELETE FROM xfullname(X) indexed_opt(I) where_opt_ret(W). { let (where_clause, returning) = W; - self.ctx.stmt = Some(Stmt::Delete{ with: C, tbl_name: X, indexed: I, where_clause, returning, + self.ctx.stmt = Some(Stmt::Delete{ with: C, tbl_name: X, indexed: I, where_clause: where_clause.map(Box::new), returning, order_by: None, limit: None }); } %endif From 0dba39b025f82905fca2d3e504f999450d2b55e8 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 18:05:13 +0200 Subject: [PATCH 015/115] sqlite3-parser: box everything in Attach --- vendored/sqlite3-parser/src/parser/ast/mod.rs | 6 +++--- vendored/sqlite3-parser/src/parser/parse.y | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 077f2b6ac..a60d2a592 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -78,11 +78,11 @@ pub enum Stmt { Attach { /// filename // TODO distinction between ATTACH and ATTACH DATABASE - expr: Expr, + expr: Box, /// schema name - db_name: Expr, + db_name: Box, /// password - key: Option, + key: Option>, }, /// `BEGIN`: tx type, tx name Begin(Option, Option), diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 76c053964..0750af969 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -1276,7 +1276,7 @@ cmd ::= DROP TRIGGER ifexists(NOERR) fullname(X). { //////////////////////// ATTACH DATABASE file AS name ///////////////////////// %ifndef SQLITE_OMIT_ATTACH cmd ::= ATTACH database_kw_opt expr(F) AS expr(D) key_opt(K). { - self.ctx.stmt = Some(Stmt::Attach{ expr: F, db_name: D, key: K }); + self.ctx.stmt = Some(Stmt::Attach{ expr: Box::new(F), db_name: Box::new(D), key: K.map(Box::new) }); } cmd ::= DETACH database_kw_opt expr(D). { self.ctx.stmt = Some(Stmt::Detach(D)); From f341474fee3c76869fd49e8e6f43ebf7234c0e86 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 18:05:49 +0200 Subject: [PATCH 016/115] sqlite3-parser: box large members of CreateTrigger --- vendored/sqlite3-parser/src/parser/ast/mod.rs | 4 ++-- vendored/sqlite3-parser/src/parser/parse.y | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index a60d2a592..91f5633e3 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -125,13 +125,13 @@ pub enum Stmt { /// `BEFORE`/`AFTER`/`INSTEAD OF` time: Option, /// `DELETE`/`INSERT`/`UPDATE` - event: TriggerEvent, + event: Box, /// table name tbl_name: QualifiedName, /// `FOR EACH ROW` for_each_row: bool, /// `WHEN` - when_clause: Option, + when_clause: Option>, /// statements commands: Vec, }, diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 0750af969..694bb4216 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -1167,8 +1167,8 @@ minus_num(A) ::= MINUS number(X). {A = Expr::unary(UnaryOperator::Negative, cmd ::= createkw temp(T) TRIGGER ifnotexists(NOERR) fullname(B) trigger_time(C) trigger_event(D) ON fullname(E) foreach_clause(X) when_clause(G) BEGIN trigger_cmd_list(S) END. { self.ctx.stmt = Some(Stmt::CreateTrigger{ - temporary: T, if_not_exists: NOERR, trigger_name: B, time: C, event: D, tbl_name: E, - for_each_row: X, when_clause: G, commands: S + temporary: T, if_not_exists: NOERR, trigger_name: B, time: C, event: Box::new(D), tbl_name: E, + for_each_row: X, when_clause: G.map(Box::new), commands: S }); } From ac7f9d67b7c62d41cb43491128dffa0b8bc04ef7 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 18:06:45 +0200 Subject: [PATCH 017/115] sqlite3-parser: box large members of Upsert --- vendored/sqlite3-parser/src/parser/ast/mod.rs | 4 ++-- vendored/sqlite3-parser/src/parser/parse.y | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 91f5633e3..ffeafea07 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -1769,9 +1769,9 @@ pub enum TransactionType { #[derive(Clone, Debug, PartialEq, Eq)] pub struct Upsert { /// conflict targets - pub index: Option, + pub index: Option>, /// `DO` clause - pub do_clause: UpsertDo, + pub do_clause: Box, /// next upsert pub next: Option>, } diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 694bb4216..af16eefa2 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -851,16 +851,16 @@ upsert(A) ::= ON CONFLICT LP sortlist(T) RP where_opt(TW) { let index = UpsertIndex{ targets: T, where_clause: TW }; let do_clause = UpsertDo::Set{ sets: Z, where_clause: W }; let (next, returning) = N; - A = (Some(Upsert{ index: Some(index), do_clause, next: next.map(Box::new) }), returning);} + A = (Some(Upsert{ index: Some(Box::new(index)), do_clause: Box::new(do_clause), next: next.map(Box::new) }), returning);} upsert(A) ::= ON CONFLICT LP sortlist(T) RP where_opt(TW) DO NOTHING upsert(N). { let index = UpsertIndex{ targets: T, where_clause: TW }; let (next, returning) = N; - A = (Some(Upsert{ index: Some(index), do_clause: UpsertDo::Nothing, next: next.map(Box::new) }), returning); } + A = (Some(Upsert{ index: Some(Box::new(index)), do_clause: Box::new(UpsertDo::Nothing), next: next.map(Box::new) }), returning); } upsert(A) ::= ON CONFLICT DO NOTHING returning(R). - { A = (Some(Upsert{ index: None, do_clause: UpsertDo::Nothing, next: None }), R); } + { A = (Some(Upsert{ index: None, do_clause: Box::new(UpsertDo::Nothing), next: None }), R); } upsert(A) ::= ON CONFLICT DO UPDATE SET setlist(Z) where_opt(W) returning(R). { let do_clause = UpsertDo::Set{ sets: Z, where_clause: W }; - A = (Some(Upsert{ index: None, do_clause, next: None }), R);} + A = (Some(Upsert{ index: None, do_clause: Box::new(do_clause), next: None }), R);} %type returning {Option>} returning(A) ::= RETURNING selcollist(X). {A = Some(X);} From 74262042040ffef407ffe516c11d967fde89ef32 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 18:07:38 +0200 Subject: [PATCH 018/115] sqlite3-parser: box Following and Preceding in FrameBound --- vendored/sqlite3-parser/src/parser/ast/mod.rs | 4 ++-- vendored/sqlite3-parser/src/parser/parse.y | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index ffeafea07..f8534e9a3 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -1872,9 +1872,9 @@ pub enum FrameBound { /// `CURRENT ROW` CurrentRow, /// `FOLLOWING` - Following(Expr), + Following(Box), /// `PRECEDING` - Preceding(Expr), + Preceding(Box), /// `UNBOUNDED FOLLOWING` UnboundedFollowing, /// `UNBOUNDED PRECEDING` diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index af16eefa2..0f17415e4 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -1454,9 +1454,9 @@ frame_bound_s(A) ::= UNBOUNDED PRECEDING. {A = FrameBound::UnboundedPreceding;} frame_bound_e(A) ::= frame_bound(X). {A = X;} frame_bound_e(A) ::= UNBOUNDED FOLLOWING. {A = FrameBound::UnboundedFollowing;} -frame_bound(A) ::= expr(X) PRECEDING. { A = FrameBound::Preceding(X); } +frame_bound(A) ::= expr(X) PRECEDING. { A = FrameBound::Preceding(Box::new(X)); } frame_bound(A) ::= CURRENT ROW. { A = FrameBound::CurrentRow; } -frame_bound(A) ::= expr(X) FOLLOWING. { A = FrameBound::Following(X); } +frame_bound(A) ::= expr(X) FOLLOWING. { A = FrameBound::Following(Box::new(X)); } %type frame_exclude_opt {Option} frame_exclude_opt(A) ::= . {A = None;} From 2a82091cb38be49a28ccb8c4fe57df2d464d3a73 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 18:08:31 +0200 Subject: [PATCH 019/115] sqlite3-parser: box the where clause in Update --- vendored/sqlite3-parser/src/parser/ast/mod.rs | 2 +- vendored/sqlite3-parser/src/parser/parse.y | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index f8534e9a3..b9e662683 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -256,7 +256,7 @@ pub enum Stmt { /// `FROM` from: Option, /// `WHERE` clause - where_clause: Option, + where_clause: Option>, /// `RETURNING` returning: Option>, /// `ORDER BY` diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 0f17415e4..7fb578158 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -791,14 +791,14 @@ cmd ::= with(C) UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y) from where_opt_ret(W) orderby_opt(O) limit_opt(L). { let (where_clause, returning) = W; self.ctx.stmt = Some(Stmt::Update { with: C, or_conflict: R, tbl_name: X, indexed: I, sets: Y, from: F, - where_clause, returning, order_by: O, limit: L }); + where_clause: where_clause.map(Box::new), returning, order_by: O, limit: L }); } %else cmd ::= with(C) UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y) from(F) where_opt_ret(W). { let (where_clause, returning) = W; self.ctx.stmt = Some(Stmt::Update { with: C, or_conflict: R, tbl_name: X, indexed: I, sets: Y, from: F, - where_clause, returning, order_by: None, limit: None }); + where_clause: where_clause.map(Box::new), returning, order_by: None, limit: None }); } %endif From 781aa3b5d61d919d429e877957285abc2fc08565 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 18:08:54 +0200 Subject: [PATCH 020/115] sqlite3-parser: box the having clause in GroupBy --- core/translate/select.rs | 2 +- vendored/sqlite3-parser/src/parser/ast/mod.rs | 2 +- vendored/sqlite3-parser/src/parser/parse.y | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/translate/select.rs b/core/translate/select.rs index 2a055afd2..b5091af64 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -305,7 +305,7 @@ pub fn prepare_select_plan( exprs: group_by.exprs, having: if let Some(having) = group_by.having { let mut predicates = vec![]; - break_predicate_at_and_boundaries(having, &mut predicates); + break_predicate_at_and_boundaries(*having, &mut predicates); for expr in predicates.iter_mut() { bind_column_references( expr, diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index b9e662683..43f2c5dfb 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -1005,7 +1005,7 @@ pub struct GroupBy { /// expressions pub exprs: Vec, /// `HAVING` - pub having: Option, // HAVING clause on a non-aggregate query + pub having: Option>, // HAVING clause on a non-aggregate query } /// identifier or one of several keywords or `INDEXED` diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 7fb578158..6402a6d5d 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -731,7 +731,7 @@ nulls(A) ::= . {A = None;} %type groupby_opt {Option} groupby_opt(A) ::= . {A = None;} -groupby_opt(A) ::= GROUP BY nexprlist(X) having_opt(Y). {A = Some(GroupBy{ exprs: X, having: Y });} +groupby_opt(A) ::= GROUP BY nexprlist(X) having_opt(Y). {A = Some(GroupBy{ exprs: X, having: Y.map(Box::new) });} %type having_opt {Option} having_opt(A) ::= . {A = None;} From 4faadd86b0af548d3d176181f7dac516ea45acde Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 18:09:58 +0200 Subject: [PATCH 021/115] sqlite3-parser: box the InsertBody --- .../sqlite3-parser/src/parser/ast/check.rs | 20 +++++++++---------- vendored/sqlite3-parser/src/parser/ast/mod.rs | 2 +- vendored/sqlite3-parser/src/parser/parse.y | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/check.rs b/vendored/sqlite3-parser/src/parser/ast/check.rs index e1e0eecd3..ca1e8cb55 100644 --- a/vendored/sqlite3-parser/src/parser/ast/check.rs +++ b/vendored/sqlite3-parser/src/parser/ast/check.rs @@ -160,19 +160,19 @@ impl Stmt { } => Err(custom_err!("ORDER BY without LIMIT on DELETE")), Self::Insert { columns: Some(columns), - body: InsertBody::Select(select, ..), + body, .. - } => match select.body.select.column_count() { - ColumnCount::Fixed(n) if n != columns.len() => { - Err(custom_err!("{} values for {} columns", n, columns.len())) + } => match &**body { + InsertBody::Select(select, ..) => match select.body.select.column_count() { + ColumnCount::Fixed(n) if n != columns.len() => { + Err(custom_err!("{} values for {} columns", n, columns.len())) + } + _ => Ok(()), + }, + InsertBody::DefaultValues => { + Err(custom_err!("0 values for {} columns", columns.len())) } - _ => Ok(()), }, - Self::Insert { - columns: Some(columns), - body: InsertBody::DefaultValues, - .. - } => Err(custom_err!("0 values for {} columns", columns.len())), Self::Update { order_by: Some(_), limit: None, diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 43f2c5dfb..8cf4f805d 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -217,7 +217,7 @@ pub enum Stmt { /// `COLUMNS` columns: Option, /// `VALUES` or `SELECT` - body: InsertBody, + body: Box, /// `RETURNING` returning: Option>, }, diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 6402a6d5d..bb3c80e91 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -828,13 +828,13 @@ cmd ::= with(W) insert_cmd(R) INTO xfullname(X) idlist_opt(F) select(S) let (upsert, returning) = U; let body = InsertBody::Select(Box::new(S), upsert); self.ctx.stmt = Some(Stmt::Insert{ with: W, or_conflict: R, tbl_name: X, columns: F, - body, returning }); + body: Box::new(body), returning }); } cmd ::= with(W) insert_cmd(R) INTO xfullname(X) idlist_opt(F) DEFAULT VALUES returning(Y). { let body = InsertBody::DefaultValues; self.ctx.stmt = Some(Stmt::Insert{ with: W, or_conflict: R, tbl_name: X, columns: F, - body, returning: Y }); + body: Box::new(body), returning: Y }); } %type upsert {(Option, Option>)} From 69d72da837d0668e93f671a6692f63546c97c064 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sat, 8 Feb 2025 12:53:25 -0500 Subject: [PATCH 022/115] fix the diff computing algorithm --- simulator/generation/plan.rs | 48 ++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index b601a8975..ce24e0693 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -47,9 +47,9 @@ impl InteractionPlan { .map(|i| i.interactions()) .collect::>(); - let (mut i, mut j1, mut j2) = (0, 0, 0); + let (mut i, mut j) = (0, 0); - while i < interactions.len() && j1 < plan.len() { + while i < interactions.len() && j < plan.len() { if interactions[i].starts_with("-- begin") || interactions[i].starts_with("-- end") || interactions[i].is_empty() @@ -57,32 +57,32 @@ impl InteractionPlan { i += 1; continue; } - if plan[j1].len() == j2 { - i += 1; - j1 += 1; - j2 = 0; - } else if interactions[i].contains(plan[j1][j2].to_string().as_str()) { - i += 1; - if j2 + 1 < plan[j1].len() { - j2 += 1; - } else { - j1 += 1; - j2 = 0; - } - } else { - plan[j1].remove(j2); - if plan[j1].is_empty() { - plan.remove(j1); - j2 = 0; + // interactions[i] is the i'th line in the human readable plan + // plan[j][k] is the k'th interaction in the j'th property + let mut k = 0; + + while k < plan[j].len() { + + if i >= interactions.len() { + let _ = plan.split_off(j + 1); + let _ = plan[j].split_off(k); + break; + } + + if interactions[i].contains(plan[j][k].to_string().as_str()) { + i += 1; + k += 1; + } else { + plan[j].remove(k); } } - } - if j1 < plan.len() { - if j2 < plan[j1].len() { - let _ = plan[j1].split_off(j2); + + if plan[j].is_empty() { + plan.remove(j); + } else { + j += 1; } - let _ = plan.split_off(j1); } plan From 8582a870fd9f9b72ccd612031ee40d8693e8bff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sat, 8 Feb 2025 21:12:56 +0100 Subject: [PATCH 023/115] core: make "result" module public, because it is needed to implement trait Wal, which is public --- core/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib.rs b/core/lib.rs index 94853aa80..17659fae7 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -9,7 +9,7 @@ mod json; pub mod mvcc; mod parameters; mod pseudo; -mod result; +pub mod result; mod schema; mod storage; mod translate; From 9aedbf2d45c1826470380bda07b334f90689043a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sat, 8 Feb 2025 21:20:51 +0100 Subject: [PATCH 024/115] core: make storage::pager::PageRef public, because it is needed to implement trait Wal --- core/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/lib.rs b/core/lib.rs index 17659fae7..956a77a9f 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -42,6 +42,7 @@ use storage::btree::btree_init_page; use storage::database::FileStorage; use storage::page_cache::DumbLruPageCache; use storage::pager::allocate_page; +pub use storage::pager::PageRef; use storage::sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE}; pub use storage::wal::WalFile; pub use storage::wal::WalFileShared; From d7a6d48d8c72b248d6d7e65628bced10c3fb28c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sat, 8 Feb 2025 21:21:07 +0100 Subject: [PATCH 025/115] core: make storage::wal::CheckPointMode public, because it is needed to implement trait Wal --- core/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/lib.rs b/core/lib.rs index 956a77a9f..aa5f90543 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -44,6 +44,7 @@ use storage::page_cache::DumbLruPageCache; use storage::pager::allocate_page; pub use storage::pager::PageRef; use storage::sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE}; +pub use storage::wal::CheckpointMode; pub use storage::wal::WalFile; pub use storage::wal::WalFileShared; use types::OwnedValue; From 7ddbcf07afe49821b1235be2f40e1a294d62fd54 Mon Sep 17 00:00:00 2001 From: alpaylan Date: Sat, 8 Feb 2025 15:41:57 -0500 Subject: [PATCH 026/115] fix formatting --- simulator/generation/plan.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index ce24e0693..cf41c1060 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -63,7 +63,6 @@ impl InteractionPlan { let mut k = 0; while k < plan[j].len() { - if i >= interactions.len() { let _ = plan.split_off(j + 1); let _ = plan[j].split_off(k); From be5ea350bba83bb9122fc7223cced0885a2b8380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sun, 9 Feb 2025 01:10:35 +0100 Subject: [PATCH 027/115] bindings: select io_uring feature from limbo_core explicitly as it will be made non-default --- bindings/go/Cargo.toml | 4 ++-- bindings/java/Cargo.toml | 2 +- bindings/python/Cargo.toml | 2 +- bindings/rust/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bindings/go/Cargo.toml b/bindings/go/Cargo.toml index b73902c5f..e268d0a27 100644 --- a/bindings/go/Cargo.toml +++ b/bindings/go/Cargo.toml @@ -17,7 +17,7 @@ io_uring = ["limbo_core/io_uring"] [dependencies] -limbo_core = { path = "../../core/" } +limbo_core = { path = "../../core" } [target.'cfg(target_os = "linux")'.dependencies] -limbo_core = { path = "../../core/", features = ["io_uring"] } +limbo_core = { path = "../../core", features = ["io_uring"] } diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 9b78b1597..fd38d1e65 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -12,6 +12,6 @@ crate-type = ["cdylib"] path = "rs_src/lib.rs" [dependencies] -limbo_core = { path = "../../core" } +limbo_core = { path = "../../core", features = ["io_uring"] } jni = "0.21.1" thiserror = "2.0.9" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 6157fa579..43639f94c 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -16,7 +16,7 @@ extension-module = ["pyo3/extension-module"] [dependencies] anyhow = "1.0" -limbo_core = { path = "../../core" } +limbo_core = { path = "../../core", features = ["io_uring"] } pyo3 = { version = "0.22.4", features = ["anyhow"] } [build-dependencies] diff --git a/bindings/rust/Cargo.toml b/bindings/rust/Cargo.toml index bdf101c56..a5c3a3829 100644 --- a/bindings/rust/Cargo.toml +++ b/bindings/rust/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true repository.workspace = true [dependencies] -limbo_core = { path = "../../core" } +limbo_core = { path = "../../core", features = ["io_uring"] } thiserror = "2.0.9" [dev-dependencies] From 62dea0b12b275ac74faa032b1e2966d2dd74dbb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sun, 9 Feb 2025 01:11:00 +0100 Subject: [PATCH 028/115] cli: select io_uring feature by default --- cli/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 1886627e4..7464b36a5 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -31,4 +31,5 @@ csv = "1.3.1" miette = { version = "7.4.0", features = ["fancy"] } [features] +default = ["io_uring"] io_uring = ["limbo_core/io_uring"] From c07c08aa988f8c85e616a15f23b9b36867905e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sun, 9 Feb 2025 01:12:27 +0100 Subject: [PATCH 029/115] core: make io_uring feature non-default. All crates that depend on core select it by default. This enables us to build CLI without io_uring, which before this commit would still have used io_uring in core. --- core/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index 687f4ff19..d13591aee 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,7 +14,7 @@ name = "limbo_core" path = "lib.rs" [features] -default = ["fs", "json", "uuid", "io_uring", "time"] +default = ["fs", "json", "uuid", "time"] fs = [] json = [ "dep:jsonb", From 83b158fb3a2bcb2ab2663dfb1d838a69fbcc0eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jorge=20L=C3=B3pez?= Date: Sun, 9 Feb 2025 01:13:12 +0100 Subject: [PATCH 030/115] core: silence some unused warnings when building without default features --- core/ext/mod.rs | 1 + core/io/mod.rs | 3 +++ core/io/unix.rs | 1 + core/storage/database.rs | 4 +++- core/translate/expr.rs | 1 + 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/core/ext/mod.rs b/core/ext/mod.rs index 67fd78491..b6fb67fbb 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -153,6 +153,7 @@ impl Database { } pub fn register_builtins(&self) -> Result<(), String> { + #[allow(unused_variables)] let ext_api = self.build_limbo_ext(); #[cfg(feature = "uuid")] if unsafe { !limbo_uuid::register_extension_static(&ext_api).is_ok() } { diff --git a/core/io/mod.rs b/core/io/mod.rs index f88a5d554..50b4551bf 100644 --- a/core/io/mod.rs +++ b/core/io/mod.rs @@ -166,14 +166,17 @@ impl Buffer { cfg_block! { #[cfg(all(target_os = "linux", feature = "io_uring"))] { mod io_uring; + #[cfg(feature = "fs")] pub use io_uring::UringIO; mod unix; + #[cfg(feature = "fs")] pub use unix::UnixIO; pub use io_uring::UringIO as PlatformIO; } #[cfg(any(all(target_os = "linux",not(feature = "io_uring")), target_os = "macos"))] { mod unix; + #[cfg(feature = "fs")] pub use unix::UnixIO; pub use unix::UnixIO as PlatformIO; } diff --git a/core/io/unix.rs b/core/io/unix.rs index effd94bf5..d06b71770 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -22,6 +22,7 @@ pub struct UnixIO { } impl UnixIO { + #[cfg(feature = "fs")] pub fn new() -> Result { debug!("Using IO backend 'syscall'"); Ok(Self { diff --git a/core/storage/database.rs b/core/storage/database.rs index e59519f38..7e30c9215 100644 --- a/core/storage/database.rs +++ b/core/storage/database.rs @@ -1,4 +1,6 @@ -use crate::{error::LimboError, io::Completion, Buffer, Result}; +#[cfg(feature = "fs")] +use crate::error::LimboError; +use crate::{io::Completion, Buffer, Result}; use std::{cell::RefCell, rc::Rc}; /// DatabaseStorage is an interface a database file that consists of pages. diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 8ddb580df..36acfa935 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -159,6 +159,7 @@ macro_rules! expect_arguments_min { }}; } +#[allow(unused_macros)] macro_rules! expect_arguments_even { ( $args:expr, From 79e2fba42467372aabf8b52501781cbe7a899a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 17:17:47 +0900 Subject: [PATCH 031/115] Implement minor methods/features in JDBC4Connection.java --- .../tursodatabase/jdbc4/JDBC4Connection.java | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index d5b73e627..34287f0f9 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -10,6 +10,8 @@ import org.github.tursodatabase.core.LimboConnection; public class JDBC4Connection extends LimboConnection { + private Map> typeMap = new HashMap<>(); + public JDBC4Connection(String url, String filePath) throws SQLException { super(url, filePath); } @@ -45,17 +47,8 @@ public class JDBC4Connection extends LimboConnection { } @Override - @SkipNullableCheck - public CallableStatement prepareCall(String sql) throws SQLException { - // TODO - return null; - } - - @Override - @SkipNullableCheck public String nativeSQL(String sql) throws SQLException { - // TODO - return ""; + return sql; } @Override @@ -108,13 +101,10 @@ public class JDBC4Connection extends LimboConnection { } @Override - public void setCatalog(String catalog) throws SQLException { - // TODO - } + public void setCatalog(String catalog) throws SQLException {} @Override public String getCatalog() throws SQLException { - // TODO return ""; } @@ -149,33 +139,30 @@ public class JDBC4Connection extends LimboConnection { return null; } - @Override - @SkipNullableCheck - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) - throws SQLException { - // TODO - return null; - } - @Override public Map> getTypeMap() throws SQLException { - // TODO - return new HashMap<>(); + return this.typeMap; } @Override public void setTypeMap(Map> map) throws SQLException { - // TODO - } - - @Override - public void setHoldability(int holdability) throws SQLException { - // TODO + synchronized (this) { + this.typeMap = map; + } } @Override public int getHoldability() throws SQLException { - return 0; + checkOpen(); + return ResultSet.CLOSE_CURSORS_AT_COMMIT; + } + + @Override + public void setHoldability(int holdability) throws SQLException { + checkOpen(); + if (holdability != ResultSet.CLOSE_CURSORS_AT_COMMIT) { + throw new SQLException("Limbo only supports CLOSE_CURSORS_AT_COMMIT"); + } } @Override @@ -212,12 +199,25 @@ public class JDBC4Connection extends LimboConnection { } @Override - @SkipNullableCheck + public CallableStatement prepareCall(String sql) throws SQLException { + return prepareCall( + sql, + ResultSet.TYPE_FORWARD_ONLY, + ResultSet.CONCUR_READ_ONLY, + ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) + throws SQLException { + return prepareCall(sql, resultSetType, resultSetConcurrency, ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + @Override public CallableStatement prepareCall( String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { - // TODO - return null; + throw new SQLException("Limbo does not support stored procedures"); } @Override From 1f3ddaeec6d749938005a298e7cf2a640787dbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 17:24:35 +0900 Subject: [PATCH 032/115] Implement prepareStatement --- .../tursodatabase/jdbc4/JDBC4Connection.java | 56 +++++++++---------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index 34287f0f9..598404b06 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -41,11 +41,6 @@ public class JDBC4Connection extends LimboConnection { return new JDBC4Statement(this); } - @Override - public PreparedStatement prepareStatement(String sql) throws SQLException { - return new JDBC4PreparedStatement(this, sql); - } - @Override public String nativeSQL(String sql) throws SQLException { return sql; @@ -131,14 +126,6 @@ public class JDBC4Connection extends LimboConnection { // TODO } - @Override - @SkipNullableCheck - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) - throws SQLException { - // TODO - return null; - } - @Override public Map> getTypeMap() throws SQLException { return this.typeMap; @@ -189,15 +176,6 @@ public class JDBC4Connection extends LimboConnection { // TODO } - @Override - @SkipNullableCheck - public PreparedStatement prepareStatement( - String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { - // TODO - return null; - } - @Override public CallableStatement prepareCall(String sql) throws SQLException { return prepareCall( @@ -221,24 +199,40 @@ public class JDBC4Connection extends LimboConnection { } @Override - @SkipNullableCheck + public PreparedStatement prepareStatement(String sql) throws SQLException { + return prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) + throws SQLException { + return prepareStatement(sql, resultSetType, resultSetConcurrency, ResultSet.CLOSE_CURSORS_AT_COMMIT); + } + + @Override + public PreparedStatement prepareStatement( + String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + checkOpen(); + checkCursor(resultSetType, resultSetConcurrency, resultSetHoldability); + return new JDBC4PreparedStatement(this, sql); + } + + @Override public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - // TODO - return null; + return prepareStatement(sql); } @Override - @SkipNullableCheck public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - // TODO - return null; + // TODO: maybe we can enhance this functionality by using columnIndexes + return prepareStatement(sql); } @Override - @SkipNullableCheck public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - // TODO - return null; + // TODO: maybe we can enhance this functionality by using columnNames + return prepareStatement(sql); } @Override From 4e067b2997db25bb6781ab6779bc8f7ca6bb5fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 17:24:52 +0900 Subject: [PATCH 033/115] Throw exceptions on unsupported methods --- .../tursodatabase/jdbc4/JDBC4Connection.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index 598404b06..61df095ff 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -236,31 +236,24 @@ public class JDBC4Connection extends LimboConnection { } @Override - @SkipNullableCheck public Clob createClob() throws SQLException { - // TODO - return null; + throw new SQLFeatureNotSupportedException("createClob not supported"); } @Override - @SkipNullableCheck public Blob createBlob() throws SQLException { - // TODO - return null; + throw new SQLFeatureNotSupportedException("createBlob not supported"); } @Override - @SkipNullableCheck public NClob createNClob() throws SQLException { - // TODO - return null; + throw new SQLFeatureNotSupportedException("createNClob not supported"); } @Override @SkipNullableCheck public SQLXML createSQLXML() throws SQLException { - // TODO - return null; + throw new SQLFeatureNotSupportedException("createSQLXML not supported"); } @Override From 968ae74810623ed82a84f76098fe4ab27c6947d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 17:28:32 +0900 Subject: [PATCH 034/115] Implement isValid --- .../tursodatabase/jdbc4/JDBC4Connection.java | 18 ++++++++++++------ .../jdbc4/JDBC4ConnectionTest.java | 11 +++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index 61df095ff..b6e2dcfe7 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -205,14 +205,15 @@ public class JDBC4Connection extends LimboConnection { @Override public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) - throws SQLException { - return prepareStatement(sql, resultSetType, resultSetConcurrency, ResultSet.CLOSE_CURSORS_AT_COMMIT); + throws SQLException { + return prepareStatement( + sql, resultSetType, resultSetConcurrency, ResultSet.CLOSE_CURSORS_AT_COMMIT); } @Override public PreparedStatement prepareStatement( - String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { + String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { checkOpen(); checkCursor(resultSetType, resultSetConcurrency, resultSetHoldability); return new JDBC4PreparedStatement(this, sql); @@ -258,8 +259,13 @@ public class JDBC4Connection extends LimboConnection { @Override public boolean isValid(int timeout) throws SQLException { - // TODO - return false; + if (isClosed()) { + return false; + } + + try (Statement statement = createStatement()) { + return statement.execute("select 1;"); + } } @Override diff --git a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java index 1bc4fb526..6302bc899 100644 --- a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java +++ b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java @@ -84,4 +84,15 @@ class JDBC4ConnectionTest { connection.createStatement( ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, -1)); } + + @Test + void isValid_should_return_true_on_open_connection() throws SQLException { + assertTrue(connection.isValid(10)); + } + + @Test + void isValid_should_return_false_on_closed_connection() throws SQLException { + connection.close(); + assertFalse(connection.isValid(10)); + } } From ed9cf63c51c195daa88ba19966f0e2b46436b075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 17:37:19 +0900 Subject: [PATCH 035/115] Implement abort --- .../org/github/tursodatabase/jdbc4/JDBC4Connection.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java index b6e2dcfe7..8026827a6 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java @@ -319,7 +319,11 @@ public class JDBC4Connection extends LimboConnection { @Override public void abort(Executor executor) throws SQLException { - // TODO + if (isClosed()) { + return; + } + + close(); } @Override From d1789d1d6eab176078defb1db6fc482594ee695d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 17:49:16 +0900 Subject: [PATCH 036/115] Implement executeUpdate --- .../jdbc4/JDBC4PreparedStatement.java | 7 ++++- .../tursodatabase/jdbc4/JDBC4Statement.java | 26 +++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4PreparedStatement.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4PreparedStatement.java index f109cb647..024fe3c3c 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4PreparedStatement.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4PreparedStatement.java @@ -24,6 +24,7 @@ import java.sql.Timestamp; import java.util.Calendar; import org.github.tursodatabase.annotations.SkipNullableCheck; import org.github.tursodatabase.core.LimboConnection; +import org.github.tursodatabase.core.LimboResultSet; public class JDBC4PreparedStatement extends JDBC4Statement implements PreparedStatement { @@ -46,7 +47,11 @@ public class JDBC4PreparedStatement extends JDBC4Statement implements PreparedSt @Override public int executeUpdate() throws SQLException { - // TODO + requireNonNull(this.statement); + final LimboResultSet resultSet = statement.getResultSet(); + resultSet.consumeAll(); + + // TODO: return updated count return 0; } diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java index 729134a44..1bc0b6f63 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java @@ -253,10 +253,8 @@ public class JDBC4Statement implements Statement { } @Override - @SkipNullableCheck - public Connection getConnection() throws SQLException { - // TODO - return null; + public Connection getConnection() { + return connection; } @Override @@ -274,32 +272,32 @@ public class JDBC4Statement implements Statement { @Override public int executeUpdate(String sql, int autoGeneratedKeys) throws SQLException { - // TODO - return 0; + // TODO: enhance + return executeUpdate(sql); } @Override public int executeUpdate(String sql, int[] columnIndexes) throws SQLException { - // TODO - return 0; + // TODO: enhance + return executeUpdate(sql); } @Override public int executeUpdate(String sql, String[] columnNames) throws SQLException { - // TODO - return 0; + // TODO: enhance + return executeUpdate(sql); } @Override public boolean execute(String sql, int autoGeneratedKeys) throws SQLException { - // TODO - return false; + // TODO: enhance + return execute(sql); } @Override public boolean execute(String sql, int[] columnIndexes) throws SQLException { - // TODO - return false; + // TODO: enhance + return execute(sql); } @Override From d51c1dc5b1e3b4997e7e5a871cccf2c00561ad3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 18:40:42 +0900 Subject: [PATCH 037/115] Remove AbstractDB and move those methods into LimboDB --- .../github/tursodatabase/core/AbstractDB.java | 75 ------------------- .../tursodatabase/core/LimboConnection.java | 6 +- .../github/tursodatabase/core/LimboDB.java | 46 ++++++++---- .../tursodatabase/core/LimboDBTest.java | 20 ++++- 4 files changed, 49 insertions(+), 98 deletions(-) delete mode 100644 bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java b/bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java deleted file mode 100644 index 4906acd9c..000000000 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.github.tursodatabase.core; - -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Interface to Limbo. It provides some helper functions used by other parts of the driver. The goal - * of the helper functions here are not only to provide functionality, but to handle contractual - * differences between the JDBC specification and the Limbo API. - */ -public abstract class AbstractDB { - protected final String url; - protected final String filePath; - private final AtomicBoolean closed = new AtomicBoolean(true); - - public AbstractDB(String url, String filePath) { - this.url = url; - this.filePath = filePath; - } - - public boolean isClosed() { - return closed.get(); - } - - /** Aborts any pending operation and returns at its earliest opportunity. */ - public abstract void interrupt() throws SQLException; - - /** - * Creates an SQLite interface to a database for the given connection. - * - * @param openFlags Flags for opening the database. - * @throws SQLException if a database access error occurs. - */ - public final synchronized void open(int openFlags) throws SQLException { - open0(filePath, openFlags); - } - - protected abstract void open0(String fileName, int openFlags) throws SQLException; - - /** - * Closes a database connection and finalizes any remaining statements before the closing - * operation. - * - * @throws SQLException if a database access error occurs. - */ - public final synchronized void close() throws SQLException { - // TODO: add implementation - throw new SQLFeatureNotSupportedException(); - } - - /** - * Connects to a database. - * - * @return Pointer to the connection. - */ - public abstract long connect() throws SQLException; - - /** - * Creates an SQLite interface to a database with the provided open flags. - * - * @param fileName The database to open. - * @param openFlags Flags for opening the database. - * @return pointer to database instance - * @throws SQLException if a database access error occurs. - */ - protected abstract long openUtf8(byte[] fileName, int openFlags) throws SQLException; - - /** - * Closes the SQLite interface to a database. - * - * @throws SQLException if a database access error occurs. - */ - protected abstract void close0() throws SQLException; -} diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboConnection.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboConnection.java index 639a2be56..3267e69e6 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboConnection.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboConnection.java @@ -15,7 +15,7 @@ public abstract class LimboConnection implements Connection { private static final Logger logger = LoggerFactory.getLogger(LimboConnection.class); private final long connectionPtr; - private final AbstractDB database; + private final LimboDB database; private boolean closed; public LimboConnection(String url, String filePath) throws SQLException { @@ -33,7 +33,7 @@ public abstract class LimboConnection implements Connection { this.connectionPtr = this.database.connect(); } - private static AbstractDB open(String url, String filePath, Properties properties) + private static LimboDB open(String url, String filePath, Properties properties) throws SQLException { return LimboDBFactory.open(url, filePath, properties); } @@ -58,7 +58,7 @@ public abstract class LimboConnection implements Connection { return closed; } - public AbstractDB getDatabase() { + public LimboDB getDatabase() { return database; } diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java index e1d619239..64b005d35 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java @@ -7,7 +7,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; -import java.util.concurrent.locks.ReentrantLock; import org.github.tursodatabase.LimboErrorCode; import org.github.tursodatabase.annotations.NativeInvocation; import org.github.tursodatabase.annotations.VisibleForTesting; @@ -16,14 +15,15 @@ import org.github.tursodatabase.utils.Logger; import org.github.tursodatabase.utils.LoggerFactory; /** This class provides a thin JNI layer over the SQLite3 C API. */ -public final class LimboDB extends AbstractDB { +public final class LimboDB implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(LimboDB.class); // Pointer to database instance private long dbPointer; private boolean isOpen; + private final String url; + private final String filePath; private static boolean isLoaded; - private ReentrantLock dbLock = new ReentrantLock(); static { if ("The Android Project".equals(System.getProperty("java.vm.vendor"))) { @@ -176,23 +176,26 @@ public final class LimboDB extends AbstractDB { // TODO: receive config as argument private LimboDB(String url, String filePath) { - super(url, filePath); + this.url = url; + this.filePath = filePath; } // TODO: add support for JNI - @Override - protected native long openUtf8(byte[] file, int openFlags) throws SQLException; - - // TODO: add support for JNI - @Override - protected native void close0() throws SQLException; - - // TODO: add support for JNI - @Override public native void interrupt(); - @Override - protected void open0(String filePath, int openFlags) throws SQLException { + public boolean isClosed() { + return !this.isOpen; + } + + public boolean isOpen() { + return this.isOpen; + } + + public void open(int openFlags) throws SQLException { + open0(filePath, openFlags); + } + + private void open0(String filePath, int openFlags) throws SQLException { if (isOpen) { throw LimboExceptionUtils.buildLimboException( LimboErrorCode.LIMBO_ETC.code, "Already opened"); @@ -209,13 +212,24 @@ public final class LimboDB extends AbstractDB { isOpen = true; } - @Override + private native long openUtf8(byte[] file, int openFlags) throws SQLException; + public long connect() throws SQLException { return connect0(dbPointer); } private native long connect0(long databasePtr) throws SQLException; + @Override + public void close() throws Exception { + if (!isOpen) return; + + close0(dbPointer); + isOpen = false; + } + + private native void close0(long databasePtr) throws SQLException; + @VisibleForTesting native void throwJavaException(int errorCode) throws SQLException; diff --git a/bindings/java/src/test/java/org/github/tursodatabase/core/LimboDBTest.java b/bindings/java/src/test/java/org/github/tursodatabase/core/LimboDBTest.java index ca75ac4c7..e2fcc8204 100644 --- a/bindings/java/src/test/java/org/github/tursodatabase/core/LimboDBTest.java +++ b/bindings/java/src/test/java/org/github/tursodatabase/core/LimboDBTest.java @@ -2,6 +2,7 @@ package org.github.tursodatabase.core; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.sql.SQLException; import org.github.tursodatabase.LimboErrorCode; @@ -13,16 +14,27 @@ public class LimboDBTest { @Test void db_should_open_normally() throws Exception { - String dbPath = TestUtils.createTempFile(); LimboDB.load(); + String dbPath = TestUtils.createTempFile(); LimboDB db = LimboDB.create("jdbc:sqlite" + dbPath, dbPath); db.open(0); } @Test - void should_throw_exception_when_opened_twice() throws Exception { - String dbPath = TestUtils.createTempFile(); + void db_should_close_normally() throws Exception { LimboDB.load(); + String dbPath = TestUtils.createTempFile(); + LimboDB db = LimboDB.create("jdbc:sqlite" + dbPath, dbPath); + db.open(0); + db.close(); + + assertFalse(db.isOpen()); + } + + @Test + void should_throw_exception_when_opened_twice() throws Exception { + LimboDB.load(); + String dbPath = TestUtils.createTempFile(); LimboDB db = LimboDB.create("jdbc:sqlite:" + dbPath, dbPath); db.open(0); @@ -31,8 +43,8 @@ public class LimboDBTest { @Test void throwJavaException_should_throw_appropriate_java_exception() throws Exception { - String dbPath = TestUtils.createTempFile(); LimboDB.load(); + String dbPath = TestUtils.createTempFile(); LimboDB db = LimboDB.create("jdbc:sqlite:" + dbPath, dbPath); final int limboExceptionCode = LimboErrorCode.LIMBO_ETC.code; From e0b0a667bbfc77ce0f389d374954524f2728e91e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 18:40:50 +0900 Subject: [PATCH 038/115] Implement close --- bindings/java/rs_src/limbo_db.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bindings/java/rs_src/limbo_db.rs b/bindings/java/rs_src/limbo_db.rs index 16cb3d66b..5f0dfb84c 100644 --- a/bindings/java/rs_src/limbo_db.rs +++ b/bindings/java/rs_src/limbo_db.rs @@ -21,7 +21,6 @@ impl LimboDB { Box::into_raw(Box::new(self)) as jlong } - #[allow(dead_code)] pub fn drop(ptr: jlong) { let _boxed = unsafe { Box::from_raw(ptr as *mut LimboDB) }; } @@ -97,6 +96,15 @@ pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB_connect0<'loca conn.to_ptr() } +#[no_mangle] +pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB_close0<'local>( + _env: JNIEnv<'local>, + _obj: JObject<'local>, + db_pointer: jlong +) { + LimboDB::drop(db_pointer); +} + #[no_mangle] pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB_throwJavaException<'local>( mut env: JNIEnv<'local>, From d177f6195bfdeb1e7bf0c43cce6b68fbd3a60f0d Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sun, 9 Feb 2025 12:34:53 +0200 Subject: [PATCH 039/115] sqlite3-parser: box big members of createindex --- vendored/sqlite3-parser/src/parser/ast/mod.rs | 4 ++-- vendored/sqlite3-parser/src/parser/parse.y | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 8cf4f805d..780af728f 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -95,13 +95,13 @@ pub enum Stmt { /// `IF NOT EXISTS` if_not_exists: bool, /// index name - idx_name: QualifiedName, + idx_name: Box, /// table name tbl_name: Name, /// indexed columns or expressions columns: Vec, /// partial index - where_clause: Option, + where_clause: Option>, }, /// `CREATE TABLE` CreateTable { diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index bb3c80e91..38fb47fda 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -1077,8 +1077,8 @@ paren_exprlist(A) ::= LP exprlist(X) RP. {A = X;} // cmd ::= createkw uniqueflag(U) INDEX ifnotexists(NE) fullname(X) ON nm(Y) LP sortlist(Z) RP where_opt(W). { - self.ctx.stmt = Some(Stmt::CreateIndex { unique: U, if_not_exists: NE, idx_name: X, - tbl_name: Y, columns: Z, where_clause: W }); + self.ctx.stmt = Some(Stmt::CreateIndex { unique: U, if_not_exists: NE, idx_name: Box::new(X), + tbl_name: Y, columns: Z, where_clause: W.map(Box::new) }); } %type uniqueflag {bool} From 358fda2ec74c6ae1b33dff8b14519b8101f99909 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sun, 9 Feb 2025 12:42:53 +0200 Subject: [PATCH 040/115] sqlite3-parser: box the create table body --- core/ext/mod.rs | 2 +- core/schema.rs | 2 +- core/translate/mod.rs | 2 +- vendored/sqlite3-parser/src/parser/ast/mod.rs | 2 +- vendored/sqlite3-parser/src/parser/parse.y | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/ext/mod.rs b/core/ext/mod.rs index 67fd78491..344b8f4a4 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -127,7 +127,7 @@ impl Database { let Stmt::CreateTable { body, .. } = stmt else { return ResultCode::Error; }; - let Ok(columns) = columns_from_create_table_body(body) else { + let Ok(columns) = columns_from_create_table_body(*body) else { return ResultCode::Error; }; let vtab_module = self.vtab_modules.get(name).unwrap().clone(); diff --git a/core/schema.rs b/core/schema.rs index f4a6aee2b..aa38cd191 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -150,7 +150,7 @@ impl BTreeTable { let cmd = parser.next()?; match cmd { Some(Cmd::Stmt(Stmt::CreateTable { tbl_name, body, .. })) => { - create_table(tbl_name, body, root_page) + create_table(tbl_name, *body, root_page) } _ => todo!("Expected CREATE TABLE statement"), } diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 79791866b..d9c3f7421 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -67,7 +67,7 @@ pub fn translate( bail_parse_error!("TEMPORARY table not supported yet"); } - translate_create_table(query_mode, tbl_name, body, if_not_exists, schema)? + translate_create_table(query_mode, tbl_name, *body, if_not_exists, schema)? } ast::Stmt::CreateTrigger { .. } => bail_parse_error!("CREATE TRIGGER not supported yet"), ast::Stmt::CreateView { .. } => bail_parse_error!("CREATE VIEW not supported yet"), diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 780af728f..049084bd5 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -112,7 +112,7 @@ pub enum Stmt { /// table name tbl_name: QualifiedName, /// table body - body: CreateTableBody, + body: Box, }, /// `CREATE TRIGGER` CreateTrigger { diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 38fb47fda..511c83c3b 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -109,7 +109,7 @@ cmd ::= ROLLBACK trans_opt(Y) TO savepoint_opt nm(X). { ///////////////////// The CREATE TABLE statement //////////////////////////// // cmd ::= createkw temp(T) TABLE ifnotexists(E) fullname(Y) create_table_args(X). { - self.ctx.stmt = Some(Stmt::CreateTable{ temporary: T, if_not_exists: E, tbl_name: Y, body: X }); + self.ctx.stmt = Some(Stmt::CreateTable{ temporary: T, if_not_exists: E, tbl_name: Y, body: Box::new(X) }); } createkw(A) ::= CREATE(A). From 575b48474065e8c2720de5ccfec2889f73b105b0 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sun, 9 Feb 2025 12:50:00 +0200 Subject: [PATCH 041/115] sqlite3-parser: separate boxed CreateTrigger struct --- vendored/sqlite3-parser/src/parser/ast/fmt.rs | 23 ++++--- vendored/sqlite3-parser/src/parser/ast/mod.rs | 67 +++++++------------ vendored/sqlite3-parser/src/parser/parse.y | 8 +-- 3 files changed, 39 insertions(+), 59 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/fmt.rs b/vendored/sqlite3-parser/src/parser/ast/fmt.rs index 34a7fa3f0..10e7c1ad9 100644 --- a/vendored/sqlite3-parser/src/parser/ast/fmt.rs +++ b/vendored/sqlite3-parser/src/parser/ast/fmt.rs @@ -211,17 +211,18 @@ impl ToTokens for Stmt { tbl_name.to_tokens(s)?; body.to_tokens(s) } - Self::CreateTrigger { - temporary, - if_not_exists, - trigger_name, - time, - event, - tbl_name, - for_each_row, - when_clause, - commands, - } => { + Self::CreateTrigger(trigger) => { + let CreateTrigger { + temporary, + if_not_exists, + trigger_name, + time, + event, + tbl_name, + for_each_row, + when_clause, + commands, + } = &**trigger; s.append(TK_CREATE, None)?; if *temporary { s.append(TK_TEMP, None)?; diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 049084bd5..814aacdb1 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -115,26 +115,7 @@ pub enum Stmt { body: Box, }, /// `CREATE TRIGGER` - CreateTrigger { - /// `TEMPORARY` - temporary: bool, - /// `IF NOT EXISTS` - if_not_exists: bool, - /// trigger name - trigger_name: QualifiedName, - /// `BEFORE`/`AFTER`/`INSTEAD OF` - time: Option, - /// `DELETE`/`INSERT`/`UPDATE` - event: Box, - /// table name - tbl_name: QualifiedName, - /// `FOR EACH ROW` - for_each_row: bool, - /// `WHEN` - when_clause: Option>, - /// statements - commands: Vec, - }, + CreateTrigger(Box), /// `CREATE VIEW` CreateView { /// `TEMPORARY` @@ -242,30 +223,28 @@ pub enum Stmt { /// `SELECT` Select(Box), /// `UPDATE` + Update(Box), + /// `VACUUM`: database name, into expr + Vacuum(Option, Option), +} + /// `CREATE TRIGGER #[derive(Clone, Debug, PartialEq, Eq)] pub struct CreateTrigger { @@ -245,6 +250,30 @@ pub struct CreateTrigger { /// statements pub commands: Vec, } + +/// `UPDATE` clause +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Update { + /// CTE + pub with: Option, + /// `OR` + pub or_conflict: Option, + /// table name + pub tbl_name: QualifiedName, + /// `INDEXED` + pub indexed: Option, + /// `SET` assignments + pub sets: Vec, + /// `FROM` + pub from: Option, + /// `WHERE` clause + pub where_clause: Option>, + /// `RETURNING` + pub returning: Option>, + /// `ORDER BY` + pub order_by: Option>, + /// `LIMIT` + pub limit: Option>, } /// SQL expression diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index b24c9b708..2e298aa9e 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -790,15 +790,15 @@ where_opt_ret(A) ::= WHERE expr(X) RETURNING selcollist(Y). cmd ::= with(C) UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y) from(F) where_opt_ret(W) orderby_opt(O) limit_opt(L). { let (where_clause, returning) = W; - self.ctx.stmt = Some(Stmt::Update { with: C, or_conflict: R, tbl_name: X, indexed: I, sets: Y, from: F, - where_clause: where_clause.map(Box::new), returning, order_by: O, limit: L }); + self.ctx.stmt = Some(Stmt::Update(Box::new(Update{ with: C, or_conflict: R, tbl_name: X, indexed: I, sets: Y, from: F, + where_clause: where_clause.map(Box::new), returning, order_by: O, limit: L }))); } %else cmd ::= with(C) UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y) from(F) where_opt_ret(W). { let (where_clause, returning) = W; - self.ctx.stmt = Some(Stmt::Update { with: C, or_conflict: R, tbl_name: X, indexed: I, sets: Y, from: F, - where_clause: where_clause.map(Box::new), returning, order_by: None, limit: None }); + self.ctx.stmt = Some(Stmt::Update(Box::new(Update{ with: C, or_conflict: R, tbl_name: X, indexed: I, sets: Y, from: F, + where_clause: where_clause.map(Box::new), returning, order_by: None, limit: None }))); } %endif From f75aca67bb00124008b1081be5e6ae0ad18031e9 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sun, 9 Feb 2025 12:52:30 +0200 Subject: [PATCH 043/115] sqlite3-parser: separate boxed TriggerCmd struct variants --- vendored/sqlite3-parser/src/parser/ast/fmt.rs | 41 +++++----- vendored/sqlite3-parser/src/parser/ast/mod.rs | 76 +++++++++++-------- vendored/sqlite3-parser/src/parser/parse.y | 6 +- 3 files changed, 67 insertions(+), 56 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/fmt.rs b/vendored/sqlite3-parser/src/parser/ast/fmt.rs index 2d44f5255..313177a39 100644 --- a/vendored/sqlite3-parser/src/parser/ast/fmt.rs +++ b/vendored/sqlite3-parser/src/parser/ast/fmt.rs @@ -1651,13 +1651,14 @@ impl ToTokens for TriggerEvent { impl ToTokens for TriggerCmd { fn to_tokens(&self, s: &mut S) -> Result<(), S::Error> { match self { - Self::Update { - or_conflict, - tbl_name, - sets, - from, - where_clause, - } => { + Self::Update(update) => { + let TriggerCmdUpdate { + or_conflict, + tbl_name, + sets, + from, + where_clause, + } = &**update; s.append(TK_UPDATE, None)?; if let Some(or_conflict) = or_conflict { s.append(TK_OR, None)?; @@ -1676,14 +1677,15 @@ impl ToTokens for TriggerCmd { } Ok(()) } - Self::Insert { - or_conflict, - tbl_name, - col_names, - select, - upsert, - returning, - } => { + Self::Insert(insert) => { + let TriggerCmdInsert { + or_conflict, + tbl_name, + col_names, + select, + upsert, + returning, + } = &**insert; if let Some(ResolveType::Replace) = or_conflict { s.append(TK_REPLACE, None)?; } else { @@ -1710,14 +1712,11 @@ impl ToTokens for TriggerCmd { } Ok(()) } - Self::Delete { - tbl_name, - where_clause, - } => { + Self::Delete(delete) => { s.append(TK_DELETE, None)?; s.append(TK_FROM, None)?; - tbl_name.to_tokens(s)?; - if let Some(where_clause) = where_clause { + delete.tbl_name.to_tokens(s)?; + if let Some(where_clause) = &delete.where_clause { s.append(TK_WHERE, None)?; where_clause.to_tokens(s)?; } diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 3c729ea70..6b0ac88df 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -1640,44 +1640,56 @@ pub enum TriggerEvent { #[derive(Clone, Debug, PartialEq, Eq)] pub enum TriggerCmd { /// `UPDATE` - Update { - /// `OR` - or_conflict: Option, - /// table name - tbl_name: Name, - /// `SET` assignments - sets: Vec, - /// `FROM` - from: Option, - /// `WHERE` clause - where_clause: Option, - }, + Update(Box), /// `INSERT` - Insert { - /// `OR` - or_conflict: Option, - /// table name - tbl_name: Name, - /// `COLUMNS` - col_names: Option, - /// `SELECT` or `VALUES` - select: Box), } +/// `UPDATE` trigger command +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TriggerCmdUpdate { + /// `OR` + pub or_conflict: Option, + /// table name + pub tbl_name: Name, + /// `SET` assignments + pub sets: Vec, + /// `FROM` + pub from: Option, + /// `WHERE` clause + pub where_clause: Option, +} + +/// `INSERT` trigger command +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TriggerCmdInsert { + /// `OR` + pub or_conflict: Option, + /// table name + pub tbl_name: Name, + /// `COLUMNS` + pub col_names: Option, + /// `SELECT` or `VALUES` + pub select: Box, }, /// `CREATE VIRTUAL TABLE` - CreateVirtualTable { - /// `IF NOT EXISTS` - if_not_exists: bool, - /// table name - tbl_name: QualifiedName, - /// module - module_name: Name, - /// args - args: Option>, // TODO smol str - }, + CreateVirtualTable(Box), /// `DELETE` Delete(Box), /// `DETACH DATABASE`: db name @@ -197,7 +188,17 @@ pub enum Stmt { /// `UPDATE` Update(Box), /// `VACUUM`: database name, into expr - Vacuum(Option, Option), +/// `CREATE VIRTUAL TABLE` +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CreateVirtualTable { + /// `IF NOT EXISTS` + pub if_not_exists: bool, + /// table name + pub tbl_name: QualifiedName, + /// module name + pub module_name: Name, + /// args + pub args: Option>, // TODO smol str } /// `CREATE TRIGGER diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 16c8b9f00..6ff139835 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -1329,15 +1329,15 @@ kwcolumn_opt ::= COLUMNKW. cmd ::= create_vtab(X). {self.ctx.stmt = Some(X);} cmd ::= create_vtab(X) LP vtabarglist RP. { let mut stmt = X; - if let Stmt::CreateVirtualTable{ ref mut args, .. } = stmt { - *args = self.ctx.module_args(); + if let Stmt::CreateVirtualTable(ref mut create_virtual_table) = stmt { + create_virtual_table.args = self.ctx.module_args(); } self.ctx.stmt = Some(stmt); } %type create_vtab {Stmt} create_vtab(A) ::= createkw VIRTUAL TABLE ifnotexists(E) fullname(X) USING nm(Z). { - A = Stmt::CreateVirtualTable{ if_not_exists: E, tbl_name: X, module_name: Z, args: None }; + A = Stmt::CreateVirtualTable(Box::new(CreateVirtualTable{ if_not_exists: E, tbl_name: X, module_name: Z, args: None })); } vtabarglist ::= vtabarg. vtabarglist ::= vtabarglist COMMA vtabarg. From a8685c80862f994f20965f825864474076b56898 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sun, 9 Feb 2025 14:11:57 +0200 Subject: [PATCH 051/115] sqlite3-parser: box the Expr in Vacuum --- vendored/sqlite3-parser/src/parser/ast/mod.rs | 3 +++ vendored/sqlite3-parser/src/parser/parse.y | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index f942d1090..3ea9d5992 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -188,6 +188,9 @@ pub enum Stmt { /// `UPDATE` Update(Box), /// `VACUUM`: database name, into expr + Vacuum(Option, Option>), +} + /// `CREATE VIRTUAL TABLE` #[derive(Clone, Debug, PartialEq, Eq)] pub struct CreateVirtualTable { diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index 6ff139835..aeb6c689f 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -1130,8 +1130,8 @@ cmd ::= DROP INDEX ifexists(E) fullname(X). {self.ctx.stmt = Some(Stmt::DropIn // %if !SQLITE_OMIT_VACUUM && !SQLITE_OMIT_ATTACH %type vinto {Option} -cmd ::= VACUUM vinto(Y). {self.ctx.stmt = Some(Stmt::Vacuum(None, Y));} -cmd ::= VACUUM nm(X) vinto(Y). {self.ctx.stmt = Some(Stmt::Vacuum(Some(X), Y));} +cmd ::= VACUUM vinto(Y). {self.ctx.stmt = Some(Stmt::Vacuum(None, Y.map(Box::new)));} +cmd ::= VACUUM nm(X) vinto(Y). {self.ctx.stmt = Some(Stmt::Vacuum(Some(X), Y.map(Box::new)));} vinto(A) ::= INTO expr(X). {A = Some(X);} vinto(A) ::= . {A = None;} %endif From a99d9a8988cc44fc57202dbf74bfd63aa217ebe1 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Sun, 9 Feb 2025 21:05:06 +0800 Subject: [PATCH 052/115] chore: remove unused dependencies --- Cargo.lock | 53 ------------------------------------ core/Cargo.toml | 3 -- extensions/core/Cargo.toml | 1 - extensions/regexp/Cargo.toml | 1 - extensions/series/Cargo.toml | 1 - tests/Cargo.toml | 3 -- 6 files changed, 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3d610591d..a57d30b7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -480,8 +480,6 @@ version = "0.0.14" dependencies = [ "anyhow", "assert_cmd", - "clap", - "dirs", "env_logger 0.10.2", "limbo_core", "log", @@ -489,7 +487,6 @@ dependencies = [ "rand_chacha 0.9.0", "rexpect", "rusqlite", - "rustyline", "tempfile", ] @@ -721,12 +718,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "downcast" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" - [[package]] name = "either" version = "1.13.0" @@ -901,12 +892,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fragile" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" - [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1617,7 +1602,6 @@ name = "limbo_core" version = "0.0.14" dependencies = [ "built", - "bumpalo", "cfg_block", "chrono", "criterion", @@ -1642,7 +1626,6 @@ dependencies = [ "log", "miette", "mimalloc", - "mockall", "parking_lot", "pest", "pest_derive", @@ -1657,7 +1640,6 @@ dependencies = [ "rusqlite", "rustix", "serde", - "sieve-cache", "sqlite3-parser", "strum", "tempfile", @@ -1683,7 +1665,6 @@ name = "limbo_ext" version = "0.0.14" dependencies = [ "limbo_macros", - "log", ] [[package]] @@ -1717,7 +1698,6 @@ name = "limbo_regexp" version = "0.0.14" dependencies = [ "limbo_ext", - "log", "mimalloc", "regex", ] @@ -1727,7 +1707,6 @@ name = "limbo_series" version = "0.0.14" dependencies = [ "limbo_ext", - "log", "mimalloc", "quickcheck", "quickcheck_macros", @@ -1908,32 +1887,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "mockall" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39a6bfcc6c8c7eed5ee98b9c3e33adc726054389233e201c95dab2d41a3839d2" -dependencies = [ - "cfg-if", - "downcast", - "fragile", - "mockall_derive", - "predicates", - "predicates-tree", -] - -[[package]] -name = "mockall_derive" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ca3004c2efe9011bd4e461bd8256445052b9615405b4f7ea43fc8ca5c20898" -dependencies = [ - "cfg-if", - "proc-macro2", - "quote", - "syn 2.0.96", -] - [[package]] name = "nibble_vec" version = "0.1.0" @@ -2796,12 +2749,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" -[[package]] -name = "sieve-cache" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bf3a9dccf2c079bf1465d449a485c85b36443caf765f2f127bfec28b180f75" - [[package]] name = "signal-hook-registry" version = "1.4.2" diff --git a/core/Cargo.toml b/core/Cargo.toml index d13591aee..933654d78 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -47,7 +47,6 @@ fallible-iterator = "0.3.0" hex = "0.4.3" libc = "0.2.155" log = "0.4.20" -sieve-cache = "0.1.4" sqlite3-parser = { path = "../vendored/sqlite3-parser" } thiserror = "1.0.61" getrandom = { version = "0.2.15", features = ["js"] } @@ -61,7 +60,6 @@ serde = { version = "1.0", features = ["derive"] } pest = { version = "2.0", optional = true } pest_derive = { version = "2.0", optional = true } rand = "0.8.5" -bumpalo = { version = "3.16.0", features = ["collections", "boxed"] } limbo_macros = { path = "../macros" } limbo_uuid = { path = "../extensions/uuid", optional = true, features = ["static"] } limbo_regexp = { path = "../extensions/regexp", optional = true, features = ["static"] } @@ -88,7 +86,6 @@ criterion = { version = "0.5", features = [ "async", "async_futures", ] } -mockall = "0.13.0" rstest = "0.18.2" rusqlite = "0.29.0" tempfile = "3.8.0" diff --git a/extensions/core/Cargo.toml b/extensions/core/Cargo.toml index 3194bcadb..7dc60b498 100644 --- a/extensions/core/Cargo.toml +++ b/extensions/core/Cargo.toml @@ -12,5 +12,4 @@ default = [] static = [] [dependencies] -log = "0.4.20" limbo_macros = { path = "../../macros" } diff --git a/extensions/regexp/Cargo.toml b/extensions/regexp/Cargo.toml index c8288e601..9702db5f1 100644 --- a/extensions/regexp/Cargo.toml +++ b/extensions/regexp/Cargo.toml @@ -17,7 +17,6 @@ crate-type = ["cdylib", "lib"] [dependencies] limbo_ext = { path = "../core", features = ["static"] } regex = "1.11.1" -log = "0.4.20" [target.'cfg(not(target_family = "wasm"))'.dependencies] mimalloc = { version = "*", default-features = false } diff --git a/extensions/series/Cargo.toml b/extensions/series/Cargo.toml index 823e95472..33f45cff2 100644 --- a/extensions/series/Cargo.toml +++ b/extensions/series/Cargo.toml @@ -15,7 +15,6 @@ crate-type = ["cdylib", "lib"] [dependencies] limbo_ext = { path = "../core", features = ["static"] } -log = "0.4.20" [target.'cfg(not(target_family = "wasm"))'.dependencies] mimalloc = { version = "*", default-features = false } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index f3e67d73a..a17c58910 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -16,11 +16,8 @@ path = "integration/mod.rs" [dependencies] anyhow = "1.0.75" -clap = { version = "4.5", features = ["derive"] } -dirs = "5.0.1" env_logger = "0.10.1" limbo_core = { path = "../core" } -rustyline = "12.0.0" rusqlite = { version = "0.29", features = ["bundled"] } tempfile = "3.0.7" log = "0.4.22" From 3557c8aada109a8ac51c15b73f7d4c7ee895d1f8 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 21:25:21 +0400 Subject: [PATCH 053/115] adjust fuzz test --- core/storage/btree.rs | 97 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 19 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index efdcb47ae..ce14c9ecf 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2459,26 +2459,47 @@ mod tests { } #[test] - pub fn btree_insert_fuzz() { + pub fn btree_insert_fuzz_ex() { let _ = env_logger::init(); - let (pager, root_page) = empty_btree(); - let mut cursor = BTreeCursor::new(pager.clone(), root_page); - let mut keys = Vec::new(); - let mut rng = ChaCha8Rng::seed_from_u64(0); - for _ in 0..16 { - let size = (rng.next_u64() % 4096) as usize; - let key = (rng.next_u64() % (1 << 30)) as i64; - keys.push(key); - log::info!("INSERT INTO t VALUES ({}, randomblob({}));", key, size); - let key = OwnedValue::Integer(key); - let value = Record::new(vec![OwnedValue::Blob(Rc::new(vec![0; size]))]); - cursor.insert(&key, &value, false).unwrap(); - log::info!( - "=========== btree ===========\n{}\n\n", - format_btree(pager.clone(), root_page, 0) - ); - - for key in keys.iter() { + for sequence in [ + &[ + (293471650, 2452), + (163608869, 627), + (544576229, 464), + (705823748, 3441), + ] + .as_slice(), + &[ + (987283511, 2924), + (261851260, 1766), + (343847101, 1657), + (315844794, 572), + ] + .as_slice(), + &[ + (987283511, 2924), + (261851260, 1766), + (343847101, 1657), + (315844794, 572), + (649272840, 1632), + (723398505, 3140), + (334416967, 3874), + ] + .as_slice(), + ] { + let (pager, root_page) = empty_btree(); + let mut cursor = BTreeCursor::new(pager.clone(), root_page); + for (key, size) in sequence.iter() { + let key = OwnedValue::Integer(*key); + let value = Record::new(vec![OwnedValue::Blob(Rc::new(vec![0; *size]))]); + log::info!("insert key:{}", key); + cursor.insert(&key, &value, false).unwrap(); + log::info!( + "=========== btree ===========\n{}\n\n", + format_btree(pager.clone(), root_page, 0) + ); + } + for (key, _) in sequence.iter() { let seek_key = SeekKey::TableRowId(*key as u64); assert!( matches!( @@ -2492,6 +2513,44 @@ mod tests { } } + #[test] + pub fn btree_insert_fuzz_run() { + let _ = env_logger::init(); + let mut rng = ChaCha8Rng::seed_from_u64(0); + for _ in 0..128 { + let (pager, root_page) = empty_btree(); + let mut cursor = BTreeCursor::new(pager.clone(), root_page); + let mut keys = Vec::new(); + let seed = rng.next_u64(); + log::info!("seed: {}", seed); + let mut rng = ChaCha8Rng::seed_from_u64(seed); + for _ in 0..16 { + let size = (rng.next_u64() % 4096) as usize; + let key = (rng.next_u64() % (1 << 30)) as i64; + keys.push(key); + log::info!("INSERT INTO t VALUES ({}, randomblob({}));", key, size); + let key = OwnedValue::Integer(key); + let value = Record::new(vec![OwnedValue::Blob(Rc::new(vec![0; size]))]); + cursor.insert(&key, &value, false).unwrap(); + log::info!( + "=========== btree ===========\n{}\n\n", + format_btree(pager.clone(), root_page, 0) + ); + for key in keys.iter() { + let seek_key = SeekKey::TableRowId(*key as u64); + assert!( + matches!( + cursor.seek(seek_key, SeekOp::EQ).unwrap(), + CursorResult::Ok(true) + ), + "key {} is not found", + key + ); + } + } + } + } + #[allow(clippy::arc_with_non_send_sync)] fn setup_test_env(database_size: u32) -> (Rc, Rc>) { let page_size = 512; From 62b4787d3d4de9157875ef124a50cbd691d28f79 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 18:35:34 +0400 Subject: [PATCH 054/115] simplify write_varint_to_vec function --- core/storage/sqlite3_ondisk.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index fbdac7dca..3cfdfca6f 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -1129,11 +1129,9 @@ pub fn write_varint(buf: &mut [u8], value: u64) -> usize { } pub fn write_varint_to_vec(value: u64, payload: &mut Vec) { - let mut varint: Vec = vec![0; 9]; - let n = write_varint(&mut varint.as_mut_slice()[0..9], value); - write_varint(&mut varint, value); - varint.truncate(n); - payload.extend_from_slice(&varint); + let mut varint = [0u8; 9]; + let n = write_varint(&mut varint, value); + payload.extend_from_slice(&varint[0..n]); } pub fn begin_read_wal_header(io: &Rc) -> Result>> { From 5ce3d12f7513f2132298a843bc9fd4698c69a450 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 21:25:01 +0400 Subject: [PATCH 055/115] fix typo --- core/storage/btree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index ce14c9ecf..51b3d32c3 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1115,7 +1115,7 @@ impl BTreeCursor { debug!("balance_non_root(page={})", current_page.get().id); // Copy of page used to reference cell bytes. - // This needs to be saved somewhere safe so taht references still point to here, + // This needs to be saved somewhere safe so that references still point to here, // this will be store in write_info below let page_copy = current_page.get().contents.as_ref().unwrap().clone(); From fc502b86c773f4163e965318a08f64d4691b5341 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 21:25:08 +0400 Subject: [PATCH 056/115] fix defragmentation code a bit --- core/storage/btree.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 51b3d32c3..d3a958d8e 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1607,6 +1607,8 @@ impl BTreeCursor { page.write_u16(PAGE_HEADER_OFFSET_CELL_CONTENT_AREA, cbrk as u16); // set free block to 0, unused spaced can be retrieved from gap between cell pointer end and content start page.write_u16(PAGE_HEADER_OFFSET_FIRST_FREEBLOCK, 0); + // set fragmented bytes counter to zero + page.write_u8(PAGE_HEADER_OFFSET_FRAGMENTED_BYTES_COUNT, 0); // set unused space to 0 let first_cell = cloned_page.cell_content_area() as u64; assert!(first_cell <= cbrk); From eec0493c60c2393677446eedd2f47cfd1eed2a18 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 21:29:09 +0400 Subject: [PATCH 057/115] remove misleading comment --- core/storage/btree.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index d3a958d8e..5169d0687 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1057,11 +1057,6 @@ impl BTreeCursor { .clone(); match state { WriteState::BalanceStart => { - // drop divider cells and find right pointer - // NOTE: since we are doing a simple split we only finding the pointer we want to update (right pointer). - // Right pointer means cell that points to the last page, as we don't really want to drop this one. This one - // can be a "rightmost pointer" or a "cell". - // we always asumme there is a parent let current_page = self.stack.top(); { // check if we don't need to balance From 6aa10701a429cb6e36867b3c892f6bc55127bef4 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 21:33:23 +0400 Subject: [PATCH 058/115] fix comment --- core/storage/btree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 5169d0687..2d2c3291b 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1206,7 +1206,7 @@ impl BTreeCursor { BTreeCell::TableInteriorCell(interior) => { interior._left_child_page as usize == current_idx } - _ => unreachable!("Parent should always be a "), + _ => unreachable!("Parent should always be an interior page"), }; if found { let (start, _len) = parent_contents.cell_get_raw_region( From ea61f31843884cf8b204159505ba0544cdb23e1b Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sat, 8 Feb 2025 21:52:04 +0400 Subject: [PATCH 059/115] clear overflow_cells --- core/storage/btree.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 2d2c3291b..d21dedeb2 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1109,10 +1109,14 @@ impl BTreeCursor { let current_page = self.stack.top(); debug!("balance_non_root(page={})", current_page.get().id); + let current_page_inner = current_page.get(); + let current_page_contents = &mut current_page_inner.contents; + let current_page_contents = current_page_contents.as_mut().unwrap(); // Copy of page used to reference cell bytes. // This needs to be saved somewhere safe so that references still point to here, // this will be store in write_info below - let page_copy = current_page.get().contents.as_ref().unwrap().clone(); + let page_copy = current_page_contents.clone(); + current_page_contents.overflow_cells.clear(); // In memory in order copy of all cells in pages we want to balance. For now let's do a 2 page split. // Right pointer in interior cells should be converted to regular cells if more than 2 pages are used for balancing. From d4bbad161b25328cca11474ad737b720484cb22b Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 17:08:00 +0400 Subject: [PATCH 060/115] handle case when we can't balance all cells between current page and one new allocated page - if we have page which is tightly packed with relatively big cells, we will be unable to balance its content if we will insert very big (~page size) cell in the middle (because nothing can't be merged with new cell - so we will need to split 1 page into 3) --- core/storage/btree.rs | 131 ++++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 62 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index d21dedeb2..7af0945fe 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -76,7 +76,7 @@ macro_rules! return_if_locked { /// State machine of a write operation. /// May involve balancing due to overflow. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] enum WriteState { Start, BalanceStart, @@ -89,8 +89,10 @@ enum WriteState { struct WriteInfo { /// State of the write operation state machine. state: WriteState, - /// Pages allocated during the write operation due to balancing. - new_pages: RefCell>, + /// Pages involved in the split of the page due to balancing (splits_pages[0] is the balancing page, while other - fresh allocated pages) + split_pages: RefCell>, + /// Amount of cells from balancing page for every split page + split_pages_cells_count: RefCell>, /// Scratch space used during balancing. scratch_cells: RefCell>, /// Bookkeeping of the rightmost pointer so the PAGE_HEADER_OFFSET_RIGHTMOST_PTR can be updated. @@ -103,7 +105,8 @@ impl WriteInfo { fn new() -> WriteInfo { WriteInfo { state: WriteState::Start, - new_pages: RefCell::new(Vec::with_capacity(4)), + split_pages: RefCell::new(Vec::with_capacity(4)), + split_pages_cells_count: RefCell::new(Vec::with_capacity(4)), scratch_cells: RefCell::new(Vec::new()), rightmost_pointer: RefCell::new(None), page_copy: RefCell::new(None), @@ -1091,12 +1094,7 @@ impl BTreeCursor { matches!(self.state, CursorState::Write(_)), "Cursor must be in balancing state" ); - let state = self - .state - .write_info() - .expect("must be balancing") - .state - .clone(); + let state = self.state.write_info().expect("must be balancing").state; let (next_write_state, result) = match state { WriteState::Start => todo!(), WriteState::BalanceStart => todo!(), @@ -1124,47 +1122,69 @@ impl BTreeCursor { let mut scratch_cells = write_info.scratch_cells.borrow_mut(); scratch_cells.clear(); + let usable_space = self.usable_space(); for cell_idx in 0..page_copy.cell_count() { let (start, len) = page_copy.cell_get_raw_region( cell_idx, self.payload_overflow_threshold_max(page_copy.page_type()), self.payload_overflow_threshold_min(page_copy.page_type()), - self.usable_space(), + usable_space, ); - let buf = page_copy.as_ptr(); - scratch_cells.push(to_static_buf(&buf[start..start + len])); + let cell_buffer = to_static_buf(&page_copy.as_ptr()[start..start + len]); + scratch_cells.push(cell_buffer); } - for overflow_cell in &page_copy.overflow_cells { - scratch_cells - .insert(overflow_cell.index, to_static_buf(&overflow_cell.payload)); + // overflow_cells are stored in order - so we need to insert them in reverse order + for cell in page_copy.overflow_cells.iter().rev() { + scratch_cells.insert(cell.index, to_static_buf(&cell.payload)); } + // amount of cells for pages involved in split (distributed with naive greedy approach) + // if we have single overflow cell in a table leaf node - we still can have 3 split pages + // + // for example, if current page has 4 entries with size ~1/4 page size, and new cell has size ~page size + // then we will need 3 pages to distribute cells between them + let split_pages_cells_count = &mut write_info.split_pages_cells_count.borrow_mut(); + split_pages_cells_count.clear(); + let mut last_page_cells_count = 0; + let mut last_page_cells_size = 0; + for scratch_cell in scratch_cells.iter() { + let cell_size = scratch_cell.len() + 2; // + cell pointer size (u16) + if last_page_cells_size + cell_size > usable_space { + split_pages_cells_count.push(last_page_cells_count); + last_page_cells_count = 0; + last_page_cells_size = 0; + } + last_page_cells_count += 1; + last_page_cells_size += cell_size; + assert!(last_page_cells_size <= usable_space); + } + split_pages_cells_count.push(last_page_cells_count); + let new_pages_count = split_pages_cells_count.len(); + + debug!( + "splitting left={} new_pages={}, cells_count={:?}", + current_page.get().id, + new_pages_count - 1, + split_pages_cells_count + ); + *write_info.rightmost_pointer.borrow_mut() = page_copy.rightmost_pointer(); write_info.page_copy.replace(Some(page_copy)); - // allocate new pages and move cells to those new pages - // split procedure let page = current_page.get().contents.as_mut().unwrap(); + let page_type = page.page_type(); assert!( - matches!( - page.page_type(), - PageType::TableLeaf | PageType::TableInterior - ), - "indexes still not supported " + matches!(page_type, PageType::TableLeaf | PageType::TableInterior), + "indexes still not supported" ); - let right_page = self.allocate_page(page.page_type(), 0); - let right_page_id = right_page.get().id; - - write_info.new_pages.borrow_mut().clear(); - write_info.new_pages.borrow_mut().push(current_page.clone()); - write_info.new_pages.borrow_mut().push(right_page.clone()); - - debug!( - "splitting left={} right={}", - current_page.get().id, - right_page_id - ); + write_info.split_pages.borrow_mut().clear(); + write_info.split_pages.borrow_mut().push(current_page); + // allocate new pages + for _ in 1..new_pages_count { + let new_page = self.allocate_page(page_type, 0); + write_info.split_pages.borrow_mut().push(new_page); + } (WriteState::BalanceGetParentPage, Ok(CursorResult::Ok(()))) } @@ -1225,23 +1245,21 @@ impl BTreeCursor { } let write_info = self.state.write_info().unwrap(); - let mut new_pages = write_info.new_pages.borrow_mut(); + let mut split_pages = write_info.split_pages.borrow_mut(); + let split_pages_len = split_pages.len(); let scratch_cells = write_info.scratch_cells.borrow(); // reset pages - for page in new_pages.iter() { + for page in split_pages.iter() { assert!(page.is_dirty()); let contents = page.get().contents.as_mut().unwrap(); contents.write_u16(PAGE_HEADER_OFFSET_FIRST_FREEBLOCK, 0); contents.write_u16(PAGE_HEADER_OFFSET_CELL_COUNT, 0); - let db_header = RefCell::borrow(&self.pager.db_header); - let cell_content_area_start = - db_header.page_size - db_header.reserved_space as u16; contents.write_u16( PAGE_HEADER_OFFSET_CELL_CONTENT_AREA, - cell_content_area_start, + self.usable_space() as u16, ); contents.write_u8(PAGE_HEADER_OFFSET_FRAGMENTED_BYTES_COUNT, 0); @@ -1250,29 +1268,17 @@ impl BTreeCursor { } } - // distribute cells - let new_pages_len = new_pages.len(); - let cells_per_page = scratch_cells.len() / new_pages.len(); let mut current_cell_index = 0_usize; - let mut divider_cells_index = Vec::new(); /* index to scratch cells that will be used as dividers in order */ + /* index to scratch cells that will be used as dividers in order */ + let mut divider_cells_index = Vec::with_capacity(split_pages.len()); - debug!( - "balance_leaf::distribute(cells={}, cells_per_page={})", - scratch_cells.len(), - cells_per_page - ); + debug!("balance_leaf::distribute(cells={})", scratch_cells.len()); - for (i, page) in new_pages.iter_mut().enumerate() { + for (i, page) in split_pages.iter_mut().enumerate() { let page_id = page.get().id; let contents = page.get().contents.as_mut().unwrap(); - let last_page = i == new_pages_len - 1; - let cells_to_copy = if last_page { - // last cells is remaining pages if division was odd - scratch_cells.len() - current_cell_index - } else { - cells_per_page - }; + let cells_to_copy = write_info.split_pages_cells_count.borrow()[i]; debug!( "balance_leaf::distribute(page={}, cells_to_copy={})", page_id, cells_to_copy @@ -1288,6 +1294,7 @@ impl BTreeCursor { divider_cells_index.push(current_cell_index + cells_to_copy - 1); current_cell_index += cells_to_copy; } + let is_leaf = { let page = self.stack.top(); let page = page.get().contents.as_ref().unwrap(); @@ -1296,7 +1303,7 @@ impl BTreeCursor { // update rightmost pointer for each page if we are in interior page if !is_leaf { - for page in new_pages.iter_mut().take(new_pages_len - 1) { + for page in split_pages.iter_mut().take(split_pages_len - 1) { let contents = page.get().contents.as_mut().unwrap(); assert_eq!(contents.cell_count(), 1); @@ -1315,7 +1322,7 @@ impl BTreeCursor { contents.write_u32(PAGE_HEADER_OFFSET_RIGHTMOST_PTR, last_cell_pointer); } // last page right most pointer points to previous right most pointer before splitting - let last_page = new_pages.last().unwrap(); + let last_page = split_pages.last().unwrap(); let last_page_contents = last_page.get().contents.as_mut().unwrap(); last_page_contents.write_u32( PAGE_HEADER_OFFSET_RIGHTMOST_PTR, @@ -1326,7 +1333,7 @@ impl BTreeCursor { // insert dividers in parent // we can consider dividers the first cell of each page starting from the second page for (page_id_index, page) in - new_pages.iter_mut().take(new_pages_len - 1).enumerate() + split_pages.iter_mut().take(split_pages_len - 1).enumerate() { let contents = page.get().contents.as_mut().unwrap(); let divider_cell_index = divider_cells_index[page_id_index]; @@ -1372,7 +1379,7 @@ impl BTreeCursor { { // copy last page id to right pointer - let last_pointer = new_pages.last().unwrap().get().id as u32; + let last_pointer = split_pages.last().unwrap().get().id as u32; parent_contents.write_u32(right_pointer, last_pointer); } self.stack.pop(); From 8659dbba8e74c6bc7ef81de1d3a49b9bdbb6f29b Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 17:22:21 +0400 Subject: [PATCH 061/115] fix pointer structure in case of root split --- core/storage/btree.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 7af0945fe..02bfda01c 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1411,7 +1411,6 @@ impl BTreeCursor { let current_root = self.stack.top(); let current_root_contents = current_root.get().contents.as_ref().unwrap(); - let new_root_page_id = new_root_page.get().id; let new_root_page_contents = new_root_page.get().contents.as_mut().unwrap(); if is_page_1 { // Copy header @@ -1421,8 +1420,10 @@ impl BTreeCursor { .copy_from_slice(¤t_root_buf[0..DATABASE_HEADER_SIZE]); } // point new root right child to previous root - new_root_page_contents - .write_u32(PAGE_HEADER_OFFSET_RIGHTMOST_PTR, new_root_page_id as u32); + new_root_page_contents.write_u32( + PAGE_HEADER_OFFSET_RIGHTMOST_PTR, + current_root.get().id as u32, + ); new_root_page_contents.write_u16(PAGE_HEADER_OFFSET_CELL_COUNT, 0); } From 2fa3a1e6ae968d28d7d2348f0d13e8b18ec128dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sun, 9 Feb 2025 22:46:48 +0900 Subject: [PATCH 062/115] Apply lint --- bindings/java/rs_src/limbo_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/java/rs_src/limbo_db.rs b/bindings/java/rs_src/limbo_db.rs index 5f0dfb84c..65f6dafc0 100644 --- a/bindings/java/rs_src/limbo_db.rs +++ b/bindings/java/rs_src/limbo_db.rs @@ -100,7 +100,7 @@ pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB_connect0<'loca pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB_close0<'local>( _env: JNIEnv<'local>, _obj: JObject<'local>, - db_pointer: jlong + db_pointer: jlong, ) { LimboDB::drop(db_pointer); } From e8a585f87ae9ebfb15b8de0362b4c631c8e39965 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 18:09:43 +0400 Subject: [PATCH 063/115] adjust logging --- core/storage/btree.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 02bfda01c..d6663540b 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -745,13 +745,14 @@ impl BTreeCursor { // insert let overflow = { let contents = page.get().contents.as_mut().unwrap(); - debug!( - "insert_into_page(overflow, cell_count={})", - contents.cell_count() - ); - self.insert_into_cell(contents, cell_payload.as_slice(), cell_idx); - contents.overflow_cells.len() + let overflow_cells = contents.overflow_cells.len(); + debug!( + "insert_into_page(overflow, cell_count={}, overflow_cells={})", + contents.cell_count(), + overflow_cells + ); + overflow_cells }; let write_info = self .state From d2251b1dd1eba56e1b617a93ab82ee5fe9b32fa0 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 18:09:53 +0400 Subject: [PATCH 064/115] fix --- core/storage/btree.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index d6663540b..138a04371 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1148,16 +1148,17 @@ impl BTreeCursor { split_pages_cells_count.clear(); let mut last_page_cells_count = 0; let mut last_page_cells_size = 0; + let content_usable_space = usable_space - page_copy.header_size(); for scratch_cell in scratch_cells.iter() { let cell_size = scratch_cell.len() + 2; // + cell pointer size (u16) - if last_page_cells_size + cell_size > usable_space { + if last_page_cells_size + cell_size > content_usable_space { split_pages_cells_count.push(last_page_cells_count); last_page_cells_count = 0; last_page_cells_size = 0; } last_page_cells_count += 1; last_page_cells_size += cell_size; - assert!(last_page_cells_size <= usable_space); + assert!(last_page_cells_size <= content_usable_space); } split_pages_cells_count.push(last_page_cells_count); let new_pages_count = split_pages_cells_count.len(); From a62265eef4c97f8da30ae39768c25be6dd2ff5bd Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 18:10:19 +0400 Subject: [PATCH 065/115] hanle balancing cases when more than 1 level is affected --- core/storage/btree.rs | 56 +++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 138a04371..4656ab28e 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1053,40 +1053,38 @@ impl BTreeCursor { matches!(self.state, CursorState::Write(_)), "Cursor must be in balancing state" ); - let state = self - .state - .write_info() - .expect("must be balancing") - .state - .clone(); - match state { - WriteState::BalanceStart => { - let current_page = self.stack.top(); - { - // check if we don't need to balance - // don't continue if there are no overflow cells - let page = current_page.get().contents.as_mut().unwrap(); - if page.overflow_cells.is_empty() { - let write_info = self.state.mut_write_info().unwrap(); - write_info.state = WriteState::Finish; + loop { + let state = self.state.write_info().expect("must be balancing").state; + match state { + WriteState::BalanceStart => { + let current_page = self.stack.top(); + { + // check if we don't need to balance + // don't continue if there are no overflow cells + let page = current_page.get().contents.as_mut().unwrap(); + if page.overflow_cells.is_empty() { + let write_info = self.state.mut_write_info().unwrap(); + write_info.state = WriteState::Finish; + return Ok(CursorResult::Ok(())); + } + } + + if !self.stack.has_parent() { + self.balance_root(); return Ok(CursorResult::Ok(())); } + + let write_info = self.state.mut_write_info().unwrap(); + write_info.state = WriteState::BalanceNonRoot; + } + WriteState::BalanceNonRoot + | WriteState::BalanceGetParentPage + | WriteState::BalanceMoveUp => { + return_if_io!(self.balance_non_root()); } - if !self.stack.has_parent() { - self.balance_root(); - return Ok(CursorResult::Ok(())); - } - - let write_info = self.state.mut_write_info().unwrap(); - write_info.state = WriteState::BalanceNonRoot; - self.balance_non_root() + _ => unreachable!("invalid balance leaf state {:?}", state), } - WriteState::BalanceNonRoot - | WriteState::BalanceGetParentPage - | WriteState::BalanceMoveUp => self.balance_non_root(), - - _ => unreachable!("invalid balance leaf state {:?}", state), } } From 8e6569434a5d18bd088babb0cd87ad1e9ca73624 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 18:10:59 +0400 Subject: [PATCH 066/115] add fuzz --- core/storage/btree.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 4656ab28e..a0202c37a 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2471,6 +2471,24 @@ mod tests { pub fn btree_insert_fuzz_ex() { let _ = env_logger::init(); for sequence in [ + &[ + (777548915, 3364), + (639157228, 3796), + (709175417, 1214), + (390824637, 210), + (906124785, 1481), + (197677875, 1305), + (457946262, 3734), + (956825466, 592), + (835875722, 1334), + (649214013, 1250), + (531143011, 1788), + (765057993, 2351), + (510007766, 1349), + (884516059, 822), + (81604840, 2545), + ] + .as_slice(), &[ (293471650, 2452), (163608869, 627), From 9e4afd1d13af22e75ab712708bc6847a869a7107 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 18:36:26 +0400 Subject: [PATCH 067/115] relax assertion --- core/storage/btree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index a0202c37a..c3dc054ba 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1306,7 +1306,7 @@ impl BTreeCursor { for page in split_pages.iter_mut().take(split_pages_len - 1) { let contents = page.get().contents.as_mut().unwrap(); - assert_eq!(contents.cell_count(), 1); + assert!(contents.cell_count() >= 1); let last_cell = contents.cell_get( contents.cell_count() - 1, self.pager.clone(), From 9049c91863b346708141cf141f44fc31b4de521f Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 18:36:37 +0400 Subject: [PATCH 068/115] find cell in parent node --- core/storage/btree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index c3dc054ba..1ea2901db 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1371,7 +1371,7 @@ impl BTreeCursor { BTreeCell::TableInteriorCell(interior) => interior._rowid, _ => unreachable!(), }; - let parent_cell_idx = self.find_cell(contents, key); + let parent_cell_idx = self.find_cell(&parent_contents, key); self.insert_into_cell(parent_contents, cell_payload, parent_cell_idx); // self.drop_cell(*page, 0); } From 6c40f52fc8b6370217418b79033a6cbdb2851cf2 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 18:37:04 +0400 Subject: [PATCH 069/115] separate fuzz tests in categories --- core/storage/btree.rs | 83 +++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 22 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 1ea2901db..9db523a59 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2469,7 +2469,6 @@ mod tests { #[test] pub fn btree_insert_fuzz_ex() { - let _ = env_logger::init(); for sequence in [ &[ (777548915, 3364), @@ -2540,44 +2539,84 @@ mod tests { } } - #[test] - pub fn btree_insert_fuzz_run() { - let _ = env_logger::init(); - let mut rng = ChaCha8Rng::seed_from_u64(0); - for _ in 0..128 { + fn rng_from_time() -> (ChaCha8Rng, u64) { + let seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + let rng = ChaCha8Rng::seed_from_u64(seed); + (rng, seed) + } + + fn btree_insert_fuzz_run( + attempts: usize, + inserts: usize, + size: impl Fn(&mut ChaCha8Rng) -> usize, + ) { + let (mut rng, seed) = rng_from_time(); + log::info!("super seed: {}", seed); + for _ in 0..attempts { let (pager, root_page) = empty_btree(); let mut cursor = BTreeCursor::new(pager.clone(), root_page); let mut keys = Vec::new(); let seed = rng.next_u64(); log::info!("seed: {}", seed); let mut rng = ChaCha8Rng::seed_from_u64(seed); - for _ in 0..16 { - let size = (rng.next_u64() % 4096) as usize; + for _ in 0..inserts { + let size = size(&mut rng); let key = (rng.next_u64() % (1 << 30)) as i64; keys.push(key); log::info!("INSERT INTO t VALUES ({}, randomblob({}));", key, size); let key = OwnedValue::Integer(key); let value = Record::new(vec![OwnedValue::Blob(Rc::new(vec![0; size]))]); cursor.insert(&key, &value, false).unwrap(); - log::info!( - "=========== btree ===========\n{}\n\n", - format_btree(pager.clone(), root_page, 0) + } + log::info!( + "=========== btree ===========\n{}\n\n", + format_btree(pager.clone(), root_page, 0) + ); + for key in keys.iter() { + let seek_key = SeekKey::TableRowId(*key as u64); + assert!( + matches!( + cursor.seek(seek_key, SeekOp::EQ).unwrap(), + CursorResult::Ok(true) + ), + "key {} is not found", + key ); - for key in keys.iter() { - let seek_key = SeekKey::TableRowId(*key as u64); - assert!( - matches!( - cursor.seek(seek_key, SeekOp::EQ).unwrap(), - CursorResult::Ok(true) - ), - "key {} is not found", - key - ); - } } } } + #[test] + pub fn btree_insert_fuzz_run_equal_size() { + for size in 1..8 { + log::info!("======= size:{} =======", size); + btree_insert_fuzz_run(2, 1024, |_| size); + } + } + + #[test] + pub fn btree_insert_fuzz_run_random() { + btree_insert_fuzz_run(128, 16, |rng| (rng.next_u32() % 4096) as usize); + } + + #[test] + pub fn btree_insert_fuzz_run_small() { + btree_insert_fuzz_run(1, 1024, |rng| (rng.next_u32() % 128) as usize); + } + + #[test] + pub fn btree_insert_fuzz_run_big() { + btree_insert_fuzz_run(64, 32, |rng| 3 * 1024 + (rng.next_u32() % 1024) as usize); + } + + #[test] + pub fn btree_insert_fuzz_run_overflow() { + btree_insert_fuzz_run(64, 32, |rng| (rng.next_u32() % 32 * 1024) as usize); + } + #[allow(clippy::arc_with_non_send_sync)] fn setup_test_env(database_size: u32) -> (Rc, Rc>) { let page_size = 512; From a59589844d0c0e0ea332cbef0c34bf2600a89239 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 19:19:36 +0400 Subject: [PATCH 070/115] fix insertion to the parent --- core/storage/btree.rs | 41 +++++++++++++---------------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 9db523a59..5e5862699 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2,7 +2,7 @@ use log::debug; use crate::storage::pager::Pager; use crate::storage::sqlite3_ondisk::{ - read_btree_cell, read_varint, write_varint, BTreeCell, DatabaseHeader, PageContent, PageType, + read_btree_cell, read_varint, BTreeCell, DatabaseHeader, PageContent, PageType, TableInteriorCell, TableLeafCell, }; @@ -1348,33 +1348,18 @@ impl BTreeCursor { self.usable_space(), )?; - if is_leaf { - // create a new divider cell and push - let key = match cell { - BTreeCell::TableLeafCell(leaf) => leaf._rowid, - _ => unreachable!(), - }; - let mut divider_cell = Vec::new(); - divider_cell.extend_from_slice(&(page.get().id as u32).to_be_bytes()); - divider_cell.extend(std::iter::repeat(0).take(9)); - let n = write_varint(&mut divider_cell.as_mut_slice()[4..], key); - divider_cell.truncate(4 + n); - let parent_cell_idx = self.find_cell(parent_contents, key); - self.insert_into_cell( - parent_contents, - divider_cell.as_slice(), - parent_cell_idx, - ); - } else { - // move cell - let key = match cell { - BTreeCell::TableInteriorCell(interior) => interior._rowid, - _ => unreachable!(), - }; - let parent_cell_idx = self.find_cell(&parent_contents, key); - self.insert_into_cell(parent_contents, cell_payload, parent_cell_idx); - // self.drop_cell(*page, 0); - } + let key = match cell { + BTreeCell::TableLeafCell(TableLeafCell { _rowid, .. }) + | BTreeCell::TableInteriorCell(TableInteriorCell { _rowid, .. }) => _rowid, + _ => unreachable!(), + }; + + let mut divider_cell = Vec::with_capacity(4 + 9); // 4 - page id, 9 - max length of varint + divider_cell.extend_from_slice(&(page.get().id as u32).to_be_bytes()); + write_varint_to_vec(key, &mut divider_cell); + + let parent_cell_idx = self.find_cell(parent_contents, key); + self.insert_into_cell(parent_contents, ÷r_cell, parent_cell_idx); } { From e23ea35993667b0843dc6e9887324b8d6a8df9bf Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 19:20:37 +0400 Subject: [PATCH 071/115] add simple B-tree validation func --- core/storage/btree.rs | 67 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 5e5862699..8c2244794 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2369,6 +2369,70 @@ mod tests { use std::cell::RefCell; use std::sync::Arc; + fn validate_btree(pager: Rc, page_idx: usize) -> (usize, bool) { + let cursor = BTreeCursor::new(pager.clone(), page_idx); + let page = pager.read_page(page_idx).unwrap(); + let page = page.get(); + let contents = page.contents.as_ref().unwrap(); + let page_type = contents.page_type(); + let mut previous_key = None; + let mut valid = true; + let mut depth = None; + for cell_idx in 0..contents.cell_count() { + let cell = contents + .cell_get( + cell_idx, + pager.clone(), + cursor.payload_overflow_threshold_max(page_type), + cursor.payload_overflow_threshold_min(page_type), + cursor.usable_space(), + ) + .unwrap(); + let current_depth = match cell { + BTreeCell::TableLeafCell(..) => 1, + BTreeCell::TableInteriorCell(TableInteriorCell { + _left_child_page, .. + }) => { + let (child_depth, child_valid) = + validate_btree(pager.clone(), _left_child_page as usize); + valid &= child_valid; + child_depth + } + _ => panic!("unsupported btree cell: {:?}", cell), + }; + depth = Some(depth.unwrap_or(current_depth + 1)); + if depth != Some(current_depth + 1) { + log::error!("depth is different for child of page {}", page_idx); + valid = false; + } + match cell { + BTreeCell::TableInteriorCell(TableInteriorCell { _rowid, .. }) + | BTreeCell::TableLeafCell(TableLeafCell { _rowid, .. }) => { + if previous_key.is_some() && previous_key.unwrap() >= _rowid { + log::error!( + "keys are in bad order: prev={:?}, current={}", + previous_key, + _rowid + ); + valid = false; + } + previous_key = Some(_rowid); + } + _ => panic!("unsupported btree cell: {:?}", cell), + } + } + if let Some(right) = contents.rightmost_pointer() { + let (right_depth, right_valid) = validate_btree(pager.clone(), right as usize); + valid &= right_valid; + depth = Some(depth.unwrap_or(right_depth + 1)); + if depth != Some(right_depth + 1) { + log::error!("depth is different for child of page {}", page_idx); + valid = false; + } + } + (depth.unwrap(), valid) + } + fn format_btree(pager: Rc, page_idx: usize, depth: usize) -> String { let cursor = BTreeCursor::new(pager.clone(), page_idx); let page = pager.read_page(page_idx).unwrap(); @@ -2560,6 +2624,9 @@ mod tests { "=========== btree ===========\n{}\n\n", format_btree(pager.clone(), root_page, 0) ); + if matches!(validate_btree(pager.clone(), root_page), (_, false)) { + panic!("invalid btree"); + } for key in keys.iter() { let seek_key = SeekKey::TableRowId(*key as u64); assert!( From bc289d314a59b0eea6d3c56cca2d64238655b711 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 19:20:48 +0400 Subject: [PATCH 072/115] adjust test a bit --- core/storage/btree.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 8c2244794..5ded4ee76 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2611,11 +2611,16 @@ mod tests { let seed = rng.next_u64(); log::info!("seed: {}", seed); let mut rng = ChaCha8Rng::seed_from_u64(seed); - for _ in 0..inserts { + for insert_id in 0..inserts { let size = size(&mut rng); let key = (rng.next_u64() % (1 << 30)) as i64; keys.push(key); - log::info!("INSERT INTO t VALUES ({}, randomblob({}));", key, size); + log::info!( + "INSERT INTO t VALUES ({}, randomblob({})); -- {}", + key, + size, + insert_id + ); let key = OwnedValue::Integer(key); let value = Record::new(vec![OwnedValue::Blob(Rc::new(vec![0; size]))]); cursor.insert(&key, &value, false).unwrap(); From 32b5b0d0197f1a75e866ccfd634ebf981bde5cc2 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 19:26:03 +0400 Subject: [PATCH 073/115] introduce additional condition for cells distribution in order to avoid almost empty pages --- core/storage/btree.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 5ded4ee76..1a537767d 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1137,7 +1137,12 @@ impl BTreeCursor { scratch_cells.insert(cell.index, to_static_buf(&cell.payload)); } - // amount of cells for pages involved in split (distributed with naive greedy approach) + // amount of cells for pages involved in split + // the algorithm accumulate cells in greedy manner with 2 conditions for split: + // 1. new cell will overflow single cell (accumulated + new > usable_space - header_size) + // 2. accumulated size already reach >50% of content_usable_size + // second condition is necessary, otherwise in case of small cells we will create a lot of almost empty pages + // // if we have single overflow cell in a table leaf node - we still can have 3 split pages // // for example, if current page has 4 entries with size ~1/4 page size, and new cell has size ~page size @@ -1149,7 +1154,9 @@ impl BTreeCursor { let content_usable_space = usable_space - page_copy.header_size(); for scratch_cell in scratch_cells.iter() { let cell_size = scratch_cell.len() + 2; // + cell pointer size (u16) - if last_page_cells_size + cell_size > content_usable_space { + if last_page_cells_size + cell_size > content_usable_space + || 2 * last_page_cells_size > content_usable_space + { split_pages_cells_count.push(last_page_cells_count); last_page_cells_count = 0; last_page_cells_size = 0; From 1b9772e9ad04f4c3f0b811e73a251deba7c4ae72 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Sun, 9 Feb 2025 19:36:14 +0400 Subject: [PATCH 074/115] fix clippy --- core/storage/btree.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 1a537767d..bafc89dd3 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2503,6 +2503,7 @@ mod tests { let db_header = DatabaseHeader::default(); let page_size = db_header.page_size as usize; + #[allow(clippy::arc_with_non_send_sync)] let io: Arc = Arc::new(MemoryIO::new().unwrap()); let io_file = io.open_file("test.db", OpenFlags::Create, false).unwrap(); let page_io = Rc::new(FileStorage::new(io_file)); From eaea02c56709273a432f5bf8d972f9c853871f0d Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Sun, 9 Feb 2025 18:05:03 +0200 Subject: [PATCH 075/115] Fix a handful of typos --- CHANGELOG.md | 2 +- README.md | 2 +- bindings/go/limbo_test.go | 2 +- bindings/go/rs_src/statement.rs | 4 ++-- bindings/rust/src/params.rs | 8 ++++---- core/storage/btree.rs | 10 +++++----- core/translate/expr.rs | 6 +++--- core/translate/main_loop.rs | 6 +++--- core/translate/mod.rs | 2 +- core/util.rs | 2 +- core/vdbe/mod.rs | 4 ++-- extensions/time/src/lib.rs | 2 +- simulator/generation/mod.rs | 6 +++--- simulator/main.rs | 2 +- simulator/runner/cli.rs | 2 +- simulator/runner/env.rs | 2 +- testing/json.test | 2 +- testing/scalar-functions.test | 4 ++-- tests/integration/fuzz/mod.rs | 2 +- vendored/sqlite3-parser/src/parser/ast/mod.rs | 2 +- 20 files changed, 36 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 847deec42..e8189a2e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -218,7 +218,7 @@ * Add support for iif() function (Alex Miller) -* Add suport for last_insert_rowid() function (Krishna Vishal) +* Add support for last_insert_rowid() function (Krishna Vishal) * Add support JOIN USING and NATURAL JOIN (Jussi Saurio) diff --git a/README.md b/README.md index 480933581..8f2665ef0 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Limbo is a _work-in-progress_, in-process OLTP database engine library written i * **Language bindings** for JavaScript/WebAssembly, Rust, Go, Python, and [Java](bindings/java) * **OS support** for Linux, macOS, and Windows -In the future, we will be also workin on: +In the future, we will be also working on: * **Integrated vector search** for embeddings and vector similarity. * **`BEGIN CONCURRENT`** for improved write throughput. diff --git a/bindings/go/limbo_test.go b/bindings/go/limbo_test.go index 31b1fc3b4..9527faa5f 100644 --- a/bindings/go/limbo_test.go +++ b/bindings/go/limbo_test.go @@ -89,7 +89,7 @@ func TestFunctions(t *testing.T) { } _, err = stmt.Exec(60, "TestFunction", 400) if err != nil { - t.Fatalf("Error executing statment with arguments: %v", err) + t.Fatalf("Error executing statement with arguments: %v", err) } stmt.Close() stmt, err = conn.Prepare("SELECT baz FROM test where foo = ?") diff --git a/bindings/go/rs_src/statement.rs b/bindings/go/rs_src/statement.rs index e8cec0618..d068f01c6 100644 --- a/bindings/go/rs_src/statement.rs +++ b/bindings/go/rs_src/statement.rs @@ -116,12 +116,12 @@ pub extern "C" fn stmt_query( let val = arg.to_value(&mut pool); statement.bind_at(NonZero::new(i + 1).unwrap(), val); } - // ownership of the statement is transfered to the LimboRows object. + // ownership of the statement is transferred to the LimboRows object. LimboRows::new(statement, stmt.conn).to_ptr() } pub struct LimboStatement<'conn> { - /// If 'query' is ran on the statement, ownership is transfered to the LimboRows object + /// If 'query' is ran on the statement, ownership is transferred to the LimboRows object pub statement: Option, pub conn: &'conn mut LimboConn, pub err: Option, diff --git a/bindings/rust/src/params.rs b/bindings/rust/src/params.rs index c15b6adb5..a627cdd3f 100644 --- a/bindings/rust/src/params.rs +++ b/bindings/rust/src/params.rs @@ -18,7 +18,7 @@ use sealed::Sealed; /// /// Many functions in this library let you pass parameters to libsql. Doing this /// lets you avoid any risk of SQL injection, and is simpler than escaping -/// things manually. These functions generally contain some paramter that generically +/// things manually. These functions generally contain some parameter that generically /// accepts some implementation this trait. /// /// # Positional parameters @@ -29,7 +29,7 @@ use sealed::Sealed; /// by doing `(1, "foo")`. /// - For hetergeneous parameter lists of 16 or greater, the [`limbo_libsql::params!`] is supported /// by doing `limbo_libsql::params![1, "foo"]`. -/// - For homogeneous paramter types (where they are all the same type), const arrays are +/// - For homogeneous parameter types (where they are all the same type), const arrays are /// supported by doing `[1, 2, 3]`. /// /// # Example (positional) @@ -58,13 +58,13 @@ use sealed::Sealed; /// # } /// ``` /// -/// # Named paramters +/// # Named parameters /// /// - For heterogeneous parameter lists of 16 or less items a tuple syntax is supported /// by doing `(("key1", 1), ("key2", "foo"))`. /// - For hetergeneous parameter lists of 16 or greater, the [`limbo_libsql::params!`] is supported /// by doing `limbo_libsql::named_params!["key1": 1, "key2": "foo"]`. -/// - For homogeneous paramter types (where they are all the same type), const arrays are +/// - For homogeneous parameter types (where they are all the same type), const arrays are /// supported by doing `[("key1", 1), ("key2, 2), ("key3", 3)]`. /// /// # Example (named) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 4a963581b..c5417a871 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -156,7 +156,7 @@ pub struct BTreeCursor { /// current_page represents the current page being used in the tree and current_page - 1 would be /// the parent. Using current_page + 1 or higher is undefined behaviour. struct PageStack { - /// Pointer to the currenet page being consumed + /// Pointer to the current page being consumed current_page: RefCell, /// List of pages in the stack. Root page will be in index 0 stack: RefCell<[Option; BTCURSOR_MAX_DEPTH + 1]>, @@ -983,7 +983,7 @@ impl BTreeCursor { db_header: Ref, ) -> Result { // NOTE: freelist is in ascending order of keys and pc - // unused_space is reserved bytes at the end of page, therefore we must substract from maxpc + // unused_space is reserved bytes at the end of page, therefore we must subtract from maxpc let mut free_list_pointer_addr = 1; let mut pc = page_ref.first_freeblock() as usize; @@ -1115,7 +1115,7 @@ impl BTreeCursor { debug!("balance_non_root(page={})", current_page.get().id); // Copy of page used to reference cell bytes. - // This needs to be saved somewhere safe so taht references still point to here, + // This needs to be saved somewhere safe so that references still point to here, // this will be store in write_info below let page_copy = current_page.get().contents.as_ref().unwrap().clone(); @@ -1420,7 +1420,7 @@ impl BTreeCursor { new_root_page_contents.write_u16(PAGE_HEADER_OFFSET_CELL_COUNT, 0); } - /* swap splitted page buffer with new root buffer so we don't have to update page idx */ + /* swap split page buffer with new root buffer so we don't have to update page idx */ { let (root_id, child_id, child) = { let page_ref = self.stack.top(); @@ -2137,7 +2137,7 @@ impl BTreeCursor { 1 => PageType::TableLeaf, 2 => PageType::IndexLeaf, _ => unreachable!( - "wrong create table falgs, should be 1 for table and 2 for index, got {}", + "wrong create table flags, should be 1 for table and 2 for index, got {}", flags, ), }; diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 36acfa935..63734b794 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -173,7 +173,7 @@ macro_rules! expect_arguments_even { ); }; // The only function right now that requires an even number is `json_object` and it allows - // to have no arguments, so thats why in this macro we do not bail with teh `function with no arguments` error + // to have no arguments, so thats why in this macro we do not bail with the `function with no arguments` error args }}; } @@ -476,7 +476,7 @@ pub fn translate_condition_expr( ); } else { crate::bail_parse_error!( - "parenthesized condtional should have exactly one expression" + "parenthesized conditional should have exactly one expression" ); } } @@ -1517,7 +1517,7 @@ pub fn translate_expr( ScalarFunc::TotalChanges => { if args.is_some() { crate::bail_parse_error!( - "{} fucntion with more than 0 arguments", + "{} function with more than 0 arguments", srf.to_string() ); } diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs index 3b918e9ea..32c2a2d2d 100644 --- a/core/translate/main_loop.rs +++ b/core/translate/main_loop.rs @@ -401,10 +401,10 @@ pub fn open_loop( program.resolve_label(loop_start, program.offset()); // TODO: We are currently only handling ascending indexes. - // For conditions like index_key > 10, we have already seeked to the first key greater than 10, and can just scan forward. + // For conditions like index_key > 10, we have already sought to the first key greater than 10, and can just scan forward. // For conditions like index_key < 10, we are at the beginning of the index, and will scan forward and emit IdxGE(10) with a conditional jump to the end. - // For conditions like index_key = 10, we have already seeked to the first key greater than or equal to 10, and can just scan forward and emit IdxGT(10) with a conditional jump to the end. - // For conditions like index_key >= 10, we have already seeked to the first key greater than or equal to 10, and can just scan forward. + // For conditions like index_key = 10, we have already sought to the first key greater than or equal to 10, and can just scan forward and emit IdxGT(10) with a conditional jump to the end. + // For conditions like index_key >= 10, we have already sought to the first key greater than or equal to 10, and can just scan forward. // For conditions like index_key <= 10, we are at the beginning of the index, and will scan forward and emit IdxGT(10) with a conditional jump to the end. // For conditions like index_key != 10, TODO. probably the optimal way is not to use an index at all. // diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 7ff780e2b..5f9e19866 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -496,7 +496,7 @@ fn translate_create_table( program.resolve_label(parse_schema_label, program.offset()); // TODO: SetCookie // - // TODO: remove format, it sucks for performance but is convinient + // TODO: remove format, it sucks for performance but is convenient let parse_schema_where_clause = format!("tbl_name = '{}' AND type != 'trigger'", tbl_name); program.emit_insn(Insn::ParseSchema { db: sqlite_schema_cursor_id, diff --git a/core/util.rs b/core/util.rs index 654951700..0d16f7794 100644 --- a/core/util.rs +++ b/core/util.rs @@ -499,7 +499,7 @@ pub mod tests { } #[test] - fn test_expressions_equivalent_multiplicaiton() { + fn test_expressions_equivalent_multiplication() { let expr1 = Expr::Binary( Box::new(Expr::Literal(Literal::Numeric("42.0".to_string()))), Multiply, diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 3ccfdc1e3..44b38564b 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -2425,7 +2425,7 @@ impl Program { let pc: u32 = pc .try_into() .unwrap_or_else(|_| panic!("EndCoroutine: pc overflow: {}", pc)); - state.pc = pc - 1; // yield jump is always next to yield. Here we substract 1 to go back to yield instruction + state.pc = pc - 1; // yield jump is always next to yield. Here we subtract 1 to go back to yield instruction } else { unreachable!(); } @@ -2652,7 +2652,7 @@ impl Program { todo!("temp databases not implemented yet"); } // SQLite returns "0" on an empty database, and 2 on the first insertion, - // so we'll mimick that behavior. + // so we'll mimic that behavior. let mut pages = pager.db_header.borrow().database_size.into(); if pages == 1 { pages = 0; diff --git a/extensions/time/src/lib.rs b/extensions/time/src/lib.rs index 5c4d5a383..2e1b8efd2 100644 --- a/extensions/time/src/lib.rs +++ b/extensions/time/src/lib.rs @@ -816,7 +816,7 @@ fn time_until(args: &[Value]) -> Value { time_sub_internal(t, now) } -// Rouding +// Rounding #[scalar(name = "time_trunc", alias = "date_trunc")] fn time_trunc(args: &[Value]) -> Value { diff --git a/simulator/generation/mod.rs b/simulator/generation/mod.rs index ac7defd54..565f65297 100644 --- a/simulator/generation/mod.rs +++ b/simulator/generation/mod.rs @@ -33,11 +33,11 @@ pub trait ArbitraryFromMaybe { } /// Frequency is a helper function for composing different generators with different frequency -/// of occurences. +/// of occurrences. /// The type signature for the `N` parameter is a bit complex, but it /// roughly corresponds to a type that can be summed, compared, subtracted and sampled, which are /// the operations we require for the implementation. -// todo: switch to a simpler type signature that can accomodate all integer and float types, which +// todo: switch to a simpler type signature that can accommodate all integer and float types, which // should be enough for our purposes. pub(crate) fn frequency< 'a, @@ -61,7 +61,7 @@ pub(crate) fn frequency< unreachable!() } -/// one_of is a helper function for composing different generators with equal probability of occurence. +/// one_of is a helper function for composing different generators with equal probability of occurrence. pub(crate) fn one_of<'a, T, R: Rng>(choices: Vec T + 'a>>, rng: &mut R) -> T { let index = rng.gen_range(0..choices.len()); choices[index](rng) diff --git a/simulator/main.rs b/simulator/main.rs index 2eb463529..e8a5b34cf 100644 --- a/simulator/main.rs +++ b/simulator/main.rs @@ -429,7 +429,7 @@ fn setup_simulation( let mut env = SimulatorEnv::new(seed, cli_opts, db_path); // todo: the loading works correctly because of a hacky decision - // Rigth now, the plan generation is the only point we use the rng, so the environment doesn't + // Right now, the plan generation is the only point we use the rng, so the environment doesn't // even need it. In the future, especially with multi-connections and multi-threading, we might // use the RNG for more things such as scheduling, so this assumption will fail. When that happens, // we'll need to reachitect this logic by saving and loading RNG state. diff --git a/simulator/runner/cli.rs b/simulator/runner/cli.rs index 93a14849f..aa3697b27 100644 --- a/simulator/runner/cli.rs +++ b/simulator/runner/cli.rs @@ -64,7 +64,7 @@ impl SimulatorCLI { return Err("Minimum size cannot be greater than maximum size".to_string()); } - // Make sure uncompatible options are not set + // Make sure incompatible options are not set if self.shrink && self.doublecheck { return Err("Cannot use shrink and doublecheck at the same time".to_string()); } diff --git a/simulator/runner/env.rs b/simulator/runner/env.rs index 2813b80e8..83f5180d3 100644 --- a/simulator/runner/env.rs +++ b/simulator/runner/env.rs @@ -43,7 +43,7 @@ impl SimulatorEnv { let opts = SimulatorOpts { ticks: rng.gen_range(cli_opts.minimum_size..=cli_opts.maximum_size), max_connections: 1, // TODO: for now let's use one connection as we didn't implement - // correct transactions procesing + // correct transactions processing max_tables: rng.gen_range(0..128), create_percent, read_percent, diff --git a/testing/json.test b/testing/json.test index 0bfec6247..7a1890369 100755 --- a/testing/json.test +++ b/testing/json.test @@ -759,7 +759,7 @@ do_execsql_test json-patch-add-all-dup-keys-from-patch { '{"z":{}, "z":5, "z":100}' ); } {{{"x":100,"x":200,"z":100}}} -do_execsql_test json-patch-first-occurance-patch { +do_execsql_test json-patch-first-occurrence-patch { select json_patch('{"x":100,"x":200}','{"x":{}, "x":5, "x":100}'); } {{{"x":100,"x":200}}} do_execsql_test json-patch-complex-nested-dup-keys { diff --git a/testing/scalar-functions.test b/testing/scalar-functions.test index f04fa1765..77694863c 100755 --- a/testing/scalar-functions.test +++ b/testing/scalar-functions.test @@ -407,7 +407,7 @@ do_execsql_test length-text { SELECT length('limbo'); } {5} -do_execsql_test lenght-text-utf8-chars { +do_execsql_test length-text-utf8-chars { SELECT length('ąłóżźć'); } {6} @@ -431,7 +431,7 @@ do_execsql_test octet-length-text { SELECT length('limbo'); } {5} -do_execsql_test octet-lenght-text-utf8-chars { +do_execsql_test octet-length-text-utf8-chars { SELECT octet_length('ąłóżźć'); } {12} diff --git a/tests/integration/fuzz/mod.rs b/tests/integration/fuzz/mod.rs index 8179dfde4..af7ae61e9 100644 --- a/tests/integration/fuzz/mod.rs +++ b/tests/integration/fuzz/mod.rs @@ -251,7 +251,7 @@ mod tests { .option_w(bin_op, 1.0) .option_w(paren, 1.0) .option_w(scalar, 1.0) - // unfortunatelly, sqlite behaves weirdly when IS operator is used with TRUE/FALSE constants + // unfortunately, sqlite behaves weirdly when IS operator is used with TRUE/FALSE constants // e.g. 8 IS TRUE == 1 (although 8 = TRUE == 0) // so, we do not use TRUE/FALSE constants as they will produce diff with sqlite results .options_str(["1", "0", "NULL", "2.0", "1.5", "-0.5", "-2.0", "(1 / 0)"]) diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index 3ea9d5992..3db759eed 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -1691,7 +1691,7 @@ pub struct TriggerCmdInsert { pub col_names: Option, /// `SELECT` or `VALUES` pub select: Box