From 2faaa9f719fb852bd3511dcc0154e3a92715e4b7 Mon Sep 17 00:00:00 2001 From: krishvishal Date: Tue, 4 Feb 2025 04:45:34 +0530 Subject: [PATCH 1/9] Add fix for paser not stopping at first error encounter. Fix is to track the state of encountering error in the `Parser` struct and set it to true whereever/whenever we encounter an error. And the beginning of the next() we return `Ok(None)` to signal to `FallibleIterator` that we should stop parsing. Fixes https://github.com/tursodatabase/limbo/issues/865 --- vendored/sqlite3-parser/src/lexer/sql/mod.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/vendored/sqlite3-parser/src/lexer/sql/mod.rs b/vendored/sqlite3-parser/src/lexer/sql/mod.rs index dc672e9f5..862d4c4ee 100644 --- a/vendored/sqlite3-parser/src/lexer/sql/mod.rs +++ b/vendored/sqlite3-parser/src/lexer/sql/mod.rs @@ -30,6 +30,7 @@ pub struct Parser<'input> { scanner: Scanner, /// lemon parser parser: yyParser<'input>, + had_error: bool, } impl<'input> Parser<'input> { @@ -43,12 +44,14 @@ impl<'input> Parser<'input> { input, scanner, parser, + had_error: false, } } /// Parse new `input` pub fn reset(&mut self, input: &'input [u8]) { self.input = input; self.scanner.reset(); + self.had_error = false; } /// Current line position in input pub fn line(&self) -> u64 { @@ -182,6 +185,10 @@ impl FallibleIterator for Parser<'_> { fn next(&mut self) -> Result, Error> { //print!("line: {}, column: {}: ", self.scanner.line(), self.scanner.column()); + // if we have already encountered an error, return None to signal that to fallible_iterator that we are done parsing + if self.had_error { + return Ok(None); + } self.parser.ctx.reset(); let mut last_token_parsed = TK_EOF; let mut eof = false; @@ -197,6 +204,7 @@ impl FallibleIterator for Parser<'_> { if token_type == TK_ILLEGAL { // break out of parsing loop and return error self.parser.sqlite3ParserFinalize(); + self.had_error = true; return Err(Error::UnrecognizedToken( Some((self.scanner.line(), self.scanner.column())), Some(start.into()), @@ -242,12 +250,18 @@ impl FallibleIterator for Parser<'_> { self.parser .sqlite3Parser(TK_SEMI, sentinel(self.input.len())) ); + if self.parser.ctx.error().is_some() { + self.had_error = true; + } } try_with_position!( self.scanner, self.parser .sqlite3Parser(TK_EOF, sentinel(self.input.len())) ); + if self.parser.ctx.error().is_some() { + self.had_error = true; + } } self.parser.sqlite3ParserFinalize(); if let Some(e) = self.parser.ctx.error() { @@ -256,6 +270,7 @@ impl FallibleIterator for Parser<'_> { Some((self.scanner.line(), self.scanner.column())), Some((self.offset() - 1).into()), ); + self.had_error = true; return Err(err); } let cmd = self.parser.ctx.cmd(); @@ -266,6 +281,7 @@ impl FallibleIterator for Parser<'_> { Some((self.scanner.line(), self.scanner.column())), Some((self.offset() - 1).into()), ); + self.had_error = true; return Err(err); } } From a88a5353a35a7006ed5ee0d7af158ac86dc716c9 Mon Sep 17 00:00:00 2001 From: krishvishal Date: Tue, 4 Feb 2025 04:51:38 +0530 Subject: [PATCH 2/9] Add unit test for checking `SELECT FROM foo;` returns only one error and stops parsing. --- vendored/sqlite3-parser/src/lexer/sql/test.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vendored/sqlite3-parser/src/lexer/sql/test.rs b/vendored/sqlite3-parser/src/lexer/sql/test.rs index 46cca8af3..2ba066bdd 100644 --- a/vendored/sqlite3-parser/src/lexer/sql/test.rs +++ b/vendored/sqlite3-parser/src/lexer/sql/test.rs @@ -338,6 +338,21 @@ fn qualified_table_name_within_triggers() { ); } +#[test] +fn select_from_error_stops_at_first_error() { + let mut parser = Parser::new(b"SELECT FROM foo;"); + + // First next() call should return the first syntax error + let err = parser.next().unwrap_err(); + assert!(matches!(err, Error::ParserError(_, _, _))); + + // Second next() call should return Ok(None) since parsing should have stopped + assert_eq!(parser.next().unwrap(), None); + + // Third next() call should also return Ok(None) + assert_eq!(parser.next().unwrap(), None); +} + #[test] fn indexed_by_clause_within_triggers() { expect_parser_err_msg( From 575a524d049de3d560932e67fce974a633f66fe2 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Fri, 7 Feb 2025 08:37:16 -0500 Subject: [PATCH 3/9] Replace hashmap for io_uring pending ops with static array --- Cargo.lock | 2 ++ core/io/io_uring.rs | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d26da84a6..986baa6fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1735,6 +1735,8 @@ dependencies = [ "notify", "rand 0.8.5", "rand_chacha 0.3.1", + "regex", + "regex-syntax", "serde", "serde_json", "tempfile", diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index 1598debfa..eef5e523d 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -4,7 +4,6 @@ use log::{debug, trace}; use rustix::fs::{self, FlockOperation, OFlags}; use rustix::io_uring::iovec; use std::cell::RefCell; -use std::collections::HashMap; use std::fmt; use std::io::ErrorKind; use std::os::fd::AsFd; @@ -40,7 +39,7 @@ pub struct UringIO { struct WrappedIOUring { ring: io_uring::IoUring, pending_ops: usize, - pub pending: HashMap>, + pub pending: [Option>; MAX_IOVECS as usize + 1], key: u64, } @@ -63,7 +62,7 @@ impl UringIO { ring: WrappedIOUring { ring, pending_ops: 0, - pending: HashMap::new(), + pending: [const { None }; MAX_IOVECS as usize + 1], key: 0, }, iovecs: [iovec { @@ -92,7 +91,7 @@ impl InnerUringIO { impl WrappedIOUring { fn submit_entry(&mut self, entry: &io_uring::squeue::Entry, c: Rc) { trace!("submit_entry({:?})", entry); - self.pending.insert(entry.get_user_data(), c); + self.pending[entry.get_user_data() as usize] = Some(c); unsafe { self.ring .submission() @@ -124,6 +123,11 @@ impl WrappedIOUring { fn get_key(&mut self) -> u64 { self.key += 1; + if self.key == MAX_IOVECS as u64 { + let key = self.key; + self.key = 0; + return key; + } self.key } } @@ -175,10 +179,11 @@ impl IO for UringIO { ))); } { - let c = ring.pending.get(&cqe.user_data()).unwrap().clone(); - c.complete(cqe.result()); + if let Some(c) = ring.pending[cqe.user_data() as usize].as_ref() { + c.complete(cqe.result()); + } } - ring.pending.remove(&cqe.user_data()); + ring.pending[cqe.user_data() as usize] = None; } Ok(()) } From cc72439032b1b569ed92ab0048abf18fcefb6887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=84=A0=EC=9A=B0?= Date: Sat, 8 Feb 2025 12:09:17 +0900 Subject: [PATCH 4/9] Add java section in README.md --- Cargo.lock | 18 +++++++++--------- README.md | 5 +++++ bindings/java/Cargo.toml | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d26da84a6..92df9db99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1394,15 +1394,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" -[[package]] -name = "java-limbo" -version = "0.0.14" -dependencies = [ - "jni", - "limbo_core", - "thiserror 2.0.11", -] - [[package]] name = "jni" version = "0.21.1" @@ -1590,6 +1581,15 @@ dependencies = [ "limbo_core", ] +[[package]] +name = "limbo-java" +version = "0.0.14" +dependencies = [ + "jni", + "limbo_core", + "thiserror 2.0.11", +] + [[package]] name = "limbo-wasm" version = "0.0.14" diff --git a/README.md b/README.md index 6e150716c..480933581 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,11 @@ for rows.Next() { } ``` +### ☕️ Java (wip) + +We integrated Limbo into JDBC. For detailed instructions on how to use Limbo with java, please refer to +the [README.md under bindings/java](bindings/java/README.md). + ## Contributing We'd love to have you contribute to Limbo! Please check out the [contribution guide] to get started. diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index e3b7660c5..9b78b1597 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "java-limbo" +name = "limbo-java" version.workspace = true authors.workspace = true edition.workspace = true From 13062a14791650066d542745f092abec4fe11741 Mon Sep 17 00:00:00 2001 From: wyhaya Date: Sat, 8 Feb 2025 15:11:09 +0800 Subject: [PATCH 5/9] cli: Add column names in Pretty mode --- cli/app.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cli/app.rs b/cli/app.rs index 7d812d7fe..729a8cf6e 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -2,7 +2,7 @@ use crate::{ import::{ImportFile, IMPORT_HELP}, opcodes_dictionary::OPCODE_DESCRIPTIONS, }; -use cli_table::{Cell, Table}; +use cli_table::{Cell, Style, Table}; use limbo_core::{Database, LimboError, Statement, StepResult, Value}; use clap::{Parser, ValueEnum}; @@ -670,6 +670,16 @@ impl Limbo { return Ok(()); } let mut table_rows: Vec> = vec![]; + if rows.num_columns() > 0 { + let columns = (0..rows.num_columns()) + .map(|i| { + rows.get_column_name(i) + .map(|name| name.cell().bold(true)) + .unwrap_or_else(|| " ".cell()) + }) + .collect::>(); + table_rows.push(columns); + } loop { match rows.step() { Ok(StepResult::Row) => { From fab105c10c716bdb32eff5260f54d060905174bc Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 10:10:09 +0200 Subject: [PATCH 6/9] MVCC: fix write conflict handling --- core/mvcc/database/mod.rs | 60 ++++++++++++++++++++++++------------- core/mvcc/database/tests.rs | 16 ++++++++-- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/core/mvcc/database/mod.rs b/core/mvcc/database/mod.rs index 6c6e49053..fb094690d 100644 --- a/core/mvcc/database/mod.rs +++ b/core/mvcc/database/mod.rs @@ -365,6 +365,10 @@ impl MvStore MvStore MvStore = tx.write_set.iter().map(|v| *v.value()).collect(); drop(tx); // Postprocessing: inserting row versions and logging the transaction to persistent storage. @@ -568,7 +570,8 @@ impl MvStore MvStore MvStore( txs: &SkipMap>, tx: &Transaction, @@ -731,12 +745,16 @@ pub(crate) fn is_write_write_conflict( Some(TxTimestampOrID::TxID(rv_end)) => { let te = txs.get(&rv_end).unwrap(); let te = te.value().read().unwrap(); - match te.state.load() { - TransactionState::Active | TransactionState::Preparing => tx.tx_id != te.tx_id, - _ => false, + if te.tx_id == tx.tx_id { + return false; } + te.state.load() != TransactionState::Aborted } - Some(TxTimestampOrID::Timestamp(_)) => false, + // A non-"infinity" end timestamp (here modeled by Some(ts)) functions as a write lock + // on the row, so it can never be updated by another transaction. + // Ref: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf , page 301, + // 2.6. Updating a Version. + Some(TxTimestampOrID::Timestamp(_)) => true, None => false, } } diff --git a/core/mvcc/database/tests.rs b/core/mvcc/database/tests.rs index 8cb1c3027..b317a15d2 100644 --- a/core/mvcc/database/tests.rs +++ b/core/mvcc/database/tests.rs @@ -382,7 +382,7 @@ fn test_fuzzy_read() { table_id: 1, row_id: 1, }, - data: "Hello".to_string(), + data: "First".to_string(), }; db.insert(tx1, tx1_row.clone()).unwrap(); let row = db @@ -419,7 +419,7 @@ fn test_fuzzy_read() { table_id: 1, row_id: 1, }, - data: "World".to_string(), + data: "Second".to_string(), }; db.update(tx3, tx3_row).unwrap(); db.commit_tx(tx3).unwrap(); @@ -436,6 +436,18 @@ fn test_fuzzy_read() { .unwrap() .unwrap(); assert_eq!(tx1_row, row); + + // T2 tries to update the row, but fails because T3 has already committed an update to the row, + // so T2 trying to write would violate snapshot isolation if it succeeded. + let tx2_newrow = Row { + id: RowID { + table_id: 1, + row_id: 1, + }, + data: "Third".to_string(), + }; + let update_result = db.update(tx2, tx2_newrow); + assert_eq!(Err(DatabaseError::WriteWriteConflict), update_result); } #[test] From 791255fd8cd9d5609c0affe3c41ac6413be07537 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Sat, 8 Feb 2025 10:20:48 +0200 Subject: [PATCH 7/9] MVCC: Add a few comments --- core/mvcc/database/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/mvcc/database/mod.rs b/core/mvcc/database/mod.rs index fb094690d..da04221a2 100644 --- a/core/mvcc/database/mod.rs +++ b/core/mvcc/database/mod.rs @@ -58,7 +58,9 @@ impl LogRecord { /// versions switch to tracking timestamps. #[derive(Clone, Debug, PartialEq, PartialOrd)] enum TxTimestampOrID { + /// A committed transaction's timestamp. Timestamp(u64), + /// The ID of a non-committed transaction. TxID(TxID), } @@ -571,6 +573,7 @@ impl MvStore MvStore Date: Sat, 8 Feb 2025 10:55:13 +0200 Subject: [PATCH 8/9] core/mvcc: Minor code cleanups Make the source file readable from top to bottom by moving private functions at the end of the struct implementation. --- core/mvcc/database/mod.rs | 98 +++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/core/mvcc/database/mod.rs b/core/mvcc/database/mod.rs index da04221a2..4ae85f195 100644 --- a/core/mvcc/database/mod.rs +++ b/core/mvcc/database/mod.rs @@ -231,55 +231,6 @@ impl MvStore u64 { - match ts_or_id { - TxTimestampOrID::Timestamp(ts) => *ts, - TxTimestampOrID::TxID(tx_id) => { - self.txs - .get(tx_id) - .unwrap() - .value() - .read() - .unwrap() - .begin_ts - } - } - } - - /// Inserts a new row version into the database, while making sure that - /// the row version is inserted in the correct order. - fn insert_version(&self, id: RowID, row_version: RowVersion) { - let versions = self.rows.get_or_insert_with(id, || RwLock::new(Vec::new())); - let mut versions = versions.value().write().unwrap(); - self.insert_version_raw(&mut versions, row_version) - } - - /// Inserts a new row version into the internal data structure for versions, - /// while making sure that the row version is inserted in the correct order. - fn insert_version_raw(&self, versions: &mut Vec>, row_version: RowVersion) { - // NOTICE: this is an insert a'la insertion sort, with pessimistic linear complexity. - // However, we expect the number of versions to be nearly sorted, so we deem it worthy - // to search linearly for the insertion point instead of paying the price of using - // another data structure, e.g. a BTreeSet. If it proves to be too quadratic empirically, - // we can either switch to a tree-like structure, or at least use partition_point() - // which performs a binary search for the insertion point. - let position = versions - .iter() - .rposition(|v| { - self.get_begin_timestamp(&v.begin) < self.get_begin_timestamp(&row_version.begin) - }) - .map(|p| p + 1) - .unwrap_or(0); - if versions.len() - position > 3 { - tracing::debug!( - "Inserting a row version {} positions from the end", - versions.len() - position - ); - } - versions.insert(position, row_version); - } - /// Inserts a new row into the database. /// /// This function inserts a new `row` into the database within the context @@ -726,6 +677,55 @@ impl MvStore u64 { + match ts_or_id { + TxTimestampOrID::Timestamp(ts) => *ts, + TxTimestampOrID::TxID(tx_id) => { + self.txs + .get(tx_id) + .unwrap() + .value() + .read() + .unwrap() + .begin_ts + } + } + } + + /// Inserts a new row version into the database, while making sure that + /// the row version is inserted in the correct order. + fn insert_version(&self, id: RowID, row_version: RowVersion) { + let versions = self.rows.get_or_insert_with(id, || RwLock::new(Vec::new())); + let mut versions = versions.value().write().unwrap(); + self.insert_version_raw(&mut versions, row_version) + } + + /// Inserts a new row version into the internal data structure for versions, + /// while making sure that the row version is inserted in the correct order. + fn insert_version_raw(&self, versions: &mut Vec>, row_version: RowVersion) { + // NOTICE: this is an insert a'la insertion sort, with pessimistic linear complexity. + // However, we expect the number of versions to be nearly sorted, so we deem it worthy + // to search linearly for the insertion point instead of paying the price of using + // another data structure, e.g. a BTreeSet. If it proves to be too quadratic empirically, + // we can either switch to a tree-like structure, or at least use partition_point() + // which performs a binary search for the insertion point. + let position = versions + .iter() + .rposition(|v| { + self.get_begin_timestamp(&v.begin) < self.get_begin_timestamp(&row_version.begin) + }) + .map(|p| p + 1) + .unwrap_or(0); + if versions.len() - position > 3 { + tracing::debug!( + "Inserting a row version {} positions from the end", + versions.len() - position + ); + } + versions.insert(position, row_version); + } } /// A write-write conflict happens when transaction T_current attempts to update a From 3deac98d406576fa7ffc5ae032e06c6550796d5c Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Sat, 8 Feb 2025 11:25:36 +0200 Subject: [PATCH 9/9] cli: Make pretty mode pretty like DuckDB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DuckDB is pretty, I want to be pretty! ``` limbo> CREATE TABLE t(x); INSERT INTO t VALUES (1), (2), (3); limbo> .mode pretty limbo> SELECT * FROM t; ┌───┐ │ x │ ├───┤ │ 1 │ ├───┤ │ 2 │ ├───┤ │ 3 │ └───┘ ``` --- cli/app.rs | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/cli/app.rs b/cli/app.rs index 729a8cf6e..7fa7b70f0 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -2,6 +2,7 @@ use crate::{ import::{ImportFile, IMPORT_HELP}, opcodes_dictionary::OPCODE_DESCRIPTIONS, }; +use cli_table::format::{Border, HorizontalLine, Separator, VerticalLine}; use cli_table::{Cell, Style, Table}; use limbo_core::{Database, LimboError, Statement, StepResult, Value}; @@ -717,11 +718,7 @@ impl Limbo { } } } - if let Ok(table) = table_rows.table().display() { - let _ = self.write_fmt(format_args!("{}", table)); - } else { - let _ = self.writeln("Error displaying table."); - } + self.print_table(table_rows); } }, Ok(None) => {} @@ -737,6 +734,40 @@ impl Limbo { Ok(()) } + fn print_table(&mut self, table_rows: Vec>) { + if table_rows.is_empty() { + return; + } + + let horizontal_line = HorizontalLine::new('┌', '┐', '┬', '─'); + let horizontal_line_mid = HorizontalLine::new('├', '┤', '┼', '─'); + let horizontal_line_bottom = HorizontalLine::new('└', '┘', '┴', '─'); + let vertical_line = VerticalLine::new('│'); + + let border = Border::builder() + .top(horizontal_line) + .bottom(horizontal_line_bottom) + .left(vertical_line.clone()) + .right(vertical_line.clone()) + .build(); + + let separator = Separator::builder() + .column(Some(vertical_line)) + .row(Some(horizontal_line_mid)) + .build(); + + if let Ok(table) = table_rows + .table() + .border(border) + .separator(separator) + .display() + { + let _ = self.write_fmt(format_args!("{}", table)); + } else { + let _ = self.writeln("Error displaying table."); + } + } + fn display_schema(&mut self, table: Option<&str>) -> anyhow::Result<()> { let sql = match table { Some(table_name) => format!(