diff --git a/COMPAT.md b/COMPAT.md index 54949c91d..5584899f4 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -263,7 +263,7 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html). | jsonb_array(value1,value2,...) | | | | json_array_length(json) | Yes | | | json_array_length(json,path) | Yes | | -| json_error_position(json) | | | +| json_error_position(json) | Yes | | | json_extract(json,path,...) | Partial | Does not fully support unicode literal syntax and does not allow numbers > 2^127 - 1 (which SQLite truncates to i32), does not support BLOBs | | jsonb_extract(json,path,...) | | | | json -> path | Yes | | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b22b1212a..cb59600de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,3 +121,25 @@ The `simulator` directory contains a deterministic simulator for testing. What this means is that the behavior of a test run is deterministic based on the seed value. If the simulator catches a bug, you can always reproduce the exact same sequence of events by passing the same seed. The simulator also performs fault injection to discover interesting bugs. + +## Python Bindings + +Limbo provides Python bindings built on top of the [PyO3](https://pyo3.rs) project. +To compile the Python bindings locally, you first need to create and activate a Python virtual environment (for example, with Python `3.12`): + +```bash +python3.12 -m venv venv +source venv/bin/activate +``` + +Then, install [Maturin](https://pypi.org/project/maturin/): + +```bash +pip install maturin +``` + +Once Maturin is installed, you can build the crate and install it as a Python module directly into the current virtual environment by running: + +```bash +cd bindings/python && maturin develop +``` \ No newline at end of file diff --git a/bindings/java/rs_src/limbo_db.rs b/bindings/java/rs_src/limbo_db.rs index e03e76fa3..4b589ba9d 100644 --- a/bindings/java/rs_src/limbo_db.rs +++ b/bindings/java/rs_src/limbo_db.rs @@ -8,7 +8,7 @@ const ERROR_CODE_ETC: i32 = 9999; #[no_mangle] #[allow(clippy::arc_with_non_send_sync)] -pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB__1open_1utf8<'local>( +pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB_openUtf8<'local>( mut env: JNIEnv<'local>, obj: JObject<'local>, file_name_byte_arr: JByteArray<'local>, diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java b/bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java index 62bd86de3..2e37dcbab 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java @@ -1,9 +1,5 @@ package org.github.tursodatabase.core; -import org.github.tursodatabase.LimboErrorCode; -import org.github.tursodatabase.NativeInvocation; -import org.github.tursodatabase.exceptions.LimboException; - import java.sql.SQLException; import java.sql.SQLFeatureNotSupportedException; import java.util.concurrent.atomic.AtomicBoolean; @@ -19,7 +15,7 @@ public abstract class AbstractDB { private final String fileName; private final AtomicBoolean closed = new AtomicBoolean(true); - public AbstractDB(String url, String filaName) throws SQLException { + public AbstractDB(String url, String filaName) { this.url = url; this.fileName = filaName; } @@ -52,10 +48,10 @@ public abstract class AbstractDB { * @throws SQLException if a database access error occurs. */ public final synchronized void open(int openFlags) throws SQLException { - _open(fileName, openFlags); + open0(fileName, openFlags); } - protected abstract void _open(String fileName, int openFlags) throws SQLException; + protected abstract void open0(String fileName, int openFlags) throws SQLException; /** * Closes a database connection and finalizes any remaining statements before the closing @@ -100,14 +96,14 @@ public abstract class AbstractDB { * @return pointer to database instance * @throws SQLException if a database access error occurs. */ - protected abstract long _open_utf8(byte[] fileName, int openFlags) throws SQLException; + protected abstract long openUtf8(byte[] fileName, int openFlags) throws SQLException; /** * Closes the SQLite interface to a database. * * @throws SQLException if a database access error occurs. */ - protected abstract void _close() throws SQLException; + protected abstract void close0() throws SQLException; /** * Compiles, evaluates, executes and commits an SQL statement. @@ -116,7 +112,7 @@ public abstract class AbstractDB { * @return Result code. * @throws SQLException if a database access error occurs. */ - public abstract int _exec(String sql) throws SQLException; + public abstract int exec(String sql) throws SQLException; /** * Compiles an SQL statement. diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java index 80c3fbe8b..f3001aead 100644 --- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java +++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java @@ -30,8 +30,6 @@ public final class LimboDB extends AbstractDB { } } - // url example: "jdbc:sqlite:{fileName} - /** * @param url e.g. "jdbc:sqlite:fileName * @param fileName e.g. path to file @@ -41,7 +39,7 @@ public final class LimboDB extends AbstractDB { } // TODO: receive config as argument - private LimboDB(String url, String fileName) throws SQLException { + private LimboDB(String url, String fileName) { super(url, fileName); } @@ -53,7 +51,6 @@ public final class LimboDB extends AbstractDB { try { System.loadLibrary("_limbo_java"); - } finally { isLoaded = true; } @@ -63,31 +60,31 @@ public final class LimboDB extends AbstractDB { // TODO: add support for JNI @Override - protected synchronized native long _open_utf8(byte[] file, int openFlags) throws SQLException; + protected synchronized native long openUtf8(byte[] file, int openFlags) throws SQLException; // TODO: add support for JNI @Override - protected synchronized native void _close() throws SQLException; + protected synchronized native void close0() throws SQLException; @Override - public synchronized int _exec(String sql) throws SQLException { + public synchronized int exec(String sql) throws SQLException { // TODO: add implementation throw new SQLFeatureNotSupportedException(); } // TODO: add support for JNI - synchronized native int _exec_utf8(byte[] sqlUtf8) throws SQLException; + synchronized native int execUtf8(byte[] sqlUtf8) throws SQLException; // TODO: add support for JNI @Override public native void interrupt(); @Override - protected void _open(String fileName, int openFlags) throws SQLException { + protected void open0(String fileName, int openFlags) throws SQLException { if (isOpen) { throwLimboException(LimboErrorCode.UNKNOWN_ERROR.code, "Already opened"); } - dbPtr = _open_utf8(stringToUtf8ByteArray(fileName), openFlags); + dbPtr = openUtf8(stringToUtf8ByteArray(fileName), openFlags); isOpen = true; } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 193230b29..1886627e4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -29,3 +29,6 @@ rustyline = "12.0.0" ctrlc = "3.4.4" csv = "1.3.1" miette = { version = "7.4.0", features = ["fancy"] } + +[features] +io_uring = ["limbo_core/io_uring"] diff --git a/cli/app.rs b/cli/app.rs index 0b2f77641..f114b63a2 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -38,6 +38,52 @@ pub struct Opts { pub quiet: bool, #[clap(short, long, help = "Print commands before execution")] pub echo: bool, + #[clap( + default_value_t, + value_enum, + short, + long, + help = "Select I/O backend. The only other choice to 'syscall' is\n\ + \t'io-uring' when built for Linux with feature 'io_uring'\n" + )] + pub io: Io, +} + +#[derive(Copy, Clone)] +pub enum DbLocation { + Memory, + Path, +} + +#[derive(Copy, Clone, ValueEnum)] +pub enum Io { + Syscall, + #[cfg(all(target_os = "linux", feature = "io_uring"))] + IoUring, +} + +impl Default for Io { + /// Custom Default impl with cfg! macro, to provide compile-time default to Clap based on platform + /// The cfg! could be elided, but Clippy complains + /// The default value can still be overridden with the Clap argument + fn default() -> Self { + match cfg!(all(target_os = "linux", feature = "io_uring")) { + true => { + #[cfg(all(target_os = "linux", feature = "io_uring"))] + { + Io::IoUring + } + #[cfg(any( + not(target_os = "linux"), + all(target_os = "linux", not(feature = "io_uring")) + ))] + { + Io::Syscall + } + } + false => Io::Syscall, + } + } } #[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)] @@ -160,6 +206,7 @@ pub struct Settings { output_mode: OutputMode, echo: bool, is_stdout: bool, + io: Io, } impl From<&Opts> for Settings { @@ -174,6 +221,7 @@ impl From<&Opts> for Settings { .database .as_ref() .map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string()), + io: opts.io, } } } @@ -207,7 +255,12 @@ impl Limbo { .as_ref() .map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string()); - let io = get_io(&db_file)?; + let io = { + match db_file.as_str() { + ":memory:" => get_io(DbLocation::Memory, opts.io)?, + _path => get_io(DbLocation::Path, opts.io)?, + } + }; let db = Database::open_file(io.clone(), &db_file)?; let conn = db.connect(); let interrupt_count = Arc::new(AtomicUsize::new(0)); @@ -293,24 +346,17 @@ impl Limbo { fn open_db(&mut self, path: &str) -> anyhow::Result<()> { self.conn.close()?; - match path { - ":memory:" => { - let io: Arc = Arc::new(limbo_core::MemoryIO::new()?); - self.io = Arc::clone(&io); - let db = Database::open_file(self.io.clone(), path)?; - self.conn = db.connect(); - self.opts.db_file = ":memory:".to_string(); - Ok(()) + let io = { + match path { + ":memory:" => get_io(DbLocation::Memory, self.opts.io)?, + _path => get_io(DbLocation::Path, self.opts.io)?, } - path => { - let io: Arc = Arc::new(limbo_core::PlatformIO::new()?); - self.io = Arc::clone(&io); - let db = Database::open_file(self.io.clone(), path)?; - self.conn = db.connect(); - self.opts.db_file = path.to_string(); - Ok(()) - } - } + }; + self.io = Arc::clone(&io); + let db = Database::open_file(self.io.clone(), path)?; + self.conn = db.connect(); + self.opts.db_file = path.to_string(); + Ok(()) } fn set_output_file(&mut self, path: &str) -> Result<(), String> { @@ -740,10 +786,28 @@ fn get_writer(output: &str) -> Box { } } -fn get_io(db: &str) -> anyhow::Result> { - Ok(match db { - ":memory:" => Arc::new(limbo_core::MemoryIO::new()?), - _ => Arc::new(limbo_core::PlatformIO::new()?), +fn get_io(db_location: DbLocation, io_choice: Io) -> anyhow::Result> { + Ok(match db_location { + DbLocation::Memory => Arc::new(limbo_core::MemoryIO::new()?), + DbLocation::Path => { + match io_choice { + Io::Syscall => { + // We are building for Linux/macOS and syscall backend has been selected + #[cfg(target_family = "unix")] + { + Arc::new(limbo_core::UnixIO::new()?) + } + // We are not building for Linux/macOS and syscall backend has been selected + #[cfg(not(target_family = "unix"))] + { + Arc::new(limbo_core::PlatformIO::new()?) + } + } + // We are building for Linux and io_uring backend has been selected + #[cfg(all(target_os = "linux", feature = "io_uring"))] + Io::IoUring => Arc::new(limbo_core::UringIO::new()?), + } + } }) } diff --git a/core/Cargo.toml b/core/Cargo.toml index c0c579152..fc1f88fe2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -50,7 +50,7 @@ regex-syntax = { version = "0.8.5", default-features = false, features = ["unico chrono = "0.4.38" julian_day_converter = "0.3.2" jsonb = { version = "0.4.4", optional = true } -indexmap = { version="2.2.6", features = ["serde"] } +indexmap = { version = "2.2.6", features = ["serde"] } serde = { version = "1.0", features = ["derive"] } pest = { version = "2.0", optional = true } pest_derive = { version = "2.0", optional = true } diff --git a/core/function.rs b/core/function.rs index 7b906406a..060a677c3 100644 --- a/core/function.rs +++ b/core/function.rs @@ -30,6 +30,7 @@ pub enum JsonFunc { JsonArrowShiftExtract, JsonExtract, JsonType, + JsonErrorPosition, } #[cfg(feature = "json")] @@ -46,6 +47,7 @@ impl Display for JsonFunc { Self::JsonArrowExtract => "->".to_string(), Self::JsonArrowShiftExtract => "->>".to_string(), Self::JsonType => "json_type".to_string(), + Self::JsonErrorPosition => "json_error_position".to_string(), } ) } @@ -379,6 +381,8 @@ impl Func { "json_extract" => Ok(Func::Json(JsonFunc::JsonExtract)), #[cfg(feature = "json")] "json_type" => Ok(Func::Json(JsonFunc::JsonType)), + #[cfg(feature = "json")] + "json_error_position" => Ok(Self::Json(JsonFunc::JsonErrorPosition)), "unixepoch" => Ok(Self::Scalar(ScalarFunc::UnixEpoch)), "julianday" => Ok(Self::Scalar(ScalarFunc::JulianDay)), "hex" => Ok(Self::Scalar(ScalarFunc::Hex)), diff --git a/core/io/generic.rs b/core/io/generic.rs index 17f51d792..79bcde49c 100644 --- a/core/io/generic.rs +++ b/core/io/generic.rs @@ -1,5 +1,5 @@ use crate::{Completion, File, LimboError, OpenFlags, Result, IO}; -use log::trace; +use log::{debug, trace}; use std::cell::RefCell; use std::io::{Read, Seek, Write}; use std::rc::Rc; @@ -8,6 +8,7 @@ pub struct GenericIO {} impl GenericIO { pub fn new() -> Result { + debug!("Using IO backend 'generic'"); Ok(Self {}) } } diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index 54a02ee61..3278d61b9 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -70,6 +70,7 @@ impl UringIO { }; MAX_IOVECS], next_iovec: 0, }; + debug!("Using IO backend 'io-uring'"); Ok(Self { inner: Rc::new(RefCell::new(inner)), }) diff --git a/core/io/memory.rs b/core/io/memory.rs index 46aa8834f..6fc960e13 100644 --- a/core/io/memory.rs +++ b/core/io/memory.rs @@ -1,6 +1,7 @@ use super::{Buffer, Completion, File, OpenFlags, IO}; use crate::Result; +use log::debug; use std::{ cell::{RefCell, RefMut}, collections::BTreeMap, @@ -20,6 +21,7 @@ type MemPage = Box<[u8; PAGE_SIZE]>; impl MemoryIO { #[allow(clippy::arc_with_non_send_sync)] pub fn new() -> Result> { + debug!("Using IO backend 'memory'"); Ok(Arc::new(Self { pages: RefCell::new(BTreeMap::new()), size: RefCell::new(0), diff --git a/core/io/mod.rs b/core/io/mod.rs index 7e910f4af..f88a5d554 100644 --- a/core/io/mod.rs +++ b/core/io/mod.rs @@ -166,11 +166,15 @@ impl Buffer { cfg_block! { #[cfg(all(target_os = "linux", feature = "io_uring"))] { mod io_uring; + pub use io_uring::UringIO; + mod unix; + pub use unix::UnixIO; pub use io_uring::UringIO as PlatformIO; } #[cfg(any(all(target_os = "linux",not(feature = "io_uring")), target_os = "macos"))] { mod unix; + pub use unix::UnixIO; pub use unix::UnixIO as PlatformIO; } diff --git a/core/io/unix.rs b/core/io/unix.rs index c86e05ee4..db0e85ab1 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -4,7 +4,7 @@ use crate::Result; use super::{Completion, File, OpenFlags, IO}; use libc::{c_short, fcntl, flock, F_SETLK}; -use log::trace; +use log::{debug, trace}; use polling::{Event, Events, Poller}; use rustix::fd::{AsFd, AsRawFd}; use rustix::fs::OpenOptionsExt; @@ -22,6 +22,7 @@ pub struct UnixIO { impl UnixIO { pub fn new() -> Result { + debug!("Using IO backend 'syscall'"); Ok(Self { poller: Rc::new(RefCell::new(Poller::new()?)), events: Rc::new(RefCell::new(Events::new())), diff --git a/core/io/windows.rs b/core/io/windows.rs index 0eea9e4da..50acfcb50 100644 --- a/core/io/windows.rs +++ b/core/io/windows.rs @@ -1,5 +1,5 @@ use crate::{Completion, File, LimboError, OpenFlags, Result, IO}; -use log::trace; +use log::{debug, trace}; use std::cell::RefCell; use std::io::{Read, Seek, Write}; use std::rc::Rc; @@ -8,6 +8,7 @@ pub struct WindowsIO {} impl WindowsIO { pub fn new() -> Result { + debug!("Using IO backend 'syscall'"); Ok(Self {}) } } diff --git a/core/json/mod.rs b/core/json/mod.rs index f0a362ab6..fb3afc3db 100644 --- a/core/json/mod.rs +++ b/core/json/mod.rs @@ -6,10 +6,12 @@ mod ser; use std::rc::Rc; pub use crate::json::de::from_str; +use crate::json::error::Error as JsonError; use crate::json::json_path::{json_path, JsonPath, PathElement}; pub use crate::json::ser::to_string; use crate::types::{LimboText, OwnedValue, TextSubtype}; use indexmap::IndexMap; +use jsonb::Error as JsonbError; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] @@ -370,6 +372,32 @@ fn json_extract_single<'a>( Ok(Some(¤t_element)) } +pub fn json_error_position(json: &OwnedValue) -> crate::Result { + match json { + OwnedValue::Text(t) => match crate::json::from_str::(&t.value) { + Ok(_) => Ok(OwnedValue::Integer(0)), + Err(JsonError::Message { location, .. }) => { + if let Some(loc) = location { + Ok(OwnedValue::Integer(loc.column as i64)) + } else { + Err(crate::error::LimboError::InternalError( + "failed to determine json error position".into(), + )) + } + } + }, + OwnedValue::Blob(b) => match jsonb::from_slice(b) { + Ok(_) => Ok(OwnedValue::Integer(0)), + Err(JsonbError::Syntax(_, pos)) => Ok(OwnedValue::Integer(pos as i64)), + _ => Err(crate::error::LimboError::InternalError( + "failed to determine json error position".into(), + )), + }, + OwnedValue::Null => Ok(OwnedValue::Null), + _ => Ok(OwnedValue::Integer(0)), + } +} + #[cfg(test)] mod tests { use super::*; @@ -698,4 +726,60 @@ mod tests { Err(e) => assert!(e.to_string().contains("JSON path error")), } } + + #[test] + fn test_json_error_position_no_error() { + let input = OwnedValue::build_text(Rc::new("[1,2,3]".to_string())); + let result = json_error_position(&input).unwrap(); + assert_eq!(result, OwnedValue::Integer(0)); + } + + #[test] + fn test_json_error_position_no_error_more() { + let input = OwnedValue::build_text(Rc::new(r#"{"a":55,"b":72 , }"#.to_string())); + let result = json_error_position(&input).unwrap(); + assert_eq!(result, OwnedValue::Integer(0)); + } + + #[test] + fn test_json_error_position_object() { + let input = OwnedValue::build_text(Rc::new(r#"{"a":55,"b":72,,}"#.to_string())); + let result = json_error_position(&input).unwrap(); + assert_eq!(result, OwnedValue::Integer(16)); + } + + #[test] + fn test_json_error_position_array() { + let input = OwnedValue::build_text(Rc::new(r#"["a",55,"b",72,,]"#.to_string())); + let result = json_error_position(&input).unwrap(); + assert_eq!(result, OwnedValue::Integer(16)); + } + + #[test] + fn test_json_error_position_null() { + let input = OwnedValue::Null; + let result = json_error_position(&input).unwrap(); + assert_eq!(result, OwnedValue::Null); + } + + #[test] + fn test_json_error_position_integer() { + let input = OwnedValue::Integer(5); + let result = json_error_position(&input).unwrap(); + assert_eq!(result, OwnedValue::Integer(0)); + } + + #[test] + fn test_json_error_position_float() { + let input = OwnedValue::Float(-5.5); + let result = json_error_position(&input).unwrap(); + assert_eq!(result, OwnedValue::Integer(0)); + } + + #[test] + fn test_json_error_position_blob() { + let input = OwnedValue::Blob(Rc::new(r#"["a",55,"b",72,,]"#.as_bytes().to_owned())); + let result = json_error_position(&input).unwrap(); + assert_eq!(result, OwnedValue::Integer(16)); + } } diff --git a/core/lib.rs b/core/lib.rs index fd563cb19..a80fab83a 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -44,8 +44,11 @@ pub type Result = std::result::Result; use crate::translate::optimizer::optimize_plan; pub use io::OpenFlags; -#[cfg(feature = "fs")] pub use io::PlatformIO; +#[cfg(all(feature = "fs", target_family = "unix"))] +pub use io::UnixIO; +#[cfg(all(feature = "fs", target_os = "linux", feature = "io_uring"))] +pub use io::UringIO; pub use io::{Buffer, Completion, File, MemoryIO, WriteCompletion, IO}; pub use storage::buffer_pool::BufferPool; pub use storage::database::DatabaseStorage; diff --git a/core/pseudo.rs b/core/pseudo.rs index 45f47856e..93d9d6a64 100644 --- a/core/pseudo.rs +++ b/core/pseudo.rs @@ -1,110 +1,19 @@ -use crate::{ - types::{SeekKey, SeekOp}, - Result, -}; -use std::cell::{Ref, RefCell}; - -use crate::types::{Cursor, CursorResult, OwnedRecord, OwnedValue}; +use crate::types::OwnedRecord; pub struct PseudoCursor { - current: RefCell>, + current: Option, } impl PseudoCursor { pub fn new() -> Self { - Self { - current: RefCell::new(None), - } - } -} - -impl Cursor for PseudoCursor { - fn is_empty(&self) -> bool { - self.current.borrow().is_none() - } - - fn root_page(&self) -> usize { - unreachable!() - } - - fn rewind(&mut self) -> Result> { - *self.current.borrow_mut() = None; - Ok(CursorResult::Ok(())) - } - - fn next(&mut self) -> Result> { - *self.current.borrow_mut() = None; - Ok(CursorResult::Ok(())) - } - - fn wait_for_completion(&mut self) -> Result<()> { - Ok(()) - } - - fn rowid(&self) -> Result> { - let x = self - .current - .borrow() - .as_ref() - .map(|record| match record.values[0] { - OwnedValue::Integer(rowid) => rowid as u64, - ref ov => { - panic!("Expected integer value, got {:?}", ov); - } - }); - Ok(x) - } - - fn seek(&mut self, _: SeekKey<'_>, _: SeekOp) -> Result> { - unimplemented!(); - } - - fn seek_to_last(&mut self) -> Result> { - unimplemented!(); - } - - fn record(&self) -> Result>> { - Ok(self.current.borrow()) - } - - fn insert( - &mut self, - key: &OwnedValue, - record: &OwnedRecord, - moved_before: bool, - ) -> Result> { - let _ = key; - let _ = moved_before; - *self.current.borrow_mut() = Some(record.clone()); - Ok(CursorResult::Ok(())) - } - - fn delete(&mut self) -> Result> { - unimplemented!() - } - - fn get_null_flag(&self) -> bool { - false - } - - fn set_null_flag(&mut self, _null_flag: bool) { - // Do nothing - } - - fn exists(&mut self, key: &OwnedValue) -> Result> { - let _ = key; - todo!() - } - - fn btree_create(&mut self, _flags: usize) -> u32 { - unreachable!("Please don't.") - } - - fn last(&mut self) -> Result> { - todo!() - } - - fn prev(&mut self) -> Result> { - todo!() + Self { current: None } + } + + pub fn record(&self) -> Option<&OwnedRecord> { + self.current.as_ref() + } + + pub fn insert(&mut self, record: OwnedRecord) { + self.current = Some(record); } } diff --git a/core/schema.rs b/core/schema.rs index 829320585..fda6c12ba 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -46,35 +46,13 @@ impl Schema { #[derive(Clone, Debug)] pub enum Table { BTree(Rc), - Index(Rc), Pseudo(Rc), } impl Table { - pub fn is_pseudo(&self) -> bool { - matches!(self, Table::Pseudo(_)) - } - - pub fn get_rowid_alias_column(&self) -> Option<(usize, &Column)> { - match self { - Self::BTree(table) => table.get_rowid_alias_column(), - Self::Index(_) => None, - Self::Pseudo(_) => None, - } - } - - pub fn column_is_rowid_alias(&self, col: &Column) -> bool { - match self { - Table::BTree(table) => table.column_is_rowid_alias(col), - Self::Index(_) => false, - Self::Pseudo(_) => false, - } - } - pub fn get_root_page(&self) -> usize { match self { Table::BTree(table) => table.root_page, - Table::Index(_) => unimplemented!(), Table::Pseudo(_) => unimplemented!(), } } @@ -82,40 +60,13 @@ impl Table { pub fn get_name(&self) -> &str { match self { Self::BTree(table) => &table.name, - Self::Index(index) => &index.name, Self::Pseudo(_) => "", } } - pub fn column_index_to_name(&self, index: usize) -> Option<&str> { - match self { - Self::BTree(table) => match table.columns.get(index) { - Some(column) => Some(&column.name), - None => None, - }, - Self::Index(i) => match i.columns.get(index) { - Some(column) => Some(&column.name), - None => None, - }, - Self::Pseudo(table) => match table.columns.get(index) { - Some(_) => None, - None => None, - }, - } - } - - pub fn get_column(&self, name: &str) -> Option<(usize, &Column)> { - match self { - Self::BTree(table) => table.get_column(name), - Self::Index(_) => unimplemented!(), - Self::Pseudo(table) => table.get_column(name), - } - } - pub fn get_column_at(&self, index: usize) -> &Column { match self { Self::BTree(table) => table.columns.get(index).unwrap(), - Self::Index(_) => unimplemented!(), Self::Pseudo(table) => table.columns.get(index).unwrap(), } } @@ -123,16 +74,14 @@ impl Table { pub fn columns(&self) -> &Vec { match self { Self::BTree(table) => &table.columns, - Self::Index(_) => unimplemented!(), Self::Pseudo(table) => &table.columns, } } - pub fn has_rowid(&self) -> bool { + pub fn btree(&self) -> Option> { match self { - Self::BTree(table) => table.has_rowid, - Self::Index(_) => unimplemented!(), - Self::Pseudo(_) => unimplemented!(), + Self::BTree(table) => Some(table.clone()), + Self::Pseudo(_) => None, } } } diff --git a/core/storage/btree.rs b/core/storage/btree.rs index c84a0c2c0..5bdcb1352 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -5,7 +5,7 @@ use crate::storage::sqlite3_ondisk::{ read_btree_cell, read_varint, write_varint, BTreeCell, DatabaseHeader, PageContent, PageType, TableInteriorCell, TableLeafCell, }; -use crate::types::{Cursor, CursorResult, OwnedRecord, OwnedValue, SeekKey, SeekOp}; +use crate::types::{CursorResult, OwnedRecord, OwnedValue, SeekKey, SeekOp}; use crate::Result; use std::cell::{Ref, RefCell}; @@ -419,7 +419,7 @@ impl BTreeCursor { /// This may be used to seek to a specific record in a point query (e.g. SELECT * FROM table WHERE col = 10) /// or e.g. find the first record greater than the seek key in a range query (e.g. SELECT * FROM table WHERE col > 10). /// We don't include the rowid in the comparison and that's why the last value from the record is not included. - fn seek( + fn do_seek( &mut self, key: SeekKey<'_>, op: SeekOp, @@ -1697,6 +1697,162 @@ impl BTreeCursor { } cell_idx } + + pub fn seek_to_last(&mut self) -> Result> { + return_if_io!(self.move_to_rightmost()); + let (rowid, record) = return_if_io!(self.get_next_record(None)); + if rowid.is_none() { + let is_empty = return_if_io!(self.is_empty_table()); + assert!(is_empty); + return Ok(CursorResult::Ok(())); + } + self.rowid.replace(rowid); + self.record.replace(record); + Ok(CursorResult::Ok(())) + } + + pub fn is_empty(&self) -> bool { + self.record.borrow().is_none() + } + + pub fn root_page(&self) -> usize { + self.root_page + } + + pub fn rewind(&mut self) -> Result> { + self.move_to_root(); + + let (rowid, record) = return_if_io!(self.get_next_record(None)); + self.rowid.replace(rowid); + self.record.replace(record); + Ok(CursorResult::Ok(())) + } + + pub fn last(&mut self) -> Result> { + match self.move_to_rightmost()? { + CursorResult::Ok(_) => self.prev(), + CursorResult::IO => Ok(CursorResult::IO), + } + } + + pub fn next(&mut self) -> Result> { + let (rowid, record) = return_if_io!(self.get_next_record(None)); + self.rowid.replace(rowid); + self.record.replace(record); + Ok(CursorResult::Ok(())) + } + + pub fn prev(&mut self) -> Result> { + match self.get_prev_record()? { + CursorResult::Ok((rowid, record)) => { + self.rowid.replace(rowid); + self.record.replace(record); + Ok(CursorResult::Ok(())) + } + CursorResult::IO => Ok(CursorResult::IO), + } + } + + pub fn wait_for_completion(&mut self) -> Result<()> { + // TODO: Wait for pager I/O to complete + Ok(()) + } + + pub fn rowid(&self) -> Result> { + Ok(*self.rowid.borrow()) + } + + pub fn seek(&mut self, key: SeekKey<'_>, op: SeekOp) -> Result> { + let (rowid, record) = return_if_io!(self.do_seek(key, op)); + self.rowid.replace(rowid); + self.record.replace(record); + Ok(CursorResult::Ok(rowid.is_some())) + } + + pub fn record(&self) -> Result>> { + Ok(self.record.borrow()) + } + + pub fn insert( + &mut self, + key: &OwnedValue, + _record: &OwnedRecord, + moved_before: bool, /* Indicate whether it's necessary to traverse to find the leaf page */ + ) -> Result> { + let int_key = match key { + OwnedValue::Integer(i) => i, + _ => unreachable!("btree tables are indexed by integers!"), + }; + if !moved_before { + return_if_io!(self.move_to(SeekKey::TableRowId(*int_key as u64), SeekOp::EQ)); + } + + return_if_io!(self.insert_into_page(key, _record)); + self.rowid.replace(Some(*int_key as u64)); + Ok(CursorResult::Ok(())) + } + + pub fn delete(&mut self) -> Result> { + println!("rowid: {:?}", self.rowid.borrow()); + Ok(CursorResult::Ok(())) + } + + pub fn set_null_flag(&mut self, flag: bool) { + self.null_flag = flag; + } + + pub fn get_null_flag(&self) -> bool { + self.null_flag + } + + pub fn exists(&mut self, key: &OwnedValue) -> Result> { + let int_key = match key { + OwnedValue::Integer(i) => i, + _ => unreachable!("btree tables are indexed by integers!"), + }; + return_if_io!(self.move_to(SeekKey::TableRowId(*int_key as u64), SeekOp::EQ)); + let page = self.stack.top(); + // TODO(pere): request load + return_if_locked!(page); + + let contents = page.get().contents.as_ref().unwrap(); + + // find cell + let int_key = match key { + OwnedValue::Integer(i) => *i as u64, + _ => unreachable!("btree tables are indexed by integers!"), + }; + let cell_idx = self.find_cell(contents, int_key); + if cell_idx >= contents.cell_count() { + Ok(CursorResult::Ok(false)) + } else { + let equals = match &contents.cell_get( + cell_idx, + self.pager.clone(), + self.payload_overflow_threshold_max(contents.page_type()), + self.payload_overflow_threshold_min(contents.page_type()), + self.usable_space(), + )? { + BTreeCell::TableLeafCell(l) => l._rowid == int_key, + _ => unreachable!(), + }; + Ok(CursorResult::Ok(equals)) + } + } + + pub fn btree_create(&mut self, flags: usize) -> u32 { + let page_type = match flags { + 1 => PageType::TableLeaf, + 2 => PageType::IndexLeaf, + _ => unreachable!( + "wrong create table falgs, should be 1 for table and 2 for index, got {}", + flags, + ), + }; + let page = self.allocate_page(page_type, 0); + let id = page.get().id; + id as u32 + } } impl PageStack { @@ -1822,164 +1978,6 @@ fn find_free_cell(page_ref: &PageContent, db_header: Ref, amount } } -impl Cursor for BTreeCursor { - fn seek_to_last(&mut self) -> Result> { - return_if_io!(self.move_to_rightmost()); - let (rowid, record) = return_if_io!(self.get_next_record(None)); - if rowid.is_none() { - let is_empty = return_if_io!(self.is_empty_table()); - assert!(is_empty); - return Ok(CursorResult::Ok(())); - } - self.rowid.replace(rowid); - self.record.replace(record); - Ok(CursorResult::Ok(())) - } - - fn is_empty(&self) -> bool { - self.record.borrow().is_none() - } - - fn root_page(&self) -> usize { - self.root_page - } - - fn rewind(&mut self) -> Result> { - self.move_to_root(); - - let (rowid, record) = return_if_io!(self.get_next_record(None)); - self.rowid.replace(rowid); - self.record.replace(record); - Ok(CursorResult::Ok(())) - } - - fn last(&mut self) -> Result> { - match self.move_to_rightmost()? { - CursorResult::Ok(_) => self.prev(), - CursorResult::IO => Ok(CursorResult::IO), - } - } - - fn next(&mut self) -> Result> { - let (rowid, record) = return_if_io!(self.get_next_record(None)); - self.rowid.replace(rowid); - self.record.replace(record); - Ok(CursorResult::Ok(())) - } - - fn prev(&mut self) -> Result> { - match self.get_prev_record()? { - CursorResult::Ok((rowid, record)) => { - self.rowid.replace(rowid); - self.record.replace(record); - Ok(CursorResult::Ok(())) - } - CursorResult::IO => Ok(CursorResult::IO), - } - } - - fn wait_for_completion(&mut self) -> Result<()> { - // TODO: Wait for pager I/O to complete - Ok(()) - } - - fn rowid(&self) -> Result> { - Ok(*self.rowid.borrow()) - } - - fn seek(&mut self, key: SeekKey<'_>, op: SeekOp) -> Result> { - let (rowid, record) = return_if_io!(self.seek(key, op)); - self.rowid.replace(rowid); - self.record.replace(record); - Ok(CursorResult::Ok(rowid.is_some())) - } - - fn record(&self) -> Result>> { - Ok(self.record.borrow()) - } - - fn insert( - &mut self, - key: &OwnedValue, - _record: &OwnedRecord, - moved_before: bool, /* Indicate whether it's necessary to traverse to find the leaf page */ - ) -> Result> { - let int_key = match key { - OwnedValue::Integer(i) => i, - _ => unreachable!("btree tables are indexed by integers!"), - }; - if !moved_before { - return_if_io!(self.move_to(SeekKey::TableRowId(*int_key as u64), SeekOp::EQ)); - } - - return_if_io!(self.insert_into_page(key, _record)); - self.rowid.replace(Some(*int_key as u64)); - Ok(CursorResult::Ok(())) - } - - fn delete(&mut self) -> Result> { - println!("rowid: {:?}", self.rowid.borrow()); - Ok(CursorResult::Ok(())) - } - - fn set_null_flag(&mut self, flag: bool) { - self.null_flag = flag; - } - - fn get_null_flag(&self) -> bool { - self.null_flag - } - - fn exists(&mut self, key: &OwnedValue) -> Result> { - let int_key = match key { - OwnedValue::Integer(i) => i, - _ => unreachable!("btree tables are indexed by integers!"), - }; - return_if_io!(self.move_to(SeekKey::TableRowId(*int_key as u64), SeekOp::EQ)); - let page = self.stack.top(); - // TODO(pere): request load - return_if_locked!(page); - - let contents = page.get().contents.as_ref().unwrap(); - - // find cell - let int_key = match key { - OwnedValue::Integer(i) => *i as u64, - _ => unreachable!("btree tables are indexed by integers!"), - }; - let cell_idx = self.find_cell(contents, int_key); - if cell_idx >= contents.cell_count() { - Ok(CursorResult::Ok(false)) - } else { - let equals = match &contents.cell_get( - cell_idx, - self.pager.clone(), - self.payload_overflow_threshold_max(contents.page_type()), - self.payload_overflow_threshold_min(contents.page_type()), - self.usable_space(), - )? { - BTreeCell::TableLeafCell(l) => l._rowid == int_key, - _ => unreachable!(), - }; - Ok(CursorResult::Ok(equals)) - } - } - - fn btree_create(&mut self, flags: usize) -> u32 { - let page_type = match flags { - 1 => PageType::TableLeaf, - 2 => PageType::IndexLeaf, - _ => unreachable!( - "wrong create table falgs, should be 1 for table and 2 for index, got {}", - flags, - ), - }; - let page = self.allocate_page(page_type, 0); - let id = page.get().id; - id as u32 - } -} - pub fn btree_init_page( page: &PageRef, page_type: PageType, diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 0d7593ecc..8dbda073b 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -912,7 +912,7 @@ fn read_payload(unread: &[u8], payload_size: usize, pager: Rc) -> (Vec for SerialType { fn try_from(value: u64) -> Result { match value { 0 => Ok(Self::Null), - 1 => Ok(Self::UInt8), + 1 => Ok(Self::Int8), 2 => Ok(Self::BEInt16), 3 => Ok(Self::BEInt24), 4 => Ok(Self::BEInt32), @@ -974,11 +974,12 @@ pub fn read_record(payload: &[u8]) -> Result { pub fn read_value(buf: &[u8], serial_type: &SerialType) -> Result<(OwnedValue, usize)> { match *serial_type { SerialType::Null => Ok((OwnedValue::Null, 0)), - SerialType::UInt8 => { + SerialType::Int8 => { if buf.is_empty() { crate::bail_corrupt_error!("Invalid UInt8 value"); } - Ok((OwnedValue::Integer(buf[0] as i64), 1)) + let val = buf[0] as i8; + Ok((OwnedValue::Integer(val as i64), 1)) } SerialType::BEInt16 => { if buf.len() < 2 { @@ -1377,7 +1378,7 @@ mod tests { #[rstest] #[case(0, SerialType::Null)] - #[case(1, SerialType::UInt8)] + #[case(1, SerialType::Int8)] #[case(2, SerialType::BEInt16)] #[case(3, SerialType::BEInt24)] #[case(4, SerialType::BEInt32)] @@ -1403,8 +1404,9 @@ mod tests { #[rstest] #[case(&[], SerialType::Null, OwnedValue::Null)] - #[case(&[255], SerialType::UInt8, OwnedValue::Integer(255))] + #[case(&[255], SerialType::Int8, OwnedValue::Integer(-1))] #[case(&[0x12, 0x34], SerialType::BEInt16, OwnedValue::Integer(0x1234))] + #[case(&[0xFE], SerialType::Int8, OwnedValue::Integer(-2))] #[case(&[0x12, 0x34, 0x56], SerialType::BEInt24, OwnedValue::Integer(0x123456))] #[case(&[0x12, 0x34, 0x56, 0x78], SerialType::BEInt32, OwnedValue::Integer(0x12345678))] #[case(&[0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC], SerialType::BEInt48, OwnedValue::Integer(0x123456789ABC))] @@ -1424,7 +1426,7 @@ mod tests { } #[rstest] - #[case(&[], SerialType::UInt8)] + #[case(&[], SerialType::Int8)] #[case(&[0x12], SerialType::BEInt16)] #[case(&[0x12, 0x34], SerialType::BEInt24)] #[case(&[0x12, 0x34, 0x56], SerialType::BEInt32)] diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 2f88c9a0d..dd35a5d26 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -812,6 +812,31 @@ pub fn translate_expr( func_ctx, ) } + JsonFunc::JsonErrorPosition => { + let args = if let Some(args) = args { + if args.len() != 1 { + crate::bail_parse_error!( + "{} function with not exactly 1 argument", + j.to_string() + ); + } + args + } else { + crate::bail_parse_error!( + "{} function with no arguments", + j.to_string() + ); + }; + let json_reg = program.alloc_register(); + translate_expr(program, referenced_tables, &args[0], json_reg, resolver)?; + program.emit_insn(Insn::Function { + constant_mask: 0, + start_reg: json_reg, + dest: target_register, + func: func_ctx, + }); + Ok(target_register) + } }, Func::Scalar(srf) => { match srf { diff --git a/core/translate/group_by.rs b/core/translate/group_by.rs index c20466f27..2b6e7afb3 100644 --- a/core/translate/group_by.rs +++ b/core/translate/group_by.rs @@ -4,9 +4,13 @@ use sqlite3_parser::ast; use crate::{ function::AggFunc, - schema::{Column, PseudoTable, Table}, + schema::{Column, PseudoTable}, types::{OwnedRecord, OwnedValue}, - vdbe::{builder::ProgramBuilder, insn::Insn, BranchOffset}, + vdbe::{ + builder::{CursorType, ProgramBuilder}, + insn::Insn, + BranchOffset, + }, Result, }; @@ -50,7 +54,7 @@ pub fn init_group_by( ) -> Result<()> { let num_aggs = aggregates.len(); - let sort_cursor = program.alloc_cursor_id(None, None); + let sort_cursor = program.alloc_cursor_id(None, CursorType::Sorter); let reg_abort_flag = program.alloc_register(); let reg_group_exprs_cmp = program.alloc_registers(group_by.exprs.len()); @@ -175,7 +179,7 @@ pub fn emit_group_by<'a>( columns: pseudo_columns, }); - let pseudo_cursor = program.alloc_cursor_id(None, Some(Table::Pseudo(pseudo_table.clone()))); + let pseudo_cursor = program.alloc_cursor_id(None, CursorType::Pseudo(pseudo_table.clone())); program.emit_insn(Insn::OpenPseudo { cursor_id: pseudo_cursor, diff --git a/core/translate/insert.rs b/core/translate/insert.rs index b9f73ba8c..2dec74248 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -6,13 +6,18 @@ use sqlite3_parser::ast::{ }; use crate::error::SQLITE_CONSTRAINT_PRIMARYKEY; +use crate::schema::BTreeTable; use crate::util::normalize_ident; use crate::vdbe::BranchOffset; use crate::{ - schema::{Column, Schema, Table}, + schema::{Column, Schema}, storage::sqlite3_ondisk::DatabaseHeader, translate::expr::translate_expr, - vdbe::{builder::ProgramBuilder, insn::Insn, Program}, + vdbe::{ + builder::{CursorType, ProgramBuilder}, + insn::Insn, + Program, + }, SymbolTable, }; use crate::{Connection, Result}; @@ -53,20 +58,15 @@ pub fn translate_insert( Some(table) => table, None => crate::bail_corrupt_error!("Parse error: no such table: {}", table_name), }; - let table = Rc::new(Table::BTree(table)); - if !table.has_rowid() { + if !table.has_rowid { crate::bail_parse_error!("INSERT into WITHOUT ROWID table is not supported"); } let cursor_id = program.alloc_cursor_id( Some(table_name.0.clone()), - Some(table.clone().deref().clone()), + CursorType::BTreeTable(table.clone()), ); - let root_page = match table.as_ref() { - Table::BTree(btree) => btree.root_page, - Table::Index(index) => index.root_page, - Table::Pseudo(_) => todo!(), - }; + let root_page = table.root_page; let values = match body { InsertBody::Select(select, None) => match &select.body.select.deref() { sqlite3_parser::ast::OneSelect::Values(values) => values, @@ -77,9 +77,9 @@ pub fn translate_insert( let column_mappings = resolve_columns_for_insert(&table, columns, values)?; // Check if rowid was provided (through INTEGER PRIMARY KEY as a rowid alias) - let rowid_alias_index = table.columns().iter().position(|c| c.is_rowid_alias); + let rowid_alias_index = table.columns.iter().position(|c| c.is_rowid_alias); let has_user_provided_rowid = { - assert!(column_mappings.len() == table.columns().len()); + assert!(column_mappings.len() == table.columns.len()); if let Some(index) = rowid_alias_index { column_mappings[index].value_index.is_some() } else { @@ -89,7 +89,7 @@ pub fn translate_insert( // allocate a register for each column in the table. if not provided by user, they will simply be set as null. // allocate an extra register for rowid regardless of whether user provided a rowid alias column. - let num_cols = table.columns().len(); + let num_cols = table.columns.len(); let rowid_reg = program.alloc_registers(num_cols + 1); let column_registers_start = rowid_reg + 1; let rowid_alias_reg = { @@ -215,14 +215,14 @@ pub fn translate_insert( target_pc: make_record_label, }); let rowid_column_name = if let Some(index) = rowid_alias_index { - table.column_index_to_name(index).unwrap() + &table.columns.get(index).unwrap().name } else { "rowid" }; program.emit_insn(Insn::Halt { err_code: SQLITE_CONSTRAINT_PRIMARYKEY, - description: format!("{}.{}", table.get_name(), rowid_column_name), + description: format!("{}.{}", table_name.0, rowid_column_name), }); program.resolve_label(make_record_label, program.offset()); @@ -293,7 +293,7 @@ struct ColumnMapping<'a> { /// - Named columns map to their corresponding value index /// - Unspecified columns map to None fn resolve_columns_for_insert<'a>( - table: &'a Table, + table: &'a BTreeTable, columns: &Option, values: &[Vec], ) -> Result>> { @@ -301,7 +301,7 @@ fn resolve_columns_for_insert<'a>( crate::bail_parse_error!("no values to insert"); } - let table_columns = table.columns(); + let table_columns = &table.columns; // Case 1: No columns specified - map values to columns in order if columns.is_none() { @@ -309,7 +309,7 @@ fn resolve_columns_for_insert<'a>( if num_values > table_columns.len() { crate::bail_parse_error!( "table {} has {} columns but {} values were supplied", - table.get_name(), + &table.name, table_columns.len(), num_values ); @@ -350,11 +350,7 @@ fn resolve_columns_for_insert<'a>( .position(|c| c.name.eq_ignore_ascii_case(&column_name)); if table_index.is_none() { - crate::bail_parse_error!( - "table {} has no column named {}", - table.get_name(), - column_name - ); + crate::bail_parse_error!("table {} has no column named {}", &table.name, column_name); } mappings[table_index.unwrap()].value_index = Some(value_index); diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs index e95b3dc22..33f76a84e 100644 --- a/core/translate/main_loop.rs +++ b/core/translate/main_loop.rs @@ -1,9 +1,12 @@ use sqlite3_parser::ast; use crate::{ - schema::Table, translate::result_row::emit_select_result, - vdbe::{builder::ProgramBuilder, insn::Insn, BranchOffset}, + vdbe::{ + builder::{CursorType, ProgramBuilder}, + insn::Insn, + BranchOffset, + }, Result, }; @@ -81,7 +84,7 @@ pub fn init_loop( } => { let cursor_id = program.alloc_cursor_id( Some(table_reference.table_identifier.clone()), - Some(table_reference.table.clone()), + CursorType::BTreeTable(table_reference.btree().unwrap().clone()), ); let root_page = table_reference.table.get_root_page(); @@ -114,7 +117,7 @@ pub fn init_loop( } => { let table_cursor_id = program.alloc_cursor_id( Some(table_reference.table_identifier.clone()), - Some(table_reference.table.clone()), + CursorType::BTreeTable(table_reference.btree().unwrap().clone()), ); match mode { @@ -138,8 +141,10 @@ pub fn init_loop( } if let Search::IndexSearch { index, .. } = search { - let index_cursor_id = program - .alloc_cursor_id(Some(index.name.clone()), Some(Table::Index(index.clone()))); + let index_cursor_id = program.alloc_cursor_id( + Some(index.name.clone()), + CursorType::BTreeIndex(index.clone()), + ); match mode { OperationMode::SELECT => { diff --git a/core/translate/mod.rs b/core/translate/mod.rs index 92b661ce1..fdbbc47e0 100644 --- a/core/translate/mod.rs +++ b/core/translate/mod.rs @@ -27,6 +27,7 @@ use crate::storage::pager::Pager; use crate::storage::sqlite3_ondisk::{DatabaseHeader, MIN_PAGE_CACHE_SIZE}; use crate::translate::delete::translate_delete; use crate::util::PRIMARY_KEY_AUTOMATIC_INDEX_NAME_PREFIX; +use crate::vdbe::builder::CursorType; use crate::vdbe::{builder::ProgramBuilder, insn::Insn, Program}; use crate::{bail_parse_error, Connection, LimboError, Result, SymbolTable}; use insert::translate_insert; @@ -463,9 +464,10 @@ fn translate_create_table( let table_id = "sqlite_schema".to_string(); let table = schema.get_table(&table_id).unwrap(); - let table = crate::schema::Table::BTree(table.clone()); - let sqlite_schema_cursor_id = - program.alloc_cursor_id(Some(table_id.to_owned()), Some(table.to_owned())); + let sqlite_schema_cursor_id = program.alloc_cursor_id( + Some(table_id.to_owned()), + CursorType::BTreeTable(table.clone()), + ); program.emit_insn(Insn::OpenWriteAsync { cursor_id: sqlite_schema_cursor_id, root_page: 1, diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index b58e7ec21..1d02639d2 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -3,10 +3,13 @@ use std::rc::Rc; use sqlite3_parser::ast; use crate::{ - schema::{Column, PseudoTable, Table}, + schema::{Column, PseudoTable}, types::{OwnedRecord, OwnedValue}, util::exprs_are_equivalent, - vdbe::{builder::ProgramBuilder, insn::Insn}, + vdbe::{ + builder::{CursorType, ProgramBuilder}, + insn::Insn, + }, Result, }; @@ -32,7 +35,7 @@ pub fn init_order_by( t_ctx: &mut TranslateCtx, order_by: &[(ast::Expr, Direction)], ) -> Result<()> { - let sort_cursor = program.alloc_cursor_id(None, None); + let sort_cursor = program.alloc_cursor_id(None, CursorType::Sorter); t_ctx.meta_sort = Some(SortMetadata { sort_cursor, reg_sorter_data: program.alloc_register(), @@ -93,12 +96,10 @@ pub fn emit_order_by( .map(|v| v.len()) .unwrap_or(0); - let pseudo_cursor = program.alloc_cursor_id( - None, - Some(Table::Pseudo(Rc::new(PseudoTable { - columns: pseudo_columns, - }))), - ); + let pseudo_table = Rc::new(PseudoTable { + columns: pseudo_columns, + }); + let pseudo_cursor = program.alloc_cursor_id(None, CursorType::Pseudo(pseudo_table.clone())); let SortMetadata { sort_cursor, reg_sorter_data, diff --git a/core/translate/plan.rs b/core/translate/plan.rs index b773d1eec..bffd384d3 100644 --- a/core/translate/plan.rs +++ b/core/translate/plan.rs @@ -7,7 +7,7 @@ use std::{ use crate::{ function::AggFunc, - schema::{Column, Index, Table}, + schema::{BTreeTable, Column, Index, Table}, vdbe::BranchOffset, Result, }; @@ -255,6 +255,12 @@ pub struct TableReference { } impl TableReference { + pub fn btree(&self) -> Option> { + match self.reference_type { + TableReferenceType::BTreeTable => self.table.btree(), + TableReferenceType::Subquery { .. } => None, + } + } pub fn new_subquery(identifier: String, table_index: usize, plan: &SelectPlan) -> Self { Self { table: Table::Pseudo(Rc::new(PseudoTable::new_with_columns( diff --git a/core/types.rs b/core/types.rs index 7b5814397..d9a496bfb 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1,5 +1,5 @@ use std::fmt::Display; -use std::{cell::Ref, rc::Rc}; +use std::rc::Rc; use crate::error::LimboError; use crate::Result; @@ -524,31 +524,6 @@ pub enum SeekKey<'a> { IndexKey(&'a OwnedRecord), } -pub trait Cursor { - fn is_empty(&self) -> bool; - fn root_page(&self) -> usize; - fn rewind(&mut self) -> Result>; - fn last(&mut self) -> Result>; - fn next(&mut self) -> Result>; - fn prev(&mut self) -> Result>; - fn wait_for_completion(&mut self) -> Result<()>; - fn rowid(&self) -> Result>; - fn seek(&mut self, key: SeekKey, op: SeekOp) -> Result>; - fn seek_to_last(&mut self) -> Result>; - fn record(&self) -> Result>>; - fn insert( - &mut self, - key: &OwnedValue, - record: &OwnedRecord, - moved_before: bool, /* Tells inserter that it doesn't need to traverse in order to find leaf page */ - ) -> Result>; // - fn delete(&mut self) -> Result>; - fn exists(&mut self, key: &OwnedValue) -> Result>; - fn set_null_flag(&mut self, flag: bool); - fn get_null_flag(&self) -> bool; - fn btree_create(&mut self, flags: usize) -> u32; -} - #[cfg(test)] mod tests { use super::*; diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs index 40f936cd1..7acc4be6f 100644 --- a/core/vdbe/builder.rs +++ b/core/vdbe/builder.rs @@ -4,9 +4,13 @@ use std::{ rc::{Rc, Weak}, }; -use crate::{storage::sqlite3_ondisk::DatabaseHeader, Connection}; +use crate::{ + schema::{BTreeTable, Index, PseudoTable}, + storage::sqlite3_ondisk::DatabaseHeader, + Connection, +}; -use super::{BranchOffset, CursorID, Insn, InsnReference, Program, Table}; +use super::{BranchOffset, CursorID, Insn, InsnReference, Program}; #[allow(dead_code)] pub struct ProgramBuilder { @@ -18,7 +22,7 @@ pub struct ProgramBuilder { constant_insns: Vec, next_insn_label: Option, // Cursors that are referenced by the program. Indexed by CursorID. - pub cursor_ref: Vec<(Option, Option)>, + pub cursor_ref: Vec<(Option, CursorType)>, // Hashmap of label to insn reference. Resolved in build(). label_to_resolved_offset: HashMap, // Bitmask of cursors that have emitted a SeekRowid instruction. @@ -27,6 +31,20 @@ pub struct ProgramBuilder { comments: HashMap, } +#[derive(Debug, Clone)] +pub enum CursorType { + BTreeTable(Rc), + BTreeIndex(Rc), + Pseudo(Rc), + Sorter, +} + +impl CursorType { + pub fn is_index(&self) -> bool { + matches!(self, CursorType::BTreeIndex(_)) + } +} + impl ProgramBuilder { pub fn new() -> Self { Self { @@ -58,11 +76,11 @@ impl ProgramBuilder { pub fn alloc_cursor_id( &mut self, table_identifier: Option, - table: Option
, + cursor_type: CursorType, ) -> usize { let cursor = self.next_free_cursor_id; self.next_free_cursor_id += 1; - self.cursor_ref.push((table_identifier, table)); + self.cursor_ref.push((table_identifier, cursor_type)); assert!(self.cursor_ref.len() == self.next_free_cursor_id); cursor } diff --git a/core/vdbe/datetime.rs b/core/vdbe/datetime.rs index ec995290c..b0497b0a8 100644 --- a/core/vdbe/datetime.rs +++ b/core/vdbe/datetime.rs @@ -884,8 +884,6 @@ mod tests { #[test] fn test_valid_get_time_from_datetime_value() { - let now = chrono::Local::now().to_utc().format("%H:%M:%S").to_string(); - let test_time_str = "22:30:45"; let prev_time_str = "20:30:45"; let next_time_str = "03:30:45"; @@ -1049,8 +1047,6 @@ mod tests { OwnedValue::build_text(Rc::new("22:30:45.123Z".to_string())), test_time_str, ), - // Test Format 11: 'now' - (OwnedValue::build_text(Rc::new("now".to_string())), &now), // Format 12: DDDDDDDDDD (Julian date as float or integer) (OwnedValue::Float(2460082.1), "14:24:00"), (OwnedValue::Integer(2460082), "12:00:00"), diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index 133172b78..40fda1a28 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -1,3 +1,5 @@ +use crate::vdbe::builder::CursorType; + use super::{Insn, InsnReference, OwnedValue, Program}; use std::rc::Rc; @@ -387,7 +389,19 @@ pub fn insn_to_str( column, dest, } => { - let (table_identifier, table) = &program.cursor_ref[*cursor_id]; + let (table_identifier, cursor_type) = &program.cursor_ref[*cursor_id]; + let column_name = match cursor_type { + CursorType::BTreeTable(table) => { + Some(&table.columns.get(*column).unwrap().name) + } + CursorType::BTreeIndex(index) => { + Some(&index.columns.get(*column).unwrap().name) + } + CursorType::Pseudo(pseudo_table) => { + Some(&pseudo_table.columns.get(*column).unwrap().name) + } + CursorType::Sorter => None, + }; ( "Column", *cursor_id as i32, @@ -401,10 +415,7 @@ pub fn insn_to_str( table_identifier .as_ref() .unwrap_or(&format!("cursor {}", cursor_id)), - table - .as_ref() - .and_then(|x| x.column_index_to_name(*column)) - .unwrap_or(format!("column {}", *column).as_str()) + column_name.unwrap_or(&format!("column {}", *column)) ), ) } diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 208c002a6..28db889cf 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -30,18 +30,17 @@ use crate::ext::{exec_ts_from_uuid7, exec_uuid, exec_uuidblob, exec_uuidstr, Ext use crate::function::{AggFunc, FuncCtx, MathFunc, MathFuncArity, ScalarFunc}; use crate::pseudo::PseudoCursor; use crate::result::LimboResult; -use crate::schema::Table; use crate::storage::sqlite3_ondisk::DatabaseHeader; use crate::storage::{btree::BTreeCursor, pager::Pager}; -use crate::types::{ - AggContext, Cursor, CursorResult, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp, -}; +use crate::types::{AggContext, CursorResult, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp}; use crate::util::parse_schema_rows; +use crate::vdbe::builder::CursorType; use crate::vdbe::insn::Insn; #[cfg(feature = "json")] use crate::{ function::JsonFunc, json::get_json, json::json_array, json::json_array_length, - json::json_arrow_extract, json::json_arrow_shift_extract, json::json_extract, json::json_type, + json::json_arrow_extract, json::json_arrow_shift_extract, json::json_error_position, + json::json_extract, json::json_type, }; use crate::{Connection, Result, Rows, TransactionState, DATABASE_VERSION}; use datetime::{exec_date, exec_datetime_full, exec_julianday, exec_time, exec_unixepoch}; @@ -53,6 +52,7 @@ use likeop::{construct_like_escape_arg, exec_glob, exec_like_with_escape}; use rand::distributions::{Distribution, Uniform}; use rand::{thread_rng, Rng}; use regex::{Regex, RegexBuilder}; +use sorter::Sorter; use std::borrow::{Borrow, BorrowMut}; use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; @@ -164,7 +164,10 @@ impl RegexCache { /// The program state describes the environment in which the program executes. pub struct ProgramState { pub pc: InsnReference, - cursors: RefCell>>, + btree_table_cursors: RefCell>, + btree_index_cursors: RefCell>, + pseudo_cursors: RefCell>, + sorter_cursors: RefCell>, registers: Vec, last_compare: Option, deferred_seek: Option<(CursorID, CursorID)>, @@ -175,12 +178,18 @@ pub struct ProgramState { impl ProgramState { pub fn new(max_registers: usize) -> Self { - let cursors = RefCell::new(BTreeMap::new()); + let btree_table_cursors = RefCell::new(BTreeMap::new()); + let btree_index_cursors = RefCell::new(BTreeMap::new()); + let pseudo_cursors = RefCell::new(BTreeMap::new()); + let sorter_cursors = RefCell::new(BTreeMap::new()); let mut registers = Vec::with_capacity(max_registers); registers.resize(max_registers, OwnedValue::Null); Self { pc: 0, - cursors, + btree_table_cursors, + btree_index_cursors, + pseudo_cursors, + sorter_cursors, registers, last_compare: None, deferred_seek: None, @@ -207,11 +216,24 @@ impl ProgramState { } } +macro_rules! must_be_btree_cursor { + ($cursor_id:expr, $cursor_ref:expr, $btree_table_cursors:expr, $btree_index_cursors:expr, $insn_name:expr) => {{ + let (_, cursor_type) = $cursor_ref.get($cursor_id).unwrap(); + let cursor = match cursor_type { + CursorType::BTreeTable(_) => $btree_table_cursors.get_mut(&$cursor_id).unwrap(), + CursorType::BTreeIndex(_) => $btree_index_cursors.get_mut(&$cursor_id).unwrap(), + CursorType::Pseudo(_) => panic!("{} on pseudo cursor", $insn_name), + CursorType::Sorter => panic!("{} on sorter cursor", $insn_name), + }; + cursor + }}; +} + #[derive(Debug)] pub struct Program { pub max_registers: usize, pub insns: Vec, - pub cursor_ref: Vec<(Option, Option
)>, + pub cursor_ref: Vec<(Option, CursorType)>, pub database_header: Rc>, pub comments: HashMap, pub connection: Weak, @@ -248,7 +270,10 @@ impl Program { } let insn = &self.insns[state.pc as usize]; trace_insn(self, state.pc as InsnReference, insn); - let mut cursors = state.cursors.borrow_mut(); + let mut btree_table_cursors = state.btree_table_cursors.borrow_mut(); + let mut btree_index_cursors = state.btree_index_cursors.borrow_mut(); + let mut pseudo_cursors = state.pseudo_cursors.borrow_mut(); + let mut sorter_cursors = state.sorter_cursors.borrow_mut(); match insn { Insn::Init { target_pc } => { assert!(target_pc.is_offset()); @@ -304,7 +329,13 @@ impl Program { state.pc += 1; } Insn::NullRow { cursor_id } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "NullRow" + ); cursor.set_null_flag(true); state.pc += 1; } @@ -569,12 +600,23 @@ impl Program { cursor_id, root_page, } => { - let cursor = Box::new(BTreeCursor::new( - pager.clone(), - *root_page, - self.database_header.clone(), - )); - cursors.insert(*cursor_id, cursor); + let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap(); + let cursor = + BTreeCursor::new(pager.clone(), *root_page, self.database_header.clone()); + match cursor_type { + CursorType::BTreeTable(_) => { + btree_table_cursors.insert(*cursor_id, cursor); + } + CursorType::BTreeIndex(_) => { + btree_index_cursors.insert(*cursor_id, cursor); + } + CursorType::Pseudo(_) => { + panic!("OpenReadAsync on pseudo cursor"); + } + CursorType::Sorter => { + panic!("OpenReadAsync on sorter cursor"); + } + } state.pc += 1; } Insn::OpenReadAwait => { @@ -585,17 +627,29 @@ impl Program { content_reg: _, num_fields: _, } => { - let cursor = Box::new(PseudoCursor::new()); - cursors.insert(*cursor_id, cursor); + let cursor = PseudoCursor::new(); + pseudo_cursors.insert(*cursor_id, cursor); state.pc += 1; } Insn::RewindAsync { cursor_id } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "RewindAsync" + ); return_if_io!(cursor.rewind()); state.pc += 1; } Insn::LastAsync { cursor_id } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "LastAsync" + ); return_if_io!(cursor.last()); state.pc += 1; } @@ -604,7 +658,13 @@ impl Program { pc_if_empty, } => { assert!(pc_if_empty.is_offset()); - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "LastAwait" + ); cursor.wait_for_completion()?; if cursor.is_empty() { state.pc = pc_if_empty.to_offset_int(); @@ -617,7 +677,13 @@ impl Program { pc_if_empty, } => { assert!(pc_if_empty.is_offset()); - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "RewindAwait" + ); cursor.wait_for_completion()?; if cursor.is_empty() { state.pc = pc_if_empty.to_offset_int(); @@ -631,9 +697,9 @@ impl Program { dest, } => { if let Some((index_cursor_id, table_cursor_id)) = state.deferred_seek.take() { - let index_cursor = cursors.get_mut(&index_cursor_id).unwrap(); + let index_cursor = btree_index_cursors.get_mut(&index_cursor_id).unwrap(); let rowid = index_cursor.rowid()?; - let table_cursor = cursors.get_mut(&table_cursor_id).unwrap(); + let table_cursor = btree_table_cursors.get_mut(&table_cursor_id).unwrap(); match table_cursor.seek(SeekKey::TableRowId(rowid.unwrap()), SeekOp::EQ)? { CursorResult::Ok(_) => {} CursorResult::IO => { @@ -642,18 +708,45 @@ impl Program { } } } - - let cursor = cursors.get_mut(cursor_id).unwrap(); - if let Some(ref record) = *cursor.record()? { - let null_flag = cursor.get_null_flag(); - state.registers[*dest] = if null_flag { - OwnedValue::Null - } else { - record.values[*column].clone() - }; - } else { - state.registers[*dest] = OwnedValue::Null; + let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap(); + match cursor_type { + CursorType::BTreeTable(_) | CursorType::BTreeIndex(_) => { + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "Column" + ); + let record = cursor.record()?; + if let Some(record) = record.as_ref() { + state.registers[*dest] = if cursor.get_null_flag() { + OwnedValue::Null + } else { + record.values[*column].clone() + }; + } else { + state.registers[*dest] = OwnedValue::Null; + } + } + CursorType::Sorter => { + let cursor = sorter_cursors.get_mut(cursor_id).unwrap(); + if let Some(record) = cursor.record() { + state.registers[*dest] = record.values[*column].clone(); + } else { + state.registers[*dest] = OwnedValue::Null; + } + } + CursorType::Pseudo(_) => { + let cursor = pseudo_cursors.get_mut(cursor_id).unwrap(); + if let Some(record) = cursor.record() { + state.registers[*dest] = record.values[*column].clone(); + } else { + state.registers[*dest] = OwnedValue::Null; + } + } } + state.pc += 1; } Insn::MakeRecord { @@ -671,13 +764,25 @@ impl Program { return Ok(StepResult::Row(record)); } Insn::NextAsync { cursor_id } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "NextAsync" + ); cursor.set_null_flag(false); return_if_io!(cursor.next()); state.pc += 1; } Insn::PrevAsync { cursor_id } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "PrevAsync" + ); cursor.set_null_flag(false); return_if_io!(cursor.prev()); state.pc += 1; @@ -687,7 +792,13 @@ impl Program { pc_if_next, } => { assert!(pc_if_next.is_offset()); - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "PrevAwait" + ); cursor.wait_for_completion()?; if !cursor.is_empty() { state.pc = pc_if_next.to_offset_int(); @@ -700,7 +811,13 @@ impl Program { pc_if_next, } => { assert!(pc_if_next.is_offset()); - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor_id, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "NextAwait" + ); cursor.wait_for_completion()?; if !cursor.is_empty() { state.pc = pc_if_next.to_offset_int(); @@ -818,9 +935,9 @@ impl Program { } Insn::RowId { cursor_id, dest } => { if let Some((index_cursor_id, table_cursor_id)) = state.deferred_seek.take() { - let index_cursor = cursors.get_mut(&index_cursor_id).unwrap(); + let index_cursor = btree_index_cursors.get_mut(&index_cursor_id).unwrap(); let rowid = index_cursor.rowid()?; - let table_cursor = cursors.get_mut(&table_cursor_id).unwrap(); + let table_cursor = btree_table_cursors.get_mut(&table_cursor_id).unwrap(); match table_cursor.seek(SeekKey::TableRowId(rowid.unwrap()), SeekOp::EQ)? { CursorResult::Ok(_) => {} CursorResult::IO => { @@ -830,7 +947,7 @@ impl Program { } } - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor_id).unwrap(); if let Some(ref rowid) = cursor.rowid()? { state.registers[*dest] = OwnedValue::Integer(*rowid as i64); } else { @@ -844,7 +961,7 @@ impl Program { target_pc, } => { assert!(target_pc.is_offset()); - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor_id).unwrap(); let rowid = match &state.registers[*src_reg] { OwnedValue::Integer(rowid) => *rowid as u64, OwnedValue::Null => { @@ -880,7 +997,7 @@ impl Program { } => { assert!(target_pc.is_offset()); if *is_index { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_index_cursors.get_mut(cursor_id).unwrap(); let record_from_regs: OwnedRecord = make_owned_record(&state.registers, start_reg, num_regs); let found = return_if_io!( @@ -892,7 +1009,7 @@ impl Program { state.pc += 1; } } else { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor_id).unwrap(); let rowid = match &state.registers[*start_reg] { OwnedValue::Null => { // All integer values are greater than null so we just rewind the cursor @@ -925,7 +1042,7 @@ impl Program { } => { assert!(target_pc.is_offset()); if *is_index { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_index_cursors.get_mut(cursor_id).unwrap(); let record_from_regs: OwnedRecord = make_owned_record(&state.registers, start_reg, num_regs); let found = return_if_io!( @@ -937,7 +1054,7 @@ impl Program { state.pc += 1; } } else { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor_id).unwrap(); let rowid = match &state.registers[*start_reg] { OwnedValue::Null => { // All integer values are greater than null so we just rewind the cursor @@ -968,7 +1085,7 @@ impl Program { target_pc, } => { assert!(target_pc.is_offset()); - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_index_cursors.get_mut(cursor_id).unwrap(); let record_from_regs: OwnedRecord = make_owned_record(&state.registers, start_reg, num_regs); if let Some(ref idx_record) = *cursor.record()? { @@ -991,7 +1108,7 @@ impl Program { target_pc, } => { assert!(target_pc.is_offset()); - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_index_cursors.get_mut(cursor_id).unwrap(); let record_from_regs: OwnedRecord = make_owned_record(&state.registers, start_reg, num_regs); if let Some(ref idx_record) = *cursor.record()? { @@ -1275,47 +1392,46 @@ impl Program { _ => unreachable!(), }) .collect(); - let cursor = Box::new(sorter::Sorter::new(order)); - cursors.insert(*cursor_id, cursor); + let cursor = sorter::Sorter::new(order); + sorter_cursors.insert(*cursor_id, cursor); state.pc += 1; } Insn::SorterData { cursor_id, dest_reg, - pseudo_cursor: sorter_cursor, + pseudo_cursor, } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); - let record = match *cursor.record()? { - Some(ref record) => record.clone(), + let sorter_cursor = sorter_cursors.get_mut(cursor_id).unwrap(); + let record = match sorter_cursor.record() { + Some(record) => record.clone(), None => { state.pc += 1; continue; } }; state.registers[*dest_reg] = OwnedValue::Record(record.clone()); - let sorter_cursor = cursors.get_mut(sorter_cursor).unwrap(); - sorter_cursor.insert(&OwnedValue::Integer(0), &record, false)?; // fix key later + let pseudo_cursor = pseudo_cursors.get_mut(pseudo_cursor).unwrap(); + pseudo_cursor.insert(record); state.pc += 1; } Insn::SorterInsert { cursor_id, record_reg, } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = sorter_cursors.get_mut(cursor_id).unwrap(); let record = match &state.registers[*record_reg] { OwnedValue::Record(record) => record, _ => unreachable!("SorterInsert on non-record register"), }; - // TODO: set correct key - cursor.insert(&OwnedValue::Integer(0), record, false)?; + cursor.insert(record); state.pc += 1; } Insn::SorterSort { cursor_id, pc_if_empty, } => { - if let Some(cursor) = cursors.get_mut(cursor_id) { - cursor.rewind()?; + if let Some(cursor) = sorter_cursors.get_mut(cursor_id) { + cursor.sort(); state.pc += 1; } else { state.pc = pc_if_empty.to_offset_int(); @@ -1326,8 +1442,8 @@ impl Program { pc_if_next, } => { assert!(pc_if_next.is_offset()); - let cursor = cursors.get_mut(cursor_id).unwrap(); - return_if_io!(cursor.next()); + let cursor = sorter_cursors.get_mut(cursor_id).unwrap(); + cursor.next(); if !cursor.is_empty() { state.pc = pc_if_next.to_offset_int(); } else { @@ -1415,6 +1531,13 @@ impl Program { Err(e) => return Err(e), } } + JsonFunc::JsonErrorPosition => { + let json_value = &state.registers[*start_reg]; + match json_error_position(json_value) { + Ok(pos) => state.registers[*dest] = pos, + Err(e) => return Err(e), + } + } }, crate::function::Func::Scalar(scalar_func) => match scalar_func { ScalarFunc::Cast => { @@ -1870,7 +1993,7 @@ impl Program { record_reg, flag: _, } => { - let cursor = cursors.get_mut(cursor).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor).unwrap(); let record = match &state.registers[*record_reg] { OwnedValue::Record(r) => r, _ => unreachable!("Not a record! Cannot insert a non record value."), @@ -1880,7 +2003,7 @@ impl Program { state.pc += 1; } Insn::InsertAwait { cursor_id } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor_id).unwrap(); cursor.wait_for_completion()?; // Only update last_insert_rowid for regular table inserts, not schema modifications if cursor.root_page() != 1 { @@ -1896,19 +2019,19 @@ impl Program { state.pc += 1; } Insn::DeleteAsync { cursor_id } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor_id).unwrap(); return_if_io!(cursor.delete()); state.pc += 1; } Insn::DeleteAwait { cursor_id } => { - let cursor = cursors.get_mut(cursor_id).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor_id).unwrap(); cursor.wait_for_completion()?; state.pc += 1; } Insn::NewRowid { cursor, rowid_reg, .. } => { - let cursor = cursors.get_mut(cursor).unwrap(); + let cursor = btree_table_cursors.get_mut(cursor).unwrap(); // TODO: make io handle rng let rowid = return_if_io!(get_new_rowid(cursor, thread_rng())); state.registers[*rowid_reg] = OwnedValue::Integer(rowid); @@ -1934,7 +2057,13 @@ impl Program { rowid_reg, target_pc, } => { - let cursor = cursors.get_mut(cursor).unwrap(); + let cursor = must_be_btree_cursor!( + *cursor, + self.cursor_ref, + btree_table_cursors, + btree_index_cursors, + "NotExists" + ); let exists = return_if_io!(cursor.exists(&state.registers[*rowid_reg])); if exists { state.pc += 1; @@ -1949,12 +2078,15 @@ impl Program { cursor_id, root_page, } => { - let cursor = Box::new(BTreeCursor::new( - pager.clone(), - *root_page, - self.database_header.clone(), - )); - cursors.insert(*cursor_id, cursor); + let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap(); + let is_index = cursor_type.is_index(); + let cursor = + BTreeCursor::new(pager.clone(), *root_page, self.database_header.clone()); + if is_index { + btree_index_cursors.insert(*cursor_id, cursor); + } else { + btree_table_cursors.insert(*cursor_id, cursor); + } state.pc += 1; } Insn::OpenWriteAwait {} => { @@ -1986,7 +2118,21 @@ impl Program { state.pc += 1; } Insn::Close { cursor_id } => { - cursors.remove(cursor_id); + let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap(); + match cursor_type { + CursorType::BTreeTable(_) => { + let _ = btree_table_cursors.remove(cursor_id); + } + CursorType::BTreeIndex(_) => { + let _ = btree_index_cursors.remove(cursor_id); + } + CursorType::Pseudo(_) => { + let _ = pseudo_cursors.remove(cursor_id); + } + CursorType::Sorter => { + let _ = sorter_cursors.remove(cursor_id); + } + } state.pc += 1; } Insn::IsNull { src, target_pc } => { @@ -2017,7 +2163,7 @@ impl Program { } } -fn get_new_rowid(cursor: &mut Box, mut rng: R) -> Result> { +fn get_new_rowid(cursor: &mut BTreeCursor, mut rng: R) -> Result> { match cursor.seek_to_last()? { CursorResult::Ok(()) => {} CursorResult::IO => return Ok(CursorResult::IO), @@ -3055,180 +3201,16 @@ fn exec_math_log(arg: &OwnedValue, base: Option<&OwnedValue>) -> OwnedValue { #[cfg(test)] mod tests { - use crate::{ - types::{SeekKey, SeekOp}, - vdbe::exec_replace, - }; + use crate::vdbe::exec_replace; use super::{ exec_abs, exec_char, exec_hex, exec_if, exec_instr, exec_length, exec_like, exec_lower, exec_ltrim, exec_max, exec_min, exec_nullif, exec_quote, exec_random, exec_randomblob, exec_round, exec_rtrim, exec_sign, exec_soundex, exec_substring, exec_trim, exec_typeof, - exec_unhex, exec_unicode, exec_upper, exec_zeroblob, execute_sqlite_version, get_new_rowid, - AggContext, Cursor, CursorResult, LimboError, OwnedRecord, OwnedValue, Result, + exec_unhex, exec_unicode, exec_upper, exec_zeroblob, execute_sqlite_version, AggContext, + OwnedValue, }; - use mockall::{mock, predicate}; - use rand::{rngs::mock::StepRng, thread_rng}; - use std::{cell::Ref, collections::HashMap, rc::Rc}; - - mock! { - Cursor { - fn seek_to_last(&mut self) -> Result>; - fn seek<'a>(&mut self, key: SeekKey<'a>, op: SeekOp) -> Result>; - fn rowid(&self) -> Result>; - fn seek_rowid(&mut self, rowid: u64) -> Result>; - } - } - - impl Cursor for MockCursor { - fn root_page(&self) -> usize { - unreachable!() - } - - fn seek_to_last(&mut self) -> Result> { - self.seek_to_last() - } - - fn rowid(&self) -> Result> { - self.rowid() - } - - fn seek(&mut self, key: SeekKey<'_>, op: SeekOp) -> Result> { - self.seek(key, op) - } - - fn rewind(&mut self) -> Result> { - unimplemented!() - } - - fn next(&mut self) -> Result> { - unimplemented!() - } - - fn record(&self) -> Result>> { - unimplemented!() - } - - fn is_empty(&self) -> bool { - unimplemented!() - } - - fn set_null_flag(&mut self, _flag: bool) { - unimplemented!() - } - - fn get_null_flag(&self) -> bool { - unimplemented!() - } - - fn insert( - &mut self, - _key: &OwnedValue, - _record: &OwnedRecord, - _is_leaf: bool, - ) -> Result> { - unimplemented!() - } - - fn delete(&mut self) -> Result> { - unimplemented!() - } - - fn wait_for_completion(&mut self) -> Result<()> { - unimplemented!() - } - - fn exists(&mut self, _key: &OwnedValue) -> Result> { - unimplemented!() - } - - fn btree_create(&mut self, _flags: usize) -> u32 { - unimplemented!() - } - - fn last(&mut self) -> Result> { - todo!() - } - - fn prev(&mut self) -> Result> { - todo!() - } - } - - #[test] - fn test_get_new_rowid() -> Result<()> { - // Test case 0: Empty table - let mut mock = MockCursor::new(); - mock.expect_seek_to_last() - .return_once(|| Ok(CursorResult::Ok(()))); - mock.expect_rowid().return_once(|| Ok(None)); - - let result = get_new_rowid(&mut (Box::new(mock) as Box), thread_rng())?; - assert_eq!( - result, - CursorResult::Ok(1), - "For an empty table, rowid should be 1" - ); - - // Test case 1: Normal case, rowid within i64::MAX - let mut mock = MockCursor::new(); - mock.expect_seek_to_last() - .return_once(|| Ok(CursorResult::Ok(()))); - mock.expect_rowid().return_once(|| Ok(Some(100))); - - let result = get_new_rowid(&mut (Box::new(mock) as Box), thread_rng())?; - assert_eq!(result, CursorResult::Ok(101)); - - // Test case 2: Rowid exceeds i64::MAX, need to generate random rowid - let mut mock = MockCursor::new(); - mock.expect_seek_to_last() - .return_once(|| Ok(CursorResult::Ok(()))); - mock.expect_rowid() - .return_once(|| Ok(Some(i64::MAX as u64))); - mock.expect_seek() - .with(predicate::always(), predicate::always()) - .returning(|rowid, _| { - if rowid == SeekKey::TableRowId(50) { - Ok(CursorResult::Ok(false)) - } else { - Ok(CursorResult::Ok(true)) - } - }); - - // Mock the random number generation - let new_rowid = - get_new_rowid(&mut (Box::new(mock) as Box), StepRng::new(1, 1))?; - assert_eq!(new_rowid, CursorResult::Ok(50)); - - // Test case 3: IO error - let mut mock = MockCursor::new(); - mock.expect_seek_to_last() - .return_once(|| Ok(CursorResult::Ok(()))); - mock.expect_rowid() - .return_once(|| Ok(Some(i64::MAX as u64))); - mock.expect_seek() - .with(predicate::always(), predicate::always()) - .return_once(|_, _| Ok(CursorResult::IO)); - - let result = get_new_rowid(&mut (Box::new(mock) as Box), thread_rng()); - assert!(matches!(result, Ok(CursorResult::IO))); - - // Test case 4: Failure to generate new rowid - let mut mock = MockCursor::new(); - mock.expect_seek_to_last() - .return_once(|| Ok(CursorResult::Ok(()))); - mock.expect_rowid() - .return_once(|| Ok(Some(i64::MAX as u64))); - mock.expect_seek() - .with(predicate::always(), predicate::always()) - .returning(|_, _| Ok(CursorResult::Ok(true))); - - // Mock the random number generation - let result = get_new_rowid(&mut (Box::new(mock) as Box), StepRng::new(1, 1)); - assert!(matches!(result, Err(LimboError::InternalError(_)))); - - Ok(()) - } + use std::{collections::HashMap, rc::Rc}; #[test] fn test_length() { diff --git a/core/vdbe/sorter.rs b/core/vdbe/sorter.rs index e3962b096..d23a3bef0 100644 --- a/core/vdbe/sorter.rs +++ b/core/vdbe/sorter.rs @@ -1,13 +1,9 @@ -use crate::{ - types::{Cursor, CursorResult, OwnedRecord, OwnedValue, SeekKey, SeekOp}, - Result, -}; -use std::cell::{Ref, RefCell}; +use crate::types::OwnedRecord; use std::cmp::Ordering; pub struct Sorter { records: Vec, - current: RefCell>, + current: Option, order: Vec, } @@ -15,23 +11,15 @@ impl Sorter { pub fn new(order: Vec) -> Self { Self { records: Vec::new(), - current: RefCell::new(None), + current: None, order, } } -} - -impl Cursor for Sorter { - fn is_empty(&self) -> bool { - self.current.borrow().is_none() + pub fn is_empty(&self) -> bool { + self.current.is_none() } - - fn root_page(&self) -> usize { - unreachable!() - } - // We do the sorting here since this is what is called by the SorterSort instruction - fn rewind(&mut self) -> Result> { + pub fn sort(&mut self) { self.records.sort_by(|a, b| { let cmp_by_idx = |idx: usize, ascending: bool| { let a = &a.values[idx]; @@ -55,73 +43,14 @@ impl Cursor for Sorter { self.records.reverse(); self.next() } - - fn next(&mut self) -> Result> { - let mut c = self.current.borrow_mut(); - *c = self.records.pop(); - Ok(CursorResult::Ok(())) + pub fn next(&mut self) { + self.current = self.records.pop(); + } + pub fn record(&self) -> Option<&OwnedRecord> { + self.current.as_ref() } - fn wait_for_completion(&mut self) -> Result<()> { - Ok(()) - } - - fn rowid(&self) -> Result> { - todo!(); - } - - fn seek(&mut self, _: SeekKey<'_>, _: SeekOp) -> Result> { - unimplemented!(); - } - - fn seek_to_last(&mut self) -> Result> { - unimplemented!(); - } - - fn record(&self) -> Result>> { - let ret = self.current.borrow(); - // log::trace!("returning {:?}", ret); - Ok(ret) - } - - fn insert( - &mut self, - key: &OwnedValue, - record: &OwnedRecord, - moved_before: bool, - ) -> Result> { - let _ = key; - let _ = moved_before; + pub fn insert(&mut self, record: &OwnedRecord) { self.records.push(OwnedRecord::new(record.values.to_vec())); - Ok(CursorResult::Ok(())) - } - - fn delete(&mut self) -> Result> { - unimplemented!() - } - - fn set_null_flag(&mut self, _flag: bool) { - todo!(); - } - - fn get_null_flag(&self) -> bool { - false - } - - fn exists(&mut self, key: &OwnedValue) -> Result> { - let _ = key; - todo!() - } - - fn btree_create(&mut self, _flags: usize) -> u32 { - unreachable!("Why did you try to build a new tree with a sorter??? Stand up, open the door and take a walk for 30 min to come back with a better plan."); - } - - fn last(&mut self) -> Result> { - todo!() - } - - fn prev(&mut self) -> Result> { - todo!() } } diff --git a/testing/json.test b/testing/json.test index a4df0e902..758243bcd 100755 --- a/testing/json.test +++ b/testing/json.test @@ -474,3 +474,35 @@ do_execsql_test json_type_cast { do_execsql_test json_type_null_arg { select json_type(null) } {{}} + +do_execsql_test json_error_position_valid { + SELECT json_error_position('{"a":55,"b":72,}'); +} {{0}} + +do_execsql_test json_error_position_valid_ws { + SELECT json_error_position('{"a":55,"b":72 , }'); +} {{0}} + +do_execsql_test json_error_position_object { + SELECT json_error_position('{"a":55,"b":72,,}'); +} {{16}} + +do_execsql_test json_error_position_array_valid { + SELECT json_error_position('["a",55,"b",72,]'); +} {{0}} + +do_execsql_test json_error_position_array_valid_ws { + SELECT json_error_position('["a",55,"b",72 , ]'); +} {{0}} + +do_execsql_test json_error_position_array { + SELECT json_error_position('["a",55,"b",72,,]'); +} {{16}} + +do_execsql_test json_error_position_null { + SELECT json_error_position(NULL); +} {{}} + +do_execsql_test json_error_position_complex { + SELECT json_error_position('{a:null,{"h":[1,[1,2,3]],"j":"abc"}:true}'); +} {{9}}