aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-02-19 23:15:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge with main

Browse Source
This commit is contained in:
Jorge Hermo
2025-01-13 22:20:17 +01:00
parent 016d9d17ab 96148af331
commit df3123e128
35 changed files with 811 additions and 788 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
COMPAT.md
View File

@@ -263,7 +263,7 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
| jsonb_array(value1,value2,...) | | |
| json_array_length(json) | Yes | |
| json_array_length(json,path) | Yes | |
| json_error_position(json) | | |
| json_error_position(json) | Yes | |
| json_extract(json,path,...) | Partial | Does not fully support unicode literal syntax and does not allow numbers > 2^127 - 1 (which SQLite truncates to i32), does not support BLOBs |
| jsonb_extract(json,path,...) | | |
| json -> path | Yes | |

22
CONTRIBUTING.md
View File

@@ -121,3 +121,25 @@ The `simulator` directory contains a deterministic simulator for testing.
What this means is that the behavior of a test run is deterministic based on the seed value.
If the simulator catches a bug, you can always reproduce the exact same sequence of events by passing the same seed.
The simulator also performs fault injection to discover interesting bugs.
## Python Bindings
Limbo provides Python bindings built on top of the [PyO3](https://pyo3.rs) project.
To compile the Python bindings locally, you first need to create and activate a Python virtual environment (for example, with Python `3.12`):
```bash
python3.12 -m venv venv
source venv/bin/activate
```
Then, install [Maturin](https://pypi.org/project/maturin/):
```bash
pip install maturin
```
Once Maturin is installed, you can build the crate and install it as a Python module directly into the current virtual environment by running:
```bash
cd bindings/python && maturin develop
```

2
bindings/java/rs_src/limbo_db.rs
View File

@@ -8,7 +8,7 @@ const ERROR_CODE_ETC: i32 = 9999;
#[no_mangle]
#[allow(clippy::arc_with_non_send_sync)]
pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB__1open_1utf8<'local>(
pub extern "system" fn Java_org_github_tursodatabase_core_LimboDB_openUtf8<'local>(
mut env: JNIEnv<'local>,
obj: JObject<'local>,
file_name_byte_arr: JByteArray<'local>,

16
bindings/java/src/main/java/org/github/tursodatabase/core/AbstractDB.java
View File

@@ -1,9 +1,5 @@
package org.github.tursodatabase.core;
import org.github.tursodatabase.LimboErrorCode;
import org.github.tursodatabase.NativeInvocation;
import org.github.tursodatabase.exceptions.LimboException;
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -19,7 +15,7 @@ public abstract class AbstractDB {
private final String fileName;
private final AtomicBoolean closed = new AtomicBoolean(true);
public AbstractDB(String url, String filaName) throws SQLException {
public AbstractDB(String url, String filaName) {
this.url = url;
this.fileName = filaName;
}
@@ -52,10 +48,10 @@ public abstract class AbstractDB {
* @throws SQLException if a database access error occurs.
*/
public final synchronized void open(int openFlags) throws SQLException {
_open(fileName, openFlags);
open0(fileName, openFlags);
}
protected abstract void _open(String fileName, int openFlags) throws SQLException;
protected abstract void open0(String fileName, int openFlags) throws SQLException;
/**
* Closes a database connection and finalizes any remaining statements before the closing
@@ -100,14 +96,14 @@ public abstract class AbstractDB {
* @return pointer to database instance
* @throws SQLException if a database access error occurs.
*/
protected abstract long _open_utf8(byte[] fileName, int openFlags) throws SQLException;
protected abstract long openUtf8(byte[] fileName, int openFlags) throws SQLException;
/**
* Closes the SQLite interface to a database.
*
* @throws SQLException if a database access error occurs.
*/
protected abstract void _close() throws SQLException;
protected abstract void close0() throws SQLException;
/**
* Compiles, evaluates, executes and commits an SQL statement.
@@ -116,7 +112,7 @@ public abstract class AbstractDB {
* @return Result code.
* @throws SQLException if a database access error occurs.
*/
public abstract int _exec(String sql) throws SQLException;
public abstract int exec(String sql) throws SQLException;
/**
* Compiles an SQL statement.

17
bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java
View File

@@ -30,8 +30,6 @@ public final class LimboDB extends AbstractDB {
}
}
// url example: "jdbc:sqlite:{fileName}
/**
* @param url e.g. "jdbc:sqlite:fileName
* @param fileName e.g. path to file
@@ -41,7 +39,7 @@ public final class LimboDB extends AbstractDB {
}
// TODO: receive config as argument
private LimboDB(String url, String fileName) throws SQLException {
private LimboDB(String url, String fileName) {
super(url, fileName);
}
@@ -53,7 +51,6 @@ public final class LimboDB extends AbstractDB {
try {
System.loadLibrary("_limbo_java");
} finally {
isLoaded = true;
}
@@ -63,31 +60,31 @@ public final class LimboDB extends AbstractDB {
// TODO: add support for JNI
@Override
protected synchronized native long _open_utf8(byte[] file, int openFlags) throws SQLException;
protected synchronized native long openUtf8(byte[] file, int openFlags) throws SQLException;
// TODO: add support for JNI
@Override
protected synchronized native void _close() throws SQLException;
protected synchronized native void close0() throws SQLException;
@Override
public synchronized int _exec(String sql) throws SQLException {
public synchronized int exec(String sql) throws SQLException {
// TODO: add implementation
throw new SQLFeatureNotSupportedException();
}
// TODO: add support for JNI
synchronized native int _exec_utf8(byte[] sqlUtf8) throws SQLException;
synchronized native int execUtf8(byte[] sqlUtf8) throws SQLException;
// TODO: add support for JNI
@Override
public native void interrupt();
@Override
protected void _open(String fileName, int openFlags) throws SQLException {
protected void open0(String fileName, int openFlags) throws SQLException {
if (isOpen) {
throwLimboException(LimboErrorCode.UNKNOWN_ERROR.code, "Already opened");
}
dbPtr = _open_utf8(stringToUtf8ByteArray(fileName), openFlags);
dbPtr = openUtf8(stringToUtf8ByteArray(fileName), openFlags);
isOpen = true;
}

3
cli/Cargo.toml
View File

@@ -29,3 +29,6 @@ rustyline = "12.0.0"
ctrlc = "3.4.4"
csv = "1.3.1"
miette = { version = "7.4.0", features = ["fancy"] }
[features]
io_uring = ["limbo_core/io_uring"]

108
cli/app.rs
View File

@@ -38,6 +38,52 @@ pub struct Opts {
pub quiet: bool,
#[clap(short, long, help = "Print commands before execution")]
pub echo: bool,
#[clap(
default_value_t,
value_enum,
short,
long,
help = "Select I/O backend. The only other choice to 'syscall' is\n\
\t'io-uring' when built for Linux with feature 'io_uring'\n"
)]
pub io: Io,
}
#[derive(Copy, Clone)]
pub enum DbLocation {
Memory,
Path,
}
#[derive(Copy, Clone, ValueEnum)]
pub enum Io {
Syscall,
#[cfg(all(target_os = "linux", feature = "io_uring"))]
IoUring,
}
impl Default for Io {
/// Custom Default impl with cfg! macro, to provide compile-time default to Clap based on platform
/// The cfg! could be elided, but Clippy complains
/// The default value can still be overridden with the Clap argument
fn default() -> Self {
match cfg!(all(target_os = "linux", feature = "io_uring")) {
true => {
#[cfg(all(target_os = "linux", feature = "io_uring"))]
{
Io::IoUring
}
#[cfg(any(
not(target_os = "linux"),
all(target_os = "linux", not(feature = "io_uring"))
))]
{
Io::Syscall
}
}
false => Io::Syscall,
}
}
}
#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
@@ -160,6 +206,7 @@ pub struct Settings {
output_mode: OutputMode,
echo: bool,
is_stdout: bool,
io: Io,
}
impl From<&Opts> for Settings {
@@ -174,6 +221,7 @@ impl From<&Opts> for Settings {
.database
.as_ref()
.map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string()),
io: opts.io,
}
}
}
@@ -207,7 +255,12 @@ impl Limbo {
.as_ref()
.map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string());
let io = get_io(&db_file)?;
let io = {
match db_file.as_str() {
":memory:" => get_io(DbLocation::Memory, opts.io)?,
_path => get_io(DbLocation::Path, opts.io)?,
}
};
let db = Database::open_file(io.clone(), &db_file)?;
let conn = db.connect();
let interrupt_count = Arc::new(AtomicUsize::new(0));
@@ -293,24 +346,17 @@ impl Limbo {
fn open_db(&mut self, path: &str) -> anyhow::Result<()> {
self.conn.close()?;
match path {
":memory:" => {
let io: Arc<dyn limbo_core::IO> = Arc::new(limbo_core::MemoryIO::new()?);
self.io = Arc::clone(&io);
let db = Database::open_file(self.io.clone(), path)?;
self.conn = db.connect();
self.opts.db_file = ":memory:".to_string();
Ok(())
let io = {
match path {
":memory:" => get_io(DbLocation::Memory, self.opts.io)?,
_path => get_io(DbLocation::Path, self.opts.io)?,
}
path => {
let io: Arc<dyn limbo_core::IO> = Arc::new(limbo_core::PlatformIO::new()?);
self.io = Arc::clone(&io);
let db = Database::open_file(self.io.clone(), path)?;
self.conn = db.connect();
self.opts.db_file = path.to_string();
Ok(())
}
}
};
self.io = Arc::clone(&io);
let db = Database::open_file(self.io.clone(), path)?;
self.conn = db.connect();
self.opts.db_file = path.to_string();
Ok(())
}
fn set_output_file(&mut self, path: &str) -> Result<(), String> {
@@ -740,10 +786,28 @@ fn get_writer(output: &str) -> Box<dyn Write> {
}
}
fn get_io(db: &str) -> anyhow::Result<Arc<dyn limbo_core::IO>> {
Ok(match db {
":memory:" => Arc::new(limbo_core::MemoryIO::new()?),
_ => Arc::new(limbo_core::PlatformIO::new()?),
fn get_io(db_location: DbLocation, io_choice: Io) -> anyhow::Result<Arc<dyn limbo_core::IO>> {
Ok(match db_location {
DbLocation::Memory => Arc::new(limbo_core::MemoryIO::new()?),
DbLocation::Path => {
match io_choice {
Io::Syscall => {
// We are building for Linux/macOS and syscall backend has been selected
#[cfg(target_family = "unix")]
{
Arc::new(limbo_core::UnixIO::new()?)
}
// We are not building for Linux/macOS and syscall backend has been selected
#[cfg(not(target_family = "unix"))]
{
Arc::new(limbo_core::PlatformIO::new()?)
}
}
// We are building for Linux and io_uring backend has been selected
#[cfg(all(target_os = "linux", feature = "io_uring"))]
Io::IoUring => Arc::new(limbo_core::UringIO::new()?),
}
}
})
}

2
core/Cargo.toml
View File

@@ -50,7 +50,7 @@ regex-syntax = { version = "0.8.5", default-features = false, features = ["unico
chrono = "0.4.38"
julian_day_converter = "0.3.2"
jsonb = { version = "0.4.4", optional = true }
indexmap = { version="2.2.6", features = ["serde"] }
indexmap = { version = "2.2.6", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }
pest = { version = "2.0", optional = true }
pest_derive = { version = "2.0", optional = true }

4
core/function.rs
View File

@@ -30,6 +30,7 @@ pub enum JsonFunc {
JsonArrowShiftExtract,
JsonExtract,
JsonType,
JsonErrorPosition,
}
#[cfg(feature = "json")]
@@ -46,6 +47,7 @@ impl Display for JsonFunc {
Self::JsonArrowExtract => "->".to_string(),
Self::JsonArrowShiftExtract => "->>".to_string(),
Self::JsonType => "json_type".to_string(),
Self::JsonErrorPosition => "json_error_position".to_string(),
}
)
}
@@ -379,6 +381,8 @@ impl Func {
"json_extract" => Ok(Func::Json(JsonFunc::JsonExtract)),
#[cfg(feature = "json")]
"json_type" => Ok(Func::Json(JsonFunc::JsonType)),
#[cfg(feature = "json")]
"json_error_position" => Ok(Self::Json(JsonFunc::JsonErrorPosition)),
"unixepoch" => Ok(Self::Scalar(ScalarFunc::UnixEpoch)),
"julianday" => Ok(Self::Scalar(ScalarFunc::JulianDay)),
"hex" => Ok(Self::Scalar(ScalarFunc::Hex)),

3
core/io/generic.rs
View File

@@ -1,5 +1,5 @@
use crate::{Completion, File, LimboError, OpenFlags, Result, IO};
use log::trace;
use log::{debug, trace};
use std::cell::RefCell;
use std::io::{Read, Seek, Write};
use std::rc::Rc;
@@ -8,6 +8,7 @@ pub struct GenericIO {}
impl GenericIO {
pub fn new() -> Result<Self> {
debug!("Using IO backend 'generic'");
Ok(Self {})
}
}

1
core/io/io_uring.rs
View File

@@ -70,6 +70,7 @@ impl UringIO {
}; MAX_IOVECS],
next_iovec: 0,
};
debug!("Using IO backend 'io-uring'");
Ok(Self {
inner: Rc::new(RefCell::new(inner)),
})

2
core/io/memory.rs
View File

@@ -1,6 +1,7 @@
use super::{Buffer, Completion, File, OpenFlags, IO};
use crate::Result;
use log::debug;
use std::{
cell::{RefCell, RefMut},
collections::BTreeMap,
@@ -20,6 +21,7 @@ type MemPage = Box<[u8; PAGE_SIZE]>;
impl MemoryIO {
#[allow(clippy::arc_with_non_send_sync)]
pub fn new() -> Result<Arc<Self>> {
debug!("Using IO backend 'memory'");
Ok(Arc::new(Self {
pages: RefCell::new(BTreeMap::new()),
size: RefCell::new(0),

4
core/io/mod.rs
View File

@@ -166,11 +166,15 @@ impl Buffer {
cfg_block! {
#[cfg(all(target_os = "linux", feature = "io_uring"))] {
mod io_uring;
pub use io_uring::UringIO;
mod unix;
pub use unix::UnixIO;
pub use io_uring::UringIO as PlatformIO;
}
#[cfg(any(all(target_os = "linux",not(feature = "io_uring")), target_os = "macos"))] {
mod unix;
pub use unix::UnixIO;
pub use unix::UnixIO as PlatformIO;
}

3
core/io/unix.rs
View File

@@ -4,7 +4,7 @@ use crate::Result;
use super::{Completion, File, OpenFlags, IO};
use libc::{c_short, fcntl, flock, F_SETLK};
use log::trace;
use log::{debug, trace};
use polling::{Event, Events, Poller};
use rustix::fd::{AsFd, AsRawFd};
use rustix::fs::OpenOptionsExt;
@@ -22,6 +22,7 @@ pub struct UnixIO {
impl UnixIO {
pub fn new() -> Result<Self> {
debug!("Using IO backend 'syscall'");
Ok(Self {
poller: Rc::new(RefCell::new(Poller::new()?)),
events: Rc::new(RefCell::new(Events::new())),

3
core/io/windows.rs
View File

@@ -1,5 +1,5 @@
use crate::{Completion, File, LimboError, OpenFlags, Result, IO};
use log::trace;
use log::{debug, trace};
use std::cell::RefCell;
use std::io::{Read, Seek, Write};
use std::rc::Rc;
@@ -8,6 +8,7 @@ pub struct WindowsIO {}
impl WindowsIO {
pub fn new() -> Result<Self> {
debug!("Using IO backend 'syscall'");
Ok(Self {})
}
}

84
core/json/mod.rs
View File

@@ -6,10 +6,12 @@ mod ser;
use std::rc::Rc;
pub use crate::json::de::from_str;
use crate::json::error::Error as JsonError;
use crate::json::json_path::{json_path, JsonPath, PathElement};
pub use crate::json::ser::to_string;
use crate::types::{LimboText, OwnedValue, TextSubtype};
use indexmap::IndexMap;
use jsonb::Error as JsonbError;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
@@ -370,6 +372,32 @@ fn json_extract_single<'a>(
Ok(Some(&current_element))
}
pub fn json_error_position(json: &OwnedValue) -> crate::Result<OwnedValue> {
match json {
OwnedValue::Text(t) => match crate::json::from_str::<Val>(&t.value) {
Ok(_) => Ok(OwnedValue::Integer(0)),
Err(JsonError::Message { location, .. }) => {
if let Some(loc) = location {
Ok(OwnedValue::Integer(loc.column as i64))
} else {
Err(crate::error::LimboError::InternalError(
"failed to determine json error position".into(),
))
}
}
},
OwnedValue::Blob(b) => match jsonb::from_slice(b) {
Ok(_) => Ok(OwnedValue::Integer(0)),
Err(JsonbError::Syntax(_, pos)) => Ok(OwnedValue::Integer(pos as i64)),
_ => Err(crate::error::LimboError::InternalError(
"failed to determine json error position".into(),
)),
},
OwnedValue::Null => Ok(OwnedValue::Null),
_ => Ok(OwnedValue::Integer(0)),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -698,4 +726,60 @@ mod tests {
Err(e) => assert!(e.to_string().contains("JSON path error")),
}
}
#[test]
fn test_json_error_position_no_error() {
let input = OwnedValue::build_text(Rc::new("[1,2,3]".to_string()));
let result = json_error_position(&input).unwrap();
assert_eq!(result, OwnedValue::Integer(0));
}
#[test]
fn test_json_error_position_no_error_more() {
let input = OwnedValue::build_text(Rc::new(r#"{"a":55,"b":72 , }"#.to_string()));
let result = json_error_position(&input).unwrap();
assert_eq!(result, OwnedValue::Integer(0));
}
#[test]
fn test_json_error_position_object() {
let input = OwnedValue::build_text(Rc::new(r#"{"a":55,"b":72,,}"#.to_string()));
let result = json_error_position(&input).unwrap();
assert_eq!(result, OwnedValue::Integer(16));
}
#[test]
fn test_json_error_position_array() {
let input = OwnedValue::build_text(Rc::new(r#"["a",55,"b",72,,]"#.to_string()));
let result = json_error_position(&input).unwrap();
assert_eq!(result, OwnedValue::Integer(16));
}
#[test]
fn test_json_error_position_null() {
let input = OwnedValue::Null;
let result = json_error_position(&input).unwrap();
assert_eq!(result, OwnedValue::Null);
}
#[test]
fn test_json_error_position_integer() {
let input = OwnedValue::Integer(5);
let result = json_error_position(&input).unwrap();
assert_eq!(result, OwnedValue::Integer(0));
}
#[test]
fn test_json_error_position_float() {
let input = OwnedValue::Float(-5.5);
let result = json_error_position(&input).unwrap();
assert_eq!(result, OwnedValue::Integer(0));
}
#[test]
fn test_json_error_position_blob() {
let input = OwnedValue::Blob(Rc::new(r#"["a",55,"b",72,,]"#.as_bytes().to_owned()));
let result = json_error_position(&input).unwrap();
assert_eq!(result, OwnedValue::Integer(16));
}
}

5
core/lib.rs
View File

@@ -44,8 +44,11 @@ pub type Result<T> = std::result::Result<T, error::LimboError>;
use crate::translate::optimizer::optimize_plan;
pub use io::OpenFlags;
#[cfg(feature = "fs")]
pub use io::PlatformIO;
#[cfg(all(feature = "fs", target_family = "unix"))]
pub use io::UnixIO;
#[cfg(all(feature = "fs", target_os = "linux", feature = "io_uring"))]
pub use io::UringIO;
pub use io::{Buffer, Completion, File, MemoryIO, WriteCompletion, IO};
pub use storage::buffer_pool::BufferPool;
pub use storage::database::DatabaseStorage;

113
core/pseudo.rs
View File

@@ -1,110 +1,19 @@
use crate::{
types::{SeekKey, SeekOp},
Result,
};
use std::cell::{Ref, RefCell};
use crate::types::{Cursor, CursorResult, OwnedRecord, OwnedValue};
use crate::types::OwnedRecord;
pub struct PseudoCursor {
current: RefCell<Option<OwnedRecord>>,
current: Option<OwnedRecord>,
}
impl PseudoCursor {
pub fn new() -> Self {
Self {
current: RefCell::new(None),
}
}
}
impl Cursor for PseudoCursor {
fn is_empty(&self) -> bool {
self.current.borrow().is_none()
}
fn root_page(&self) -> usize {
unreachable!()
}
fn rewind(&mut self) -> Result<CursorResult<()>> {
*self.current.borrow_mut() = None;
Ok(CursorResult::Ok(()))
}
fn next(&mut self) -> Result<CursorResult<()>> {
*self.current.borrow_mut() = None;
Ok(CursorResult::Ok(()))
}
fn wait_for_completion(&mut self) -> Result<()> {
Ok(())
}
fn rowid(&self) -> Result<Option<u64>> {
let x = self
.current
.borrow()
.as_ref()
.map(|record| match record.values[0] {
OwnedValue::Integer(rowid) => rowid as u64,
ref ov => {
panic!("Expected integer value, got {:?}", ov);
}
});
Ok(x)
}
fn seek(&mut self, _: SeekKey<'_>, _: SeekOp) -> Result<CursorResult<bool>> {
unimplemented!();
}
fn seek_to_last(&mut self) -> Result<CursorResult<()>> {
unimplemented!();
}
fn record(&self) -> Result<Ref<Option<OwnedRecord>>> {
Ok(self.current.borrow())
}
fn insert(
&mut self,
key: &OwnedValue,
record: &OwnedRecord,
moved_before: bool,
) -> Result<CursorResult<()>> {
let _ = key;
let _ = moved_before;
*self.current.borrow_mut() = Some(record.clone());
Ok(CursorResult::Ok(()))
}
fn delete(&mut self) -> Result<CursorResult<()>> {
unimplemented!()
}
fn get_null_flag(&self) -> bool {
false
}
fn set_null_flag(&mut self, _null_flag: bool) {
// Do nothing
}
fn exists(&mut self, key: &OwnedValue) -> Result<CursorResult<bool>> {
let _ = key;
todo!()
}
fn btree_create(&mut self, _flags: usize) -> u32 {
unreachable!("Please don't.")
}
fn last(&mut self) -> Result<CursorResult<()>> {
todo!()
}
fn prev(&mut self) -> Result<CursorResult<()>> {
todo!()
Self { current: None }
}
pub fn record(&self) -> Option<&OwnedRecord> {
self.current.as_ref()
}
pub fn insert(&mut self, record: OwnedRecord) {
self.current = Some(record);
}
}

57
core/schema.rs
View File

@@ -46,35 +46,13 @@ impl Schema {
#[derive(Clone, Debug)]
pub enum Table {
BTree(Rc<BTreeTable>),
Index(Rc<Index>),
Pseudo(Rc<PseudoTable>),
}
impl Table {
pub fn is_pseudo(&self) -> bool {
matches!(self, Table::Pseudo(_))
}
pub fn get_rowid_alias_column(&self) -> Option<(usize, &Column)> {
match self {
Self::BTree(table) => table.get_rowid_alias_column(),
Self::Index(_) => None,
Self::Pseudo(_) => None,
}
}
pub fn column_is_rowid_alias(&self, col: &Column) -> bool {
match self {
Table::BTree(table) => table.column_is_rowid_alias(col),
Self::Index(_) => false,
Self::Pseudo(_) => false,
}
}
pub fn get_root_page(&self) -> usize {
match self {
Table::BTree(table) => table.root_page,
Table::Index(_) => unimplemented!(),
Table::Pseudo(_) => unimplemented!(),
}
}
@@ -82,40 +60,13 @@ impl Table {
pub fn get_name(&self) -> &str {
match self {
Self::BTree(table) => &table.name,
Self::Index(index) => &index.name,
Self::Pseudo(_) => "",
}
}
pub fn column_index_to_name(&self, index: usize) -> Option<&str> {
match self {
Self::BTree(table) => match table.columns.get(index) {
Some(column) => Some(&column.name),
None => None,
},
Self::Index(i) => match i.columns.get(index) {
Some(column) => Some(&column.name),
None => None,
},
Self::Pseudo(table) => match table.columns.get(index) {
Some(_) => None,
None => None,
},
}
}
pub fn get_column(&self, name: &str) -> Option<(usize, &Column)> {
match self {
Self::BTree(table) => table.get_column(name),
Self::Index(_) => unimplemented!(),
Self::Pseudo(table) => table.get_column(name),
}
}
pub fn get_column_at(&self, index: usize) -> &Column {
match self {
Self::BTree(table) => table.columns.get(index).unwrap(),
Self::Index(_) => unimplemented!(),
Self::Pseudo(table) => table.columns.get(index).unwrap(),
}
}
@@ -123,16 +74,14 @@ impl Table {
pub fn columns(&self) -> &Vec<Column> {
match self {
Self::BTree(table) => &table.columns,
Self::Index(_) => unimplemented!(),
Self::Pseudo(table) => &table.columns,
}
}
pub fn has_rowid(&self) -> bool {
pub fn btree(&self) -> Option<Rc<BTreeTable>> {
match self {
Self::BTree(table) => table.has_rowid,
Self::Index(_) => unimplemented!(),
Self::Pseudo(_) => unimplemented!(),
Self::BTree(table) => Some(table.clone()),
Self::Pseudo(_) => None,
}
}
}

318
core/storage/btree.rs
View File

@@ -5,7 +5,7 @@ use crate::storage::sqlite3_ondisk::{
read_btree_cell, read_varint, write_varint, BTreeCell, DatabaseHeader, PageContent, PageType,
TableInteriorCell, TableLeafCell,
};
use crate::types::{Cursor, CursorResult, OwnedRecord, OwnedValue, SeekKey, SeekOp};
use crate::types::{CursorResult, OwnedRecord, OwnedValue, SeekKey, SeekOp};
use crate::Result;
use std::cell::{Ref, RefCell};
@@ -419,7 +419,7 @@ impl BTreeCursor {
/// This may be used to seek to a specific record in a point query (e.g. SELECT * FROM table WHERE col = 10)
/// or e.g. find the first record greater than the seek key in a range query (e.g. SELECT * FROM table WHERE col > 10).
/// We don't include the rowid in the comparison and that's why the last value from the record is not included.
fn seek(
fn do_seek(
&mut self,
key: SeekKey<'_>,
op: SeekOp,
@@ -1697,6 +1697,162 @@ impl BTreeCursor {
}
cell_idx
}
pub fn seek_to_last(&mut self) -> Result<CursorResult<()>> {
return_if_io!(self.move_to_rightmost());
let (rowid, record) = return_if_io!(self.get_next_record(None));
if rowid.is_none() {
let is_empty = return_if_io!(self.is_empty_table());
assert!(is_empty);
return Ok(CursorResult::Ok(()));
}
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(()))
}
pub fn is_empty(&self) -> bool {
self.record.borrow().is_none()
}
pub fn root_page(&self) -> usize {
self.root_page
}
pub fn rewind(&mut self) -> Result<CursorResult<()>> {
self.move_to_root();
let (rowid, record) = return_if_io!(self.get_next_record(None));
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(()))
}
pub fn last(&mut self) -> Result<CursorResult<()>> {
match self.move_to_rightmost()? {
CursorResult::Ok(_) => self.prev(),
CursorResult::IO => Ok(CursorResult::IO),
}
}
pub fn next(&mut self) -> Result<CursorResult<()>> {
let (rowid, record) = return_if_io!(self.get_next_record(None));
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(()))
}
pub fn prev(&mut self) -> Result<CursorResult<()>> {
match self.get_prev_record()? {
CursorResult::Ok((rowid, record)) => {
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(()))
}
CursorResult::IO => Ok(CursorResult::IO),
}
}
pub fn wait_for_completion(&mut self) -> Result<()> {
// TODO: Wait for pager I/O to complete
Ok(())
}
pub fn rowid(&self) -> Result<Option<u64>> {
Ok(*self.rowid.borrow())
}
pub fn seek(&mut self, key: SeekKey<'_>, op: SeekOp) -> Result<CursorResult<bool>> {
let (rowid, record) = return_if_io!(self.do_seek(key, op));
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(rowid.is_some()))
}
pub fn record(&self) -> Result<Ref<Option<OwnedRecord>>> {
Ok(self.record.borrow())
}
pub fn insert(
&mut self,
key: &OwnedValue,
_record: &OwnedRecord,
moved_before: bool, /* Indicate whether it's necessary to traverse to find the leaf page */
) -> Result<CursorResult<()>> {
let int_key = match key {
OwnedValue::Integer(i) => i,
_ => unreachable!("btree tables are indexed by integers!"),
};
if !moved_before {
return_if_io!(self.move_to(SeekKey::TableRowId(*int_key as u64), SeekOp::EQ));
}
return_if_io!(self.insert_into_page(key, _record));
self.rowid.replace(Some(*int_key as u64));
Ok(CursorResult::Ok(()))
}
pub fn delete(&mut self) -> Result<CursorResult<()>> {
println!("rowid: {:?}", self.rowid.borrow());
Ok(CursorResult::Ok(()))
}
pub fn set_null_flag(&mut self, flag: bool) {
self.null_flag = flag;
}
pub fn get_null_flag(&self) -> bool {
self.null_flag
}
pub fn exists(&mut self, key: &OwnedValue) -> Result<CursorResult<bool>> {
let int_key = match key {
OwnedValue::Integer(i) => i,
_ => unreachable!("btree tables are indexed by integers!"),
};
return_if_io!(self.move_to(SeekKey::TableRowId(*int_key as u64), SeekOp::EQ));
let page = self.stack.top();
// TODO(pere): request load
return_if_locked!(page);
let contents = page.get().contents.as_ref().unwrap();
// find cell
let int_key = match key {
OwnedValue::Integer(i) => *i as u64,
_ => unreachable!("btree tables are indexed by integers!"),
};
let cell_idx = self.find_cell(contents, int_key);
if cell_idx >= contents.cell_count() {
Ok(CursorResult::Ok(false))
} else {
let equals = match &contents.cell_get(
cell_idx,
self.pager.clone(),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)? {
BTreeCell::TableLeafCell(l) => l._rowid == int_key,
_ => unreachable!(),
};
Ok(CursorResult::Ok(equals))
}
}
pub fn btree_create(&mut self, flags: usize) -> u32 {
let page_type = match flags {
1 => PageType::TableLeaf,
2 => PageType::IndexLeaf,
_ => unreachable!(
"wrong create table falgs, should be 1 for table and 2 for index, got {}",
flags,
),
};
let page = self.allocate_page(page_type, 0);
let id = page.get().id;
id as u32
}
}
impl PageStack {
@@ -1822,164 +1978,6 @@ fn find_free_cell(page_ref: &PageContent, db_header: Ref<DatabaseHeader>, amount
}
}
impl Cursor for BTreeCursor {
fn seek_to_last(&mut self) -> Result<CursorResult<()>> {
return_if_io!(self.move_to_rightmost());
let (rowid, record) = return_if_io!(self.get_next_record(None));
if rowid.is_none() {
let is_empty = return_if_io!(self.is_empty_table());
assert!(is_empty);
return Ok(CursorResult::Ok(()));
}
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(()))
}
fn is_empty(&self) -> bool {
self.record.borrow().is_none()
}
fn root_page(&self) -> usize {
self.root_page
}
fn rewind(&mut self) -> Result<CursorResult<()>> {
self.move_to_root();
let (rowid, record) = return_if_io!(self.get_next_record(None));
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(()))
}
fn last(&mut self) -> Result<CursorResult<()>> {
match self.move_to_rightmost()? {
CursorResult::Ok(_) => self.prev(),
CursorResult::IO => Ok(CursorResult::IO),
}
}
fn next(&mut self) -> Result<CursorResult<()>> {
let (rowid, record) = return_if_io!(self.get_next_record(None));
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(()))
}
fn prev(&mut self) -> Result<CursorResult<()>> {
match self.get_prev_record()? {
CursorResult::Ok((rowid, record)) => {
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(()))
}
CursorResult::IO => Ok(CursorResult::IO),
}
}
fn wait_for_completion(&mut self) -> Result<()> {
// TODO: Wait for pager I/O to complete
Ok(())
}
fn rowid(&self) -> Result<Option<u64>> {
Ok(*self.rowid.borrow())
}
fn seek(&mut self, key: SeekKey<'_>, op: SeekOp) -> Result<CursorResult<bool>> {
let (rowid, record) = return_if_io!(self.seek(key, op));
self.rowid.replace(rowid);
self.record.replace(record);
Ok(CursorResult::Ok(rowid.is_some()))
}
fn record(&self) -> Result<Ref<Option<OwnedRecord>>> {
Ok(self.record.borrow())
}
fn insert(
&mut self,
key: &OwnedValue,
_record: &OwnedRecord,
moved_before: bool, /* Indicate whether it's necessary to traverse to find the leaf page */
) -> Result<CursorResult<()>> {
let int_key = match key {
OwnedValue::Integer(i) => i,
_ => unreachable!("btree tables are indexed by integers!"),
};
if !moved_before {
return_if_io!(self.move_to(SeekKey::TableRowId(*int_key as u64), SeekOp::EQ));
}
return_if_io!(self.insert_into_page(key, _record));
self.rowid.replace(Some(*int_key as u64));
Ok(CursorResult::Ok(()))
}
fn delete(&mut self) -> Result<CursorResult<()>> {
println!("rowid: {:?}", self.rowid.borrow());
Ok(CursorResult::Ok(()))
}
fn set_null_flag(&mut self, flag: bool) {
self.null_flag = flag;
}
fn get_null_flag(&self) -> bool {
self.null_flag
}
fn exists(&mut self, key: &OwnedValue) -> Result<CursorResult<bool>> {
let int_key = match key {
OwnedValue::Integer(i) => i,
_ => unreachable!("btree tables are indexed by integers!"),
};
return_if_io!(self.move_to(SeekKey::TableRowId(*int_key as u64), SeekOp::EQ));
let page = self.stack.top();
// TODO(pere): request load
return_if_locked!(page);
let contents = page.get().contents.as_ref().unwrap();
// find cell
let int_key = match key {
OwnedValue::Integer(i) => *i as u64,
_ => unreachable!("btree tables are indexed by integers!"),
};
let cell_idx = self.find_cell(contents, int_key);
if cell_idx >= contents.cell_count() {
Ok(CursorResult::Ok(false))
} else {
let equals = match &contents.cell_get(
cell_idx,
self.pager.clone(),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)? {
BTreeCell::TableLeafCell(l) => l._rowid == int_key,
_ => unreachable!(),
};
Ok(CursorResult::Ok(equals))
}
}
fn btree_create(&mut self, flags: usize) -> u32 {
let page_type = match flags {
1 => PageType::TableLeaf,
2 => PageType::IndexLeaf,
_ => unreachable!(
"wrong create table falgs, should be 1 for table and 2 for index, got {}",
flags,
),
};
let page = self.allocate_page(page_type, 0);
let id = page.get().id;
id as u32
}
}
pub fn btree_init_page(
page: &PageRef,
page_type: PageType,

16
core/storage/sqlite3_ondisk.rs
View File

@@ -912,7 +912,7 @@ fn read_payload(unread: &[u8], payload_size: usize, pager: Rc<Pager>) -> (Vec<u8
#[derive(Debug, PartialEq)]
pub enum SerialType {
Null,
UInt8,
Int8, // 8-bit twos-complement integer: https://www.sqlite.org/fileformat.html
BEInt16,
BEInt24,
BEInt32,
@@ -931,7 +931,7 @@ impl TryFrom<u64> for SerialType {
fn try_from(value: u64) -> Result<Self> {
match value {
0 => Ok(Self::Null),
1 => Ok(Self::UInt8),
1 => Ok(Self::Int8),
2 => Ok(Self::BEInt16),
3 => Ok(Self::BEInt24),
4 => Ok(Self::BEInt32),
@@ -974,11 +974,12 @@ pub fn read_record(payload: &[u8]) -> Result<OwnedRecord> {
pub fn read_value(buf: &[u8], serial_type: &SerialType) -> Result<(OwnedValue, usize)> {
match *serial_type {
SerialType::Null => Ok((OwnedValue::Null, 0)),
SerialType::UInt8 => {
SerialType::Int8 => {
if buf.is_empty() {
crate::bail_corrupt_error!("Invalid UInt8 value");
}
Ok((OwnedValue::Integer(buf[0] as i64), 1))
let val = buf[0] as i8;
Ok((OwnedValue::Integer(val as i64), 1))
}
SerialType::BEInt16 => {
if buf.len() < 2 {
@@ -1377,7 +1378,7 @@ mod tests {
#[rstest]
#[case(0, SerialType::Null)]
#[case(1, SerialType::UInt8)]
#[case(1, SerialType::Int8)]
#[case(2, SerialType::BEInt16)]
#[case(3, SerialType::BEInt24)]
#[case(4, SerialType::BEInt32)]
@@ -1403,8 +1404,9 @@ mod tests {
#[rstest]
#[case(&[], SerialType::Null, OwnedValue::Null)]
#[case(&[255], SerialType::UInt8, OwnedValue::Integer(255))]
#[case(&[255], SerialType::Int8, OwnedValue::Integer(-1))]
#[case(&[0x12, 0x34], SerialType::BEInt16, OwnedValue::Integer(0x1234))]
#[case(&[0xFE], SerialType::Int8, OwnedValue::Integer(-2))]
#[case(&[0x12, 0x34, 0x56], SerialType::BEInt24, OwnedValue::Integer(0x123456))]
#[case(&[0x12, 0x34, 0x56, 0x78], SerialType::BEInt32, OwnedValue::Integer(0x12345678))]
#[case(&[0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC], SerialType::BEInt48, OwnedValue::Integer(0x123456789ABC))]
@@ -1424,7 +1426,7 @@ mod tests {
}
#[rstest]
#[case(&[], SerialType::UInt8)]
#[case(&[], SerialType::Int8)]
#[case(&[0x12], SerialType::BEInt16)]
#[case(&[0x12, 0x34], SerialType::BEInt24)]
#[case(&[0x12, 0x34, 0x56], SerialType::BEInt32)]

25
core/translate/expr.rs
View File

@@ -812,6 +812,31 @@ pub fn translate_expr(
func_ctx,
)
}
JsonFunc::JsonErrorPosition => {
let args = if let Some(args) = args {
if args.len() != 1 {
crate::bail_parse_error!(
"{} function with not exactly 1 argument",
j.to_string()
);
}
args
} else {
crate::bail_parse_error!(
"{} function with no arguments",
j.to_string()
);
};
let json_reg = program.alloc_register();
translate_expr(program, referenced_tables, &args[0], json_reg, resolver)?;
program.emit_insn(Insn::Function {
constant_mask: 0,
start_reg: json_reg,
dest: target_register,
func: func_ctx,
});
Ok(target_register)
}
},
Func::Scalar(srf) => {
match srf {

12
core/translate/group_by.rs
View File

@@ -4,9 +4,13 @@ use sqlite3_parser::ast;
use crate::{
function::AggFunc,
schema::{Column, PseudoTable, Table},
schema::{Column, PseudoTable},
types::{OwnedRecord, OwnedValue},
vdbe::{builder::ProgramBuilder, insn::Insn, BranchOffset},
vdbe::{
builder::{CursorType, ProgramBuilder},
insn::Insn,
BranchOffset,
},
Result,
};
@@ -50,7 +54,7 @@ pub fn init_group_by(
) -> Result<()> {
let num_aggs = aggregates.len();
let sort_cursor = program.alloc_cursor_id(None, None);
let sort_cursor = program.alloc_cursor_id(None, CursorType::Sorter);
let reg_abort_flag = program.alloc_register();
let reg_group_exprs_cmp = program.alloc_registers(group_by.exprs.len());
@@ -175,7 +179,7 @@ pub fn emit_group_by<'a>(
columns: pseudo_columns,
});
let pseudo_cursor = program.alloc_cursor_id(None, Some(Table::Pseudo(pseudo_table.clone())));
let pseudo_cursor = program.alloc_cursor_id(None, CursorType::Pseudo(pseudo_table.clone()));
program.emit_insn(Insn::OpenPseudo {
cursor_id: pseudo_cursor,

42
core/translate/insert.rs
View File

@@ -6,13 +6,18 @@ use sqlite3_parser::ast::{
};
use crate::error::SQLITE_CONSTRAINT_PRIMARYKEY;
use crate::schema::BTreeTable;
use crate::util::normalize_ident;
use crate::vdbe::BranchOffset;
use crate::{
schema::{Column, Schema, Table},
schema::{Column, Schema},
storage::sqlite3_ondisk::DatabaseHeader,
translate::expr::translate_expr,
vdbe::{builder::ProgramBuilder, insn::Insn, Program},
vdbe::{
builder::{CursorType, ProgramBuilder},
insn::Insn,
Program,
},
SymbolTable,
};
use crate::{Connection, Result};
@@ -53,20 +58,15 @@ pub fn translate_insert(
Some(table) => table,
None => crate::bail_corrupt_error!("Parse error: no such table: {}", table_name),
};
let table = Rc::new(Table::BTree(table));
if !table.has_rowid() {
if !table.has_rowid {
crate::bail_parse_error!("INSERT into WITHOUT ROWID table is not supported");
}
let cursor_id = program.alloc_cursor_id(
Some(table_name.0.clone()),
Some(table.clone().deref().clone()),
CursorType::BTreeTable(table.clone()),
);
let root_page = match table.as_ref() {
Table::BTree(btree) => btree.root_page,
Table::Index(index) => index.root_page,
Table::Pseudo(_) => todo!(),
};
let root_page = table.root_page;
let values = match body {
InsertBody::Select(select, None) => match &select.body.select.deref() {
sqlite3_parser::ast::OneSelect::Values(values) => values,
@@ -77,9 +77,9 @@ pub fn translate_insert(
let column_mappings = resolve_columns_for_insert(&table, columns, values)?;
// Check if rowid was provided (through INTEGER PRIMARY KEY as a rowid alias)
let rowid_alias_index = table.columns().iter().position(|c| c.is_rowid_alias);
let rowid_alias_index = table.columns.iter().position(|c| c.is_rowid_alias);
let has_user_provided_rowid = {
assert!(column_mappings.len() == table.columns().len());
assert!(column_mappings.len() == table.columns.len());
if let Some(index) = rowid_alias_index {
column_mappings[index].value_index.is_some()
} else {
@@ -89,7 +89,7 @@ pub fn translate_insert(
// allocate a register for each column in the table. if not provided by user, they will simply be set as null.
// allocate an extra register for rowid regardless of whether user provided a rowid alias column.
let num_cols = table.columns().len();
let num_cols = table.columns.len();
let rowid_reg = program.alloc_registers(num_cols + 1);
let column_registers_start = rowid_reg + 1;
let rowid_alias_reg = {
@@ -215,14 +215,14 @@ pub fn translate_insert(
target_pc: make_record_label,
});
let rowid_column_name = if let Some(index) = rowid_alias_index {
table.column_index_to_name(index).unwrap()
&table.columns.get(index).unwrap().name
} else {
"rowid"
};
program.emit_insn(Insn::Halt {
err_code: SQLITE_CONSTRAINT_PRIMARYKEY,
description: format!("{}.{}", table.get_name(), rowid_column_name),
description: format!("{}.{}", table_name.0, rowid_column_name),
});
program.resolve_label(make_record_label, program.offset());
@@ -293,7 +293,7 @@ struct ColumnMapping<'a> {
/// - Named columns map to their corresponding value index
/// - Unspecified columns map to None
fn resolve_columns_for_insert<'a>(
table: &'a Table,
table: &'a BTreeTable,
columns: &Option<DistinctNames>,
values: &[Vec<Expr>],
) -> Result<Vec<ColumnMapping<'a>>> {
@@ -301,7 +301,7 @@ fn resolve_columns_for_insert<'a>(
crate::bail_parse_error!("no values to insert");
}
let table_columns = table.columns();
let table_columns = &table.columns;
// Case 1: No columns specified - map values to columns in order
if columns.is_none() {
@@ -309,7 +309,7 @@ fn resolve_columns_for_insert<'a>(
if num_values > table_columns.len() {
crate::bail_parse_error!(
"table {} has {} columns but {} values were supplied",
table.get_name(),
&table.name,
table_columns.len(),
num_values
);
@@ -350,11 +350,7 @@ fn resolve_columns_for_insert<'a>(
.position(|c| c.name.eq_ignore_ascii_case(&column_name));
if table_index.is_none() {
crate::bail_parse_error!(
"table {} has no column named {}",
table.get_name(),
column_name
);
crate::bail_parse_error!("table {} has no column named {}", &table.name, column_name);
}
mappings[table_index.unwrap()].value_index = Some(value_index);

17
core/translate/main_loop.rs
View File

@@ -1,9 +1,12 @@
use sqlite3_parser::ast;
use crate::{
schema::Table,
translate::result_row::emit_select_result,
vdbe::{builder::ProgramBuilder, insn::Insn, BranchOffset},
vdbe::{
builder::{CursorType, ProgramBuilder},
insn::Insn,
BranchOffset,
},
Result,
};
@@ -81,7 +84,7 @@ pub fn init_loop(
} => {
let cursor_id = program.alloc_cursor_id(
Some(table_reference.table_identifier.clone()),
Some(table_reference.table.clone()),
CursorType::BTreeTable(table_reference.btree().unwrap().clone()),
);
let root_page = table_reference.table.get_root_page();
@@ -114,7 +117,7 @@ pub fn init_loop(
} => {
let table_cursor_id = program.alloc_cursor_id(
Some(table_reference.table_identifier.clone()),
Some(table_reference.table.clone()),
CursorType::BTreeTable(table_reference.btree().unwrap().clone()),
);
match mode {
@@ -138,8 +141,10 @@ pub fn init_loop(
}
if let Search::IndexSearch { index, .. } = search {
let index_cursor_id = program
.alloc_cursor_id(Some(index.name.clone()), Some(Table::Index(index.clone())));
let index_cursor_id = program.alloc_cursor_id(
Some(index.name.clone()),
CursorType::BTreeIndex(index.clone()),
);
match mode {
OperationMode::SELECT => {

8
core/translate/mod.rs
View File

@@ -27,6 +27,7 @@ use crate::storage::pager::Pager;
use crate::storage::sqlite3_ondisk::{DatabaseHeader, MIN_PAGE_CACHE_SIZE};
use crate::translate::delete::translate_delete;
use crate::util::PRIMARY_KEY_AUTOMATIC_INDEX_NAME_PREFIX;
use crate::vdbe::builder::CursorType;
use crate::vdbe::{builder::ProgramBuilder, insn::Insn, Program};
use crate::{bail_parse_error, Connection, LimboError, Result, SymbolTable};
use insert::translate_insert;
@@ -463,9 +464,10 @@ fn translate_create_table(
let table_id = "sqlite_schema".to_string();
let table = schema.get_table(&table_id).unwrap();
let table = crate::schema::Table::BTree(table.clone());
let sqlite_schema_cursor_id =
program.alloc_cursor_id(Some(table_id.to_owned()), Some(table.to_owned()));
let sqlite_schema_cursor_id = program.alloc_cursor_id(
Some(table_id.to_owned()),
CursorType::BTreeTable(table.clone()),
);
program.emit_insn(Insn::OpenWriteAsync {
cursor_id: sqlite_schema_cursor_id,
root_page: 1,

19
core/translate/order_by.rs
View File

@@ -3,10 +3,13 @@ use std::rc::Rc;
use sqlite3_parser::ast;
use crate::{
schema::{Column, PseudoTable, Table},
schema::{Column, PseudoTable},
types::{OwnedRecord, OwnedValue},
util::exprs_are_equivalent,
vdbe::{builder::ProgramBuilder, insn::Insn},
vdbe::{
builder::{CursorType, ProgramBuilder},
insn::Insn,
},
Result,
};
@@ -32,7 +35,7 @@ pub fn init_order_by(
t_ctx: &mut TranslateCtx,
order_by: &[(ast::Expr, Direction)],
) -> Result<()> {
let sort_cursor = program.alloc_cursor_id(None, None);
let sort_cursor = program.alloc_cursor_id(None, CursorType::Sorter);
t_ctx.meta_sort = Some(SortMetadata {
sort_cursor,
reg_sorter_data: program.alloc_register(),
@@ -93,12 +96,10 @@ pub fn emit_order_by(
.map(|v| v.len())
.unwrap_or(0);
let pseudo_cursor = program.alloc_cursor_id(
None,
Some(Table::Pseudo(Rc::new(PseudoTable {
columns: pseudo_columns,
}))),
);
let pseudo_table = Rc::new(PseudoTable {
columns: pseudo_columns,
});
let pseudo_cursor = program.alloc_cursor_id(None, CursorType::Pseudo(pseudo_table.clone()));
let SortMetadata {
sort_cursor,
reg_sorter_data,

8
core/translate/plan.rs
View File

@@ -7,7 +7,7 @@ use std::{
use crate::{
function::AggFunc,
schema::{Column, Index, Table},
schema::{BTreeTable, Column, Index, Table},
vdbe::BranchOffset,
Result,
};
@@ -255,6 +255,12 @@ pub struct TableReference {
}
impl TableReference {
pub fn btree(&self) -> Option<Rc<BTreeTable>> {
match self.reference_type {
TableReferenceType::BTreeTable => self.table.btree(),
TableReferenceType::Subquery { .. } => None,
}
}
pub fn new_subquery(identifier: String, table_index: usize, plan: &SelectPlan) -> Self {
Self {
table: Table::Pseudo(Rc::new(PseudoTable::new_with_columns(

27
core/types.rs
View File

@@ -1,5 +1,5 @@
use std::fmt::Display;
use std::{cell::Ref, rc::Rc};
use std::rc::Rc;
use crate::error::LimboError;
use crate::Result;
@@ -524,31 +524,6 @@ pub enum SeekKey<'a> {
IndexKey(&'a OwnedRecord),
}
pub trait Cursor {
fn is_empty(&self) -> bool;
fn root_page(&self) -> usize;
fn rewind(&mut self) -> Result<CursorResult<()>>;
fn last(&mut self) -> Result<CursorResult<()>>;
fn next(&mut self) -> Result<CursorResult<()>>;
fn prev(&mut self) -> Result<CursorResult<()>>;
fn wait_for_completion(&mut self) -> Result<()>;
fn rowid(&self) -> Result<Option<u64>>;
fn seek(&mut self, key: SeekKey, op: SeekOp) -> Result<CursorResult<bool>>;
fn seek_to_last(&mut self) -> Result<CursorResult<()>>;
fn record(&self) -> Result<Ref<Option<OwnedRecord>>>;
fn insert(
&mut self,
key: &OwnedValue,
record: &OwnedRecord,
moved_before: bool, /* Tells inserter that it doesn't need to traverse in order to find leaf page */
) -> Result<CursorResult<()>>; //
fn delete(&mut self) -> Result<CursorResult<()>>;
fn exists(&mut self, key: &OwnedValue) -> Result<CursorResult<bool>>;
fn set_null_flag(&mut self, flag: bool);
fn get_null_flag(&self) -> bool;
fn btree_create(&mut self, flags: usize) -> u32;
}
#[cfg(test)]
mod tests {
use super::*;

28
core/vdbe/builder.rs
View File

@@ -4,9 +4,13 @@ use std::{
rc::{Rc, Weak},
};
use crate::{storage::sqlite3_ondisk::DatabaseHeader, Connection};
use crate::{
schema::{BTreeTable, Index, PseudoTable},
storage::sqlite3_ondisk::DatabaseHeader,
Connection,
};
use super::{BranchOffset, CursorID, Insn, InsnReference, Program, Table};
use super::{BranchOffset, CursorID, Insn, InsnReference, Program};
#[allow(dead_code)]
pub struct ProgramBuilder {
@@ -18,7 +22,7 @@ pub struct ProgramBuilder {
constant_insns: Vec<Insn>,
next_insn_label: Option<BranchOffset>,
// Cursors that are referenced by the program. Indexed by CursorID.
pub cursor_ref: Vec<(Option<String>, Option<Table>)>,
pub cursor_ref: Vec<(Option<String>, CursorType)>,
// Hashmap of label to insn reference. Resolved in build().
label_to_resolved_offset: HashMap<i32, u32>,
// Bitmask of cursors that have emitted a SeekRowid instruction.
@@ -27,6 +31,20 @@ pub struct ProgramBuilder {
comments: HashMap<InsnReference, &'static str>,
}
#[derive(Debug, Clone)]
pub enum CursorType {
BTreeTable(Rc<BTreeTable>),
BTreeIndex(Rc<Index>),
Pseudo(Rc<PseudoTable>),
Sorter,
}
impl CursorType {
pub fn is_index(&self) -> bool {
matches!(self, CursorType::BTreeIndex(_))
}
}
impl ProgramBuilder {
pub fn new() -> Self {
Self {
@@ -58,11 +76,11 @@ impl ProgramBuilder {
pub fn alloc_cursor_id(
&mut self,
table_identifier: Option<String>,
table: Option<Table>,
cursor_type: CursorType,
) -> usize {
let cursor = self.next_free_cursor_id;
self.next_free_cursor_id += 1;
self.cursor_ref.push((table_identifier, table));
self.cursor_ref.push((table_identifier, cursor_type));
assert!(self.cursor_ref.len() == self.next_free_cursor_id);
cursor
}

4
core/vdbe/datetime.rs
View File

@@ -884,8 +884,6 @@ mod tests {
#[test]
fn test_valid_get_time_from_datetime_value() {
let now = chrono::Local::now().to_utc().format("%H:%M:%S").to_string();
let test_time_str = "22:30:45";
let prev_time_str = "20:30:45";
let next_time_str = "03:30:45";
@@ -1049,8 +1047,6 @@ mod tests {
OwnedValue::build_text(Rc::new("22:30:45.123Z".to_string())),
test_time_str,
),
// Test Format 11: 'now'
(OwnedValue::build_text(Rc::new("now".to_string())), &now),
// Format 12: DDDDDDDDDD (Julian date as float or integer)
(OwnedValue::Float(2460082.1), "14:24:00"),
(OwnedValue::Integer(2460082), "12:00:00"),

21
core/vdbe/explain.rs
View File

@@ -1,3 +1,5 @@
use crate::vdbe::builder::CursorType;
use super::{Insn, InsnReference, OwnedValue, Program};
use std::rc::Rc;
@@ -387,7 +389,19 @@ pub fn insn_to_str(
column,
dest,
} => {
let (table_identifier, table) = &program.cursor_ref[*cursor_id];
let (table_identifier, cursor_type) = &program.cursor_ref[*cursor_id];
let column_name = match cursor_type {
CursorType::BTreeTable(table) => {
Some(&table.columns.get(*column).unwrap().name)
}
CursorType::BTreeIndex(index) => {
Some(&index.columns.get(*column).unwrap().name)
}
CursorType::Pseudo(pseudo_table) => {
Some(&pseudo_table.columns.get(*column).unwrap().name)
}
CursorType::Sorter => None,
};
(
"Column",
*cursor_id as i32,
@@ -401,10 +415,7 @@ pub fn insn_to_str(
table_identifier
.as_ref()
.unwrap_or(&format!("cursor {}", cursor_id)),
table
.as_ref()
.and_then(|x| x.column_index_to_name(*column))
.unwrap_or(format!("column {}", *column).as_str())
column_name.unwrap_or(&format!("column {}", *column))
),
)
}

476
core/vdbe/mod.rs
View File

@@ -30,18 +30,17 @@ use crate::ext::{exec_ts_from_uuid7, exec_uuid, exec_uuidblob, exec_uuidstr, Ext
use crate::function::{AggFunc, FuncCtx, MathFunc, MathFuncArity, ScalarFunc};
use crate::pseudo::PseudoCursor;
use crate::result::LimboResult;
use crate::schema::Table;
use crate::storage::sqlite3_ondisk::DatabaseHeader;
use crate::storage::{btree::BTreeCursor, pager::Pager};
use crate::types::{
AggContext, Cursor, CursorResult, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp,
};
use crate::types::{AggContext, CursorResult, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp};
use crate::util::parse_schema_rows;
use crate::vdbe::builder::CursorType;
use crate::vdbe::insn::Insn;
#[cfg(feature = "json")]
use crate::{
function::JsonFunc, json::get_json, json::json_array, json::json_array_length,
json::json_arrow_extract, json::json_arrow_shift_extract, json::json_extract, json::json_type,
json::json_arrow_extract, json::json_arrow_shift_extract, json::json_error_position,
json::json_extract, json::json_type,
};
use crate::{Connection, Result, Rows, TransactionState, DATABASE_VERSION};
use datetime::{exec_date, exec_datetime_full, exec_julianday, exec_time, exec_unixepoch};
@@ -53,6 +52,7 @@ use likeop::{construct_like_escape_arg, exec_glob, exec_like_with_escape};
use rand::distributions::{Distribution, Uniform};
use rand::{thread_rng, Rng};
use regex::{Regex, RegexBuilder};
use sorter::Sorter;
use std::borrow::{Borrow, BorrowMut};
use std::cell::RefCell;
use std::collections::{BTreeMap, HashMap};
@@ -164,7 +164,10 @@ impl RegexCache {
/// The program state describes the environment in which the program executes.
pub struct ProgramState {
pub pc: InsnReference,
cursors: RefCell<BTreeMap<CursorID, Box<dyn Cursor>>>,
btree_table_cursors: RefCell<BTreeMap<CursorID, BTreeCursor>>,
btree_index_cursors: RefCell<BTreeMap<CursorID, BTreeCursor>>,
pseudo_cursors: RefCell<BTreeMap<CursorID, PseudoCursor>>,
sorter_cursors: RefCell<BTreeMap<CursorID, Sorter>>,
registers: Vec<OwnedValue>,
last_compare: Option<std::cmp::Ordering>,
deferred_seek: Option<(CursorID, CursorID)>,
@@ -175,12 +178,18 @@ pub struct ProgramState {
impl ProgramState {
pub fn new(max_registers: usize) -> Self {
let cursors = RefCell::new(BTreeMap::new());
let btree_table_cursors = RefCell::new(BTreeMap::new());
let btree_index_cursors = RefCell::new(BTreeMap::new());
let pseudo_cursors = RefCell::new(BTreeMap::new());
let sorter_cursors = RefCell::new(BTreeMap::new());
let mut registers = Vec::with_capacity(max_registers);
registers.resize(max_registers, OwnedValue::Null);
Self {
pc: 0,
cursors,
btree_table_cursors,
btree_index_cursors,
pseudo_cursors,
sorter_cursors,
registers,
last_compare: None,
deferred_seek: None,
@@ -207,11 +216,24 @@ impl ProgramState {
}
}
macro_rules! must_be_btree_cursor {
($cursor_id:expr, $cursor_ref:expr, $btree_table_cursors:expr, $btree_index_cursors:expr, $insn_name:expr) => {{
let (_, cursor_type) = $cursor_ref.get($cursor_id).unwrap();
let cursor = match cursor_type {
CursorType::BTreeTable(_) => $btree_table_cursors.get_mut(&$cursor_id).unwrap(),
CursorType::BTreeIndex(_) => $btree_index_cursors.get_mut(&$cursor_id).unwrap(),
CursorType::Pseudo(_) => panic!("{} on pseudo cursor", $insn_name),
CursorType::Sorter => panic!("{} on sorter cursor", $insn_name),
};
cursor
}};
}
#[derive(Debug)]
pub struct Program {
pub max_registers: usize,
pub insns: Vec<Insn>,
pub cursor_ref: Vec<(Option<String>, Option<Table>)>,
pub cursor_ref: Vec<(Option<String>, CursorType)>,
pub database_header: Rc<RefCell<DatabaseHeader>>,
pub comments: HashMap<InsnReference, &'static str>,
pub connection: Weak<Connection>,
@@ -248,7 +270,10 @@ impl Program {
}
let insn = &self.insns[state.pc as usize];
trace_insn(self, state.pc as InsnReference, insn);
let mut cursors = state.cursors.borrow_mut();
let mut btree_table_cursors = state.btree_table_cursors.borrow_mut();
let mut btree_index_cursors = state.btree_index_cursors.borrow_mut();
let mut pseudo_cursors = state.pseudo_cursors.borrow_mut();
let mut sorter_cursors = state.sorter_cursors.borrow_mut();
match insn {
Insn::Init { target_pc } => {
assert!(target_pc.is_offset());
@@ -304,7 +329,13 @@ impl Program {
state.pc += 1;
}
Insn::NullRow { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"NullRow"
);
cursor.set_null_flag(true);
state.pc += 1;
}
@@ -569,12 +600,23 @@ impl Program {
cursor_id,
root_page,
} => {
let cursor = Box::new(BTreeCursor::new(
pager.clone(),
*root_page,
self.database_header.clone(),
));
cursors.insert(*cursor_id, cursor);
let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap();
let cursor =
BTreeCursor::new(pager.clone(), *root_page, self.database_header.clone());
match cursor_type {
CursorType::BTreeTable(_) => {
btree_table_cursors.insert(*cursor_id, cursor);
}
CursorType::BTreeIndex(_) => {
btree_index_cursors.insert(*cursor_id, cursor);
}
CursorType::Pseudo(_) => {
panic!("OpenReadAsync on pseudo cursor");
}
CursorType::Sorter => {
panic!("OpenReadAsync on sorter cursor");
}
}
state.pc += 1;
}
Insn::OpenReadAwait => {
@@ -585,17 +627,29 @@ impl Program {
content_reg: _,
num_fields: _,
} => {
let cursor = Box::new(PseudoCursor::new());
cursors.insert(*cursor_id, cursor);
let cursor = PseudoCursor::new();
pseudo_cursors.insert(*cursor_id, cursor);
state.pc += 1;
}
Insn::RewindAsync { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"RewindAsync"
);
return_if_io!(cursor.rewind());
state.pc += 1;
}
Insn::LastAsync { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"LastAsync"
);
return_if_io!(cursor.last());
state.pc += 1;
}
@@ -604,7 +658,13 @@ impl Program {
pc_if_empty,
} => {
assert!(pc_if_empty.is_offset());
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"LastAwait"
);
cursor.wait_for_completion()?;
if cursor.is_empty() {
state.pc = pc_if_empty.to_offset_int();
@@ -617,7 +677,13 @@ impl Program {
pc_if_empty,
} => {
assert!(pc_if_empty.is_offset());
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"RewindAwait"
);
cursor.wait_for_completion()?;
if cursor.is_empty() {
state.pc = pc_if_empty.to_offset_int();
@@ -631,9 +697,9 @@ impl Program {
dest,
} => {
if let Some((index_cursor_id, table_cursor_id)) = state.deferred_seek.take() {
let index_cursor = cursors.get_mut(&index_cursor_id).unwrap();
let index_cursor = btree_index_cursors.get_mut(&index_cursor_id).unwrap();
let rowid = index_cursor.rowid()?;
let table_cursor = cursors.get_mut(&table_cursor_id).unwrap();
let table_cursor = btree_table_cursors.get_mut(&table_cursor_id).unwrap();
match table_cursor.seek(SeekKey::TableRowId(rowid.unwrap()), SeekOp::EQ)? {
CursorResult::Ok(_) => {}
CursorResult::IO => {
@@ -642,18 +708,45 @@ impl Program {
}
}
}
let cursor = cursors.get_mut(cursor_id).unwrap();
if let Some(ref record) = *cursor.record()? {
let null_flag = cursor.get_null_flag();
state.registers[*dest] = if null_flag {
OwnedValue::Null
} else {
record.values[*column].clone()
};
} else {
state.registers[*dest] = OwnedValue::Null;
let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap();
match cursor_type {
CursorType::BTreeTable(_) | CursorType::BTreeIndex(_) => {
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"Column"
);
let record = cursor.record()?;
if let Some(record) = record.as_ref() {
state.registers[*dest] = if cursor.get_null_flag() {
OwnedValue::Null
} else {
record.values[*column].clone()
};
} else {
state.registers[*dest] = OwnedValue::Null;
}
}
CursorType::Sorter => {
let cursor = sorter_cursors.get_mut(cursor_id).unwrap();
if let Some(record) = cursor.record() {
state.registers[*dest] = record.values[*column].clone();
} else {
state.registers[*dest] = OwnedValue::Null;
}
}
CursorType::Pseudo(_) => {
let cursor = pseudo_cursors.get_mut(cursor_id).unwrap();
if let Some(record) = cursor.record() {
state.registers[*dest] = record.values[*column].clone();
} else {
state.registers[*dest] = OwnedValue::Null;
}
}
}
state.pc += 1;
}
Insn::MakeRecord {
@@ -671,13 +764,25 @@ impl Program {
return Ok(StepResult::Row(record));
}
Insn::NextAsync { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"NextAsync"
);
cursor.set_null_flag(false);
return_if_io!(cursor.next());
state.pc += 1;
}
Insn::PrevAsync { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"PrevAsync"
);
cursor.set_null_flag(false);
return_if_io!(cursor.prev());
state.pc += 1;
@@ -687,7 +792,13 @@ impl Program {
pc_if_next,
} => {
assert!(pc_if_next.is_offset());
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"PrevAwait"
);
cursor.wait_for_completion()?;
if !cursor.is_empty() {
state.pc = pc_if_next.to_offset_int();
@@ -700,7 +811,13 @@ impl Program {
pc_if_next,
} => {
assert!(pc_if_next.is_offset());
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = must_be_btree_cursor!(
*cursor_id,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"NextAwait"
);
cursor.wait_for_completion()?;
if !cursor.is_empty() {
state.pc = pc_if_next.to_offset_int();
@@ -818,9 +935,9 @@ impl Program {
}
Insn::RowId { cursor_id, dest } => {
if let Some((index_cursor_id, table_cursor_id)) = state.deferred_seek.take() {
let index_cursor = cursors.get_mut(&index_cursor_id).unwrap();
let index_cursor = btree_index_cursors.get_mut(&index_cursor_id).unwrap();
let rowid = index_cursor.rowid()?;
let table_cursor = cursors.get_mut(&table_cursor_id).unwrap();
let table_cursor = btree_table_cursors.get_mut(&table_cursor_id).unwrap();
match table_cursor.seek(SeekKey::TableRowId(rowid.unwrap()), SeekOp::EQ)? {
CursorResult::Ok(_) => {}
CursorResult::IO => {
@@ -830,7 +947,7 @@ impl Program {
}
}
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_table_cursors.get_mut(cursor_id).unwrap();
if let Some(ref rowid) = cursor.rowid()? {
state.registers[*dest] = OwnedValue::Integer(*rowid as i64);
} else {
@@ -844,7 +961,7 @@ impl Program {
target_pc,
} => {
assert!(target_pc.is_offset());
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_table_cursors.get_mut(cursor_id).unwrap();
let rowid = match &state.registers[*src_reg] {
OwnedValue::Integer(rowid) => *rowid as u64,
OwnedValue::Null => {
@@ -880,7 +997,7 @@ impl Program {
} => {
assert!(target_pc.is_offset());
if *is_index {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_index_cursors.get_mut(cursor_id).unwrap();
let record_from_regs: OwnedRecord =
make_owned_record(&state.registers, start_reg, num_regs);
let found = return_if_io!(
@@ -892,7 +1009,7 @@ impl Program {
state.pc += 1;
}
} else {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_table_cursors.get_mut(cursor_id).unwrap();
let rowid = match &state.registers[*start_reg] {
OwnedValue::Null => {
// All integer values are greater than null so we just rewind the cursor
@@ -925,7 +1042,7 @@ impl Program {
} => {
assert!(target_pc.is_offset());
if *is_index {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_index_cursors.get_mut(cursor_id).unwrap();
let record_from_regs: OwnedRecord =
make_owned_record(&state.registers, start_reg, num_regs);
let found = return_if_io!(
@@ -937,7 +1054,7 @@ impl Program {
state.pc += 1;
}
} else {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_table_cursors.get_mut(cursor_id).unwrap();
let rowid = match &state.registers[*start_reg] {
OwnedValue::Null => {
// All integer values are greater than null so we just rewind the cursor
@@ -968,7 +1085,7 @@ impl Program {
target_pc,
} => {
assert!(target_pc.is_offset());
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_index_cursors.get_mut(cursor_id).unwrap();
let record_from_regs: OwnedRecord =
make_owned_record(&state.registers, start_reg, num_regs);
if let Some(ref idx_record) = *cursor.record()? {
@@ -991,7 +1108,7 @@ impl Program {
target_pc,
} => {
assert!(target_pc.is_offset());
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_index_cursors.get_mut(cursor_id).unwrap();
let record_from_regs: OwnedRecord =
make_owned_record(&state.registers, start_reg, num_regs);
if let Some(ref idx_record) = *cursor.record()? {
@@ -1275,47 +1392,46 @@ impl Program {
_ => unreachable!(),
})
.collect();
let cursor = Box::new(sorter::Sorter::new(order));
cursors.insert(*cursor_id, cursor);
let cursor = sorter::Sorter::new(order);
sorter_cursors.insert(*cursor_id, cursor);
state.pc += 1;
}
Insn::SorterData {
cursor_id,
dest_reg,
pseudo_cursor: sorter_cursor,
pseudo_cursor,
} => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let record = match *cursor.record()? {
Some(ref record) => record.clone(),
let sorter_cursor = sorter_cursors.get_mut(cursor_id).unwrap();
let record = match sorter_cursor.record() {
Some(record) => record.clone(),
None => {
state.pc += 1;
continue;
}
};
state.registers[*dest_reg] = OwnedValue::Record(record.clone());
let sorter_cursor = cursors.get_mut(sorter_cursor).unwrap();
sorter_cursor.insert(&OwnedValue::Integer(0), &record, false)?; // fix key later
let pseudo_cursor = pseudo_cursors.get_mut(pseudo_cursor).unwrap();
pseudo_cursor.insert(record);
state.pc += 1;
}
Insn::SorterInsert {
cursor_id,
record_reg,
} => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = sorter_cursors.get_mut(cursor_id).unwrap();
let record = match &state.registers[*record_reg] {
OwnedValue::Record(record) => record,
_ => unreachable!("SorterInsert on non-record register"),
};
// TODO: set correct key
cursor.insert(&OwnedValue::Integer(0), record, false)?;
cursor.insert(record);
state.pc += 1;
}
Insn::SorterSort {
cursor_id,
pc_if_empty,
} => {
if let Some(cursor) = cursors.get_mut(cursor_id) {
cursor.rewind()?;
if let Some(cursor) = sorter_cursors.get_mut(cursor_id) {
cursor.sort();
state.pc += 1;
} else {
state.pc = pc_if_empty.to_offset_int();
@@ -1326,8 +1442,8 @@ impl Program {
pc_if_next,
} => {
assert!(pc_if_next.is_offset());
let cursor = cursors.get_mut(cursor_id).unwrap();
return_if_io!(cursor.next());
let cursor = sorter_cursors.get_mut(cursor_id).unwrap();
cursor.next();
if !cursor.is_empty() {
state.pc = pc_if_next.to_offset_int();
} else {
@@ -1415,6 +1531,13 @@ impl Program {
Err(e) => return Err(e),
}
}
JsonFunc::JsonErrorPosition => {
let json_value = &state.registers[*start_reg];
match json_error_position(json_value) {
Ok(pos) => state.registers[*dest] = pos,
Err(e) => return Err(e),
}
}
},
crate::function::Func::Scalar(scalar_func) => match scalar_func {
ScalarFunc::Cast => {
@@ -1870,7 +1993,7 @@ impl Program {
record_reg,
flag: _,
} => {
let cursor = cursors.get_mut(cursor).unwrap();
let cursor = btree_table_cursors.get_mut(cursor).unwrap();
let record = match &state.registers[*record_reg] {
OwnedValue::Record(r) => r,
_ => unreachable!("Not a record! Cannot insert a non record value."),
@@ -1880,7 +2003,7 @@ impl Program {
state.pc += 1;
}
Insn::InsertAwait { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_table_cursors.get_mut(cursor_id).unwrap();
cursor.wait_for_completion()?;
// Only update last_insert_rowid for regular table inserts, not schema modifications
if cursor.root_page() != 1 {
@@ -1896,19 +2019,19 @@ impl Program {
state.pc += 1;
}
Insn::DeleteAsync { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_table_cursors.get_mut(cursor_id).unwrap();
return_if_io!(cursor.delete());
state.pc += 1;
}
Insn::DeleteAwait { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
let cursor = btree_table_cursors.get_mut(cursor_id).unwrap();
cursor.wait_for_completion()?;
state.pc += 1;
}
Insn::NewRowid {
cursor, rowid_reg, ..
} => {
let cursor = cursors.get_mut(cursor).unwrap();
let cursor = btree_table_cursors.get_mut(cursor).unwrap();
// TODO: make io handle rng
let rowid = return_if_io!(get_new_rowid(cursor, thread_rng()));
state.registers[*rowid_reg] = OwnedValue::Integer(rowid);
@@ -1934,7 +2057,13 @@ impl Program {
rowid_reg,
target_pc,
} => {
let cursor = cursors.get_mut(cursor).unwrap();
let cursor = must_be_btree_cursor!(
*cursor,
self.cursor_ref,
btree_table_cursors,
btree_index_cursors,
"NotExists"
);
let exists = return_if_io!(cursor.exists(&state.registers[*rowid_reg]));
if exists {
state.pc += 1;
@@ -1949,12 +2078,15 @@ impl Program {
cursor_id,
root_page,
} => {
let cursor = Box::new(BTreeCursor::new(
pager.clone(),
*root_page,
self.database_header.clone(),
));
cursors.insert(*cursor_id, cursor);
let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap();
let is_index = cursor_type.is_index();
let cursor =
BTreeCursor::new(pager.clone(), *root_page, self.database_header.clone());
if is_index {
btree_index_cursors.insert(*cursor_id, cursor);
} else {
btree_table_cursors.insert(*cursor_id, cursor);
}
state.pc += 1;
}
Insn::OpenWriteAwait {} => {
@@ -1986,7 +2118,21 @@ impl Program {
state.pc += 1;
}
Insn::Close { cursor_id } => {
cursors.remove(cursor_id);
let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap();
match cursor_type {
CursorType::BTreeTable(_) => {
let _ = btree_table_cursors.remove(cursor_id);
}
CursorType::BTreeIndex(_) => {
let _ = btree_index_cursors.remove(cursor_id);
}
CursorType::Pseudo(_) => {
let _ = pseudo_cursors.remove(cursor_id);
}
CursorType::Sorter => {
let _ = sorter_cursors.remove(cursor_id);
}
}
state.pc += 1;
}
Insn::IsNull { src, target_pc } => {
@@ -2017,7 +2163,7 @@ impl Program {
}
}
fn get_new_rowid<R: Rng>(cursor: &mut Box<dyn Cursor>, mut rng: R) -> Result<CursorResult<i64>> {
fn get_new_rowid<R: Rng>(cursor: &mut BTreeCursor, mut rng: R) -> Result<CursorResult<i64>> {
match cursor.seek_to_last()? {
CursorResult::Ok(()) => {}
CursorResult::IO => return Ok(CursorResult::IO),
@@ -3055,180 +3201,16 @@ fn exec_math_log(arg: &OwnedValue, base: Option<&OwnedValue>) -> OwnedValue {
#[cfg(test)]
mod tests {
use crate::{
types::{SeekKey, SeekOp},
vdbe::exec_replace,
};
use crate::vdbe::exec_replace;
use super::{
exec_abs, exec_char, exec_hex, exec_if, exec_instr, exec_length, exec_like, exec_lower,
exec_ltrim, exec_max, exec_min, exec_nullif, exec_quote, exec_random, exec_randomblob,
exec_round, exec_rtrim, exec_sign, exec_soundex, exec_substring, exec_trim, exec_typeof,
exec_unhex, exec_unicode, exec_upper, exec_zeroblob, execute_sqlite_version, get_new_rowid,
AggContext, Cursor, CursorResult, LimboError, OwnedRecord, OwnedValue, Result,
exec_unhex, exec_unicode, exec_upper, exec_zeroblob, execute_sqlite_version, AggContext,
OwnedValue,
};
use mockall::{mock, predicate};
use rand::{rngs::mock::StepRng, thread_rng};
use std::{cell::Ref, collections::HashMap, rc::Rc};
mock! {
Cursor {
fn seek_to_last(&mut self) -> Result<CursorResult<()>>;
fn seek<'a>(&mut self, key: SeekKey<'a>, op: SeekOp) -> Result<CursorResult<bool>>;
fn rowid(&self) -> Result<Option<u64>>;
fn seek_rowid(&mut self, rowid: u64) -> Result<CursorResult<bool>>;
}
}
impl Cursor for MockCursor {
fn root_page(&self) -> usize {
unreachable!()
}
fn seek_to_last(&mut self) -> Result<CursorResult<()>> {
self.seek_to_last()
}
fn rowid(&self) -> Result<Option<u64>> {
self.rowid()
}
fn seek(&mut self, key: SeekKey<'_>, op: SeekOp) -> Result<CursorResult<bool>> {
self.seek(key, op)
}
fn rewind(&mut self) -> Result<CursorResult<()>> {
unimplemented!()
}
fn next(&mut self) -> Result<CursorResult<()>> {
unimplemented!()
}
fn record(&self) -> Result<Ref<Option<OwnedRecord>>> {
unimplemented!()
}
fn is_empty(&self) -> bool {
unimplemented!()
}
fn set_null_flag(&mut self, _flag: bool) {
unimplemented!()
}
fn get_null_flag(&self) -> bool {
unimplemented!()
}
fn insert(
&mut self,
_key: &OwnedValue,
_record: &OwnedRecord,
_is_leaf: bool,
) -> Result<CursorResult<()>> {
unimplemented!()
}
fn delete(&mut self) -> Result<CursorResult<()>> {
unimplemented!()
}
fn wait_for_completion(&mut self) -> Result<()> {
unimplemented!()
}
fn exists(&mut self, _key: &OwnedValue) -> Result<CursorResult<bool>> {
unimplemented!()
}
fn btree_create(&mut self, _flags: usize) -> u32 {
unimplemented!()
}
fn last(&mut self) -> Result<CursorResult<()>> {
todo!()
}
fn prev(&mut self) -> Result<CursorResult<()>> {
todo!()
}
}
#[test]
fn test_get_new_rowid() -> Result<()> {
// Test case 0: Empty table
let mut mock = MockCursor::new();
mock.expect_seek_to_last()
.return_once(|| Ok(CursorResult::Ok(())));
mock.expect_rowid().return_once(|| Ok(None));
let result = get_new_rowid(&mut (Box::new(mock) as Box<dyn Cursor>), thread_rng())?;
assert_eq!(
result,
CursorResult::Ok(1),
"For an empty table, rowid should be 1"
);
// Test case 1: Normal case, rowid within i64::MAX
let mut mock = MockCursor::new();
mock.expect_seek_to_last()
.return_once(|| Ok(CursorResult::Ok(())));
mock.expect_rowid().return_once(|| Ok(Some(100)));
let result = get_new_rowid(&mut (Box::new(mock) as Box<dyn Cursor>), thread_rng())?;
assert_eq!(result, CursorResult::Ok(101));
// Test case 2: Rowid exceeds i64::MAX, need to generate random rowid
let mut mock = MockCursor::new();
mock.expect_seek_to_last()
.return_once(|| Ok(CursorResult::Ok(())));
mock.expect_rowid()
.return_once(|| Ok(Some(i64::MAX as u64)));
mock.expect_seek()
.with(predicate::always(), predicate::always())
.returning(|rowid, _| {
if rowid == SeekKey::TableRowId(50) {
Ok(CursorResult::Ok(false))
} else {
Ok(CursorResult::Ok(true))
}
});
// Mock the random number generation
let new_rowid =
get_new_rowid(&mut (Box::new(mock) as Box<dyn Cursor>), StepRng::new(1, 1))?;
assert_eq!(new_rowid, CursorResult::Ok(50));
// Test case 3: IO error
let mut mock = MockCursor::new();
mock.expect_seek_to_last()
.return_once(|| Ok(CursorResult::Ok(())));
mock.expect_rowid()
.return_once(|| Ok(Some(i64::MAX as u64)));
mock.expect_seek()
.with(predicate::always(), predicate::always())
.return_once(|_, _| Ok(CursorResult::IO));
let result = get_new_rowid(&mut (Box::new(mock) as Box<dyn Cursor>), thread_rng());
assert!(matches!(result, Ok(CursorResult::IO)));
// Test case 4: Failure to generate new rowid
let mut mock = MockCursor::new();
mock.expect_seek_to_last()
.return_once(|| Ok(CursorResult::Ok(())));
mock.expect_rowid()
.return_once(|| Ok(Some(i64::MAX as u64)));
mock.expect_seek()
.with(predicate::always(), predicate::always())
.returning(|_, _| Ok(CursorResult::Ok(true)));
// Mock the random number generation
let result = get_new_rowid(&mut (Box::new(mock) as Box<dyn Cursor>), StepRng::new(1, 1));
assert!(matches!(result, Err(LimboError::InternalError(_))));
Ok(())
}
use std::{collections::HashMap, rc::Rc};
#[test]
fn test_length() {

95
core/vdbe/sorter.rs
View File

@@ -1,13 +1,9 @@
use crate::{
types::{Cursor, CursorResult, OwnedRecord, OwnedValue, SeekKey, SeekOp},
Result,
};
use std::cell::{Ref, RefCell};
use crate::types::OwnedRecord;
use std::cmp::Ordering;
pub struct Sorter {
records: Vec<OwnedRecord>,
current: RefCell<Option<OwnedRecord>>,
current: Option<OwnedRecord>,
order: Vec<bool>,
}
@@ -15,23 +11,15 @@ impl Sorter {
pub fn new(order: Vec<bool>) -> Self {
Self {
records: Vec::new(),
current: RefCell::new(None),
current: None,
order,
}
}
}
impl Cursor for Sorter {
fn is_empty(&self) -> bool {
self.current.borrow().is_none()
pub fn is_empty(&self) -> bool {
self.current.is_none()
}
fn root_page(&self) -> usize {
unreachable!()
}
// We do the sorting here since this is what is called by the SorterSort instruction
fn rewind(&mut self) -> Result<CursorResult<()>> {
pub fn sort(&mut self) {
self.records.sort_by(|a, b| {
let cmp_by_idx = |idx: usize, ascending: bool| {
let a = &a.values[idx];
@@ -55,73 +43,14 @@ impl Cursor for Sorter {
self.records.reverse();
self.next()
}
fn next(&mut self) -> Result<CursorResult<()>> {
let mut c = self.current.borrow_mut();
*c = self.records.pop();
Ok(CursorResult::Ok(()))
pub fn next(&mut self) {
self.current = self.records.pop();
}
pub fn record(&self) -> Option<&OwnedRecord> {
self.current.as_ref()
}
fn wait_for_completion(&mut self) -> Result<()> {
Ok(())
}
fn rowid(&self) -> Result<Option<u64>> {
todo!();
}
fn seek(&mut self, _: SeekKey<'_>, _: SeekOp) -> Result<CursorResult<bool>> {
unimplemented!();
}
fn seek_to_last(&mut self) -> Result<CursorResult<()>> {
unimplemented!();
}
fn record(&self) -> Result<Ref<Option<OwnedRecord>>> {
let ret = self.current.borrow();
// log::trace!("returning {:?}", ret);
Ok(ret)
}
fn insert(
&mut self,
key: &OwnedValue,
record: &OwnedRecord,
moved_before: bool,
) -> Result<CursorResult<()>> {
let _ = key;
let _ = moved_before;
pub fn insert(&mut self, record: &OwnedRecord) {
self.records.push(OwnedRecord::new(record.values.to_vec()));
Ok(CursorResult::Ok(()))
}
fn delete(&mut self) -> Result<CursorResult<()>> {
unimplemented!()
}
fn set_null_flag(&mut self, _flag: bool) {
todo!();
}
fn get_null_flag(&self) -> bool {
false
}
fn exists(&mut self, key: &OwnedValue) -> Result<CursorResult<bool>> {
let _ = key;
todo!()
}
fn btree_create(&mut self, _flags: usize) -> u32 {
unreachable!("Why did you try to build a new tree with a sorter??? Stand up, open the door and take a walk for 30 min to come back with a better plan.");
}
fn last(&mut self) -> Result<CursorResult<()>> {
todo!()
}
fn prev(&mut self) -> Result<CursorResult<()>> {
todo!()
}
}

32
testing/json.test
View File

@@ -474,3 +474,35 @@ do_execsql_test json_type_cast {
do_execsql_test json_type_null_arg {
select json_type(null)
} {{}}
do_execsql_test json_error_position_valid {
SELECT json_error_position('{"a":55,"b":72,}');
} {{0}}
do_execsql_test json_error_position_valid_ws {
SELECT json_error_position('{"a":55,"b":72 , }');
} {{0}}
do_execsql_test json_error_position_object {
SELECT json_error_position('{"a":55,"b":72,,}');
} {{16}}
do_execsql_test json_error_position_array_valid {
SELECT json_error_position('["a",55,"b",72,]');
} {{0}}
do_execsql_test json_error_position_array_valid_ws {
SELECT json_error_position('["a",55,"b",72 , ]');
} {{0}}
do_execsql_test json_error_position_array {
SELECT json_error_position('["a",55,"b",72,,]');
} {{16}}
do_execsql_test json_error_position_null {
SELECT json_error_position(NULL);
} {{}}
do_execsql_test json_error_position_complex {
SELECT json_error_position('{a:null,{"h":[1,[1,2,3]],"j":"abc"}:true}');
} {{9}}