aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-02-18 06:24:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into json-extract

Browse Source
This commit is contained in:
Kacper Madej
2024-12-31 15:36:51 +07:00
parent ad9acf7400 b9187d57f6
commit 2e730ead25
69 changed files with 6112 additions and 2208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
.github/workflows/rust.yml vendored
View File

@@ -13,7 +13,7 @@ jobs:
cargo-fmt-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Check formatting
run: cargo fmt --check
@@ -39,10 +39,19 @@ jobs:
run: cargo test --verbose
timeout-minutes: 5
clippy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Clippy
run: |
cargo clippy -- -A clippy::all -W clippy::correctness -W clippy::perf -W clippy::suspicious --deny=warnings
build-wasm:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Install
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
- run: wasm-pack build --target nodejs bindings/wasm
@@ -50,7 +59,7 @@ jobs:
bench:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Bench
run: cargo bench
@@ -66,14 +75,14 @@ jobs:
run: |
curl -L $LINK/$CARGO_C_FILE | tar xz -C ~/.cargo/bin
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Test
run: make test
test-sqlite:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Install sqlite
run: sudo apt update && sudo apt install -y sqlite3 libsqlite3-dev
- name: Test

262
COMPAT.md
View File

@@ -20,81 +20,81 @@ This document describes the SQLite compatibility status of Limbo:
## SQL statements
| Statement | Status | Comment |
|---------------------------|---------|-----------------------------------------------------------------------------------|
| ALTER TABLE | No | |
| ANALYZE | No | |
| ATTACH DATABASE | No | |
| BEGIN TRANSACTION | No | |
| COMMIT TRANSACTION | No | |
| CREATE INDEX | No | |
| CREATE TABLE | Partial | |
| CREATE TRIGGER | No | |
| CREATE VIEW | No | |
| CREATE VIRTUAL TABLE | No | |
| DELETE | No | |
| DETACH DATABASE | No | |
| DROP INDEX | No | |
| DROP TABLE | No | |
| DROP TRIGGER | No | |
| DROP VIEW | No | |
| END TRANSACTION | No | |
| EXPLAIN | Yes | |
| INDEXED BY | No | |
| INSERT | Partial | |
| ON CONFLICT clause | No | |
| PRAGMA | Partial | |
| PRAGMA cache_size | Yes | |
| REINDEX | No | |
| RELEASE SAVEPOINT | No | |
| REPLACE | No | |
| RETURNING clause | No | |
| ROLLBACK TRANSACTION | No | |
| SAVEPOINT | No | |
| SELECT | Yes | |
| SELECT ... WHERE | Yes | |
| SELECT ... WHERE ... LIKE | Yes | |
| SELECT ... LIMIT | Yes | |
| SELECT ... ORDER BY | Yes | |
| SELECT ... GROUP BY | Yes | |
| SELECT ... HAVING | Yes | |
| SELECT ... JOIN | Yes | |
| Statement | Status | Comment |
| ------------------------- | ------- | ------- |
| ALTER TABLE | No | |
| ANALYZE | No | |
| ATTACH DATABASE | No | |
| BEGIN TRANSACTION | No | |
| COMMIT TRANSACTION | No | |
| CREATE INDEX | No | |
| CREATE TABLE | Partial | |
| CREATE TRIGGER | No | |
| CREATE VIEW | No | |
| CREATE VIRTUAL TABLE | No | |
| DELETE | No | |
| DETACH DATABASE | No | |
| DROP INDEX | No | |
| DROP TABLE | No | |
| DROP TRIGGER | No | |
| DROP VIEW | No | |
| END TRANSACTION | No | |
| EXPLAIN | Yes | |
| INDEXED BY | No | |
| INSERT | Partial | |
| ON CONFLICT clause | No | |
| PRAGMA | Partial | |
| PRAGMA cache_size | Yes | |
| REINDEX | No | |
| RELEASE SAVEPOINT | No | |
| REPLACE | No | |
| RETURNING clause | No | |
| ROLLBACK TRANSACTION | No | |
| SAVEPOINT | No | |
| SELECT | Yes | |
| SELECT ... WHERE | Yes | |
| SELECT ... WHERE ... LIKE | Yes | |
| SELECT ... LIMIT | Yes | |
| SELECT ... ORDER BY | Yes | |
| SELECT ... GROUP BY | Yes | |
| SELECT ... HAVING | Yes | |
| SELECT ... JOIN | Yes | |
| SELECT ... CROSS JOIN | Yes | SQLite CROSS JOIN means "do not reorder joins". We don't support that yet anyway. |
| SELECT ... INNER JOIN | Yes | |
| SELECT ... OUTER JOIN | Partial | no RIGHT JOIN |
| SELECT ... JOIN USING | Yes | |
| SELECT ... NATURAL JOIN | Yes | |
| UPDATE | No | |
| UPSERT | No | |
| VACUUM | No | |
| WITH clause | No | |
| SELECT ... INNER JOIN | Yes | |
| SELECT ... OUTER JOIN | Partial | no RIGHT JOIN |
| SELECT ... JOIN USING | Yes | |
| SELECT ... NATURAL JOIN | Yes | |
| UPDATE | No | |
| UPSERT | No | |
| VACUUM | No | |
| WITH clause | No | |
### SELECT Expressions
Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
| Syntax | Status | Comment |
|---------------------------|---------|------------------------------------------|
| literals | Yes | |
| schema.table.column | Partial | Schemas aren't supported |
| unary operator | Yes | |
| binary operator | Partial | Only `%`, `!<`, and `!>` are unsupported |
| agg() FILTER (WHERE ...) | No | Is incorrectly ignored |
| ... OVER (...) | No | Is incorrectly ignored |
| (expr) | Yes | |
| CAST (expr AS type) | Yes | |
| COLLATE | No | |
| (NOT) LIKE | No | |
| (NOT) GLOB | No | |
| (NOT) REGEXP | No | |
| (NOT) MATCH | No | |
| IS (NOT) | No | |
| IS (NOT) DISTINCT FROM | No | |
| (NOT) BETWEEN ... AND ... | No | |
| (NOT) IN (subquery) | No | |
| (NOT) EXISTS (subquery) | No | |
| CASE WHEN THEN ELSE END | Yes | |
| RAISE | No | |
| Syntax | Status | Comment |
|------------------------------|---------|---------|
| literals | Yes | |
| schema.table.column | Partial | Schemas aren't supported |
| unary operator | Yes | |
| binary operator | Partial | Only `%`, `!<`, and `!>` are unsupported |
| agg() FILTER (WHERE ...) | No | Is incorrectly ignored |
| ... OVER (...) | No | Is incorrectly ignored |
| (expr) | Yes | |
| CAST (expr AS type) | Yes | |
| COLLATE | No | |
| (NOT) LIKE | No | |
| (NOT) GLOB | No | |
| (NOT) REGEXP | No | |
| (NOT) MATCH | No | |
| IS (NOT) | No | |
| IS (NOT) DISTINCT FROM | No | |
| (NOT) BETWEEN ... AND ... | No | |
| (NOT) IN (subquery) | No | |
| (NOT) EXISTS (subquery) | No | |
| CASE WHEN THEN ELSE END | Yes | |
| RAISE | No | |
## SQL functions
@@ -116,8 +116,8 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
| instr(X,Y) | Yes | |
| last_insert_rowid() | Yes | |
| length(X) | Yes | |
| like(X,Y) | No | |
| like(X,Y,Z) | No | |
| like(X,Y) | Yes | |
| like(X,Y,Z) | Yes | |
| likelihood(X,Y) | No | |
| likely(X) | No | |
| load_extension(X) | No | |
@@ -160,10 +160,14 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
| upper(X) | Yes | |
| zeroblob(N) | Yes | |
### Mathematical functions
| Function | Status | Comment |
|------------|--------|---------|
| ---------- | ------ | ------- |
| acos(X) | Yes | |
| acosh(X) | Yes | |
| asin(X) | Yes | |
@@ -197,18 +201,18 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
### Aggregate functions
| Function | Status | Comment |
|-------------------|--------|---------|
| avg(X) | Yes | |
| count() | Yes | |
| count(*) | Yes | |
| group_concat(X) | Yes | |
| group_concat(X,Y) | Yes | |
| string_agg(X,Y) | Yes | |
| max(X) | Yes | |
| min(X) | Yes | |
| sum(X) | Yes | |
| total(X) | Yes | |
| Function | Status | Comment |
|------------------------------|---------|---------|
| avg(X) | Yes | |
| count() | Yes | |
| count(*) | Yes | |
| group_concat(X) | Yes | |
| group_concat(X,Y) | Yes | |
| string_agg(X,Y) | Yes | |
| max(X) | Yes | |
| min(X) | Yes | |
| sum(X) | Yes | |
| total(X) | Yes | |
### Date and time functions
@@ -224,45 +228,45 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
### JSON functions
| Function | Status | Comment |
|------------------------------------|---------|----------------------------------------------------------------------------------------------------------------------|
| json(json) | Partial | |
| jsonb(json) | | |
| json_array(value1,value2,...) | Yes | |
| jsonb_array(value1,value2,...) | | |
| json_array_length(json) | | |
| json_array_length(json,path) | | |
| json_error_position(json) | | |
| json_extract(json,path,...) | Partial | Does not fully support unicode literal syntax and does not allow numbers > 2^127 - 1 (which SQLite truncates to i32) |
| jsonb_extract(json,path,...) | | |
| json -> path | | |
| json ->> path | | |
| json_insert(json,path,value,...) | | |
| jsonb_insert(json,path,value,...) | | |
| json_object(label1,value1,...) | | |
| jsonb_object(label1,value1,...) | | |
| json_patch(json1,json2) | | |
| jsonb_patch(json1,json2) | | |
| json_pretty(json) | | |
| json_remove(json,path,...) | | |
| jsonb_remove(json,path,...) | | |
| json_replace(json,path,value,...) | | |
| jsonb_replace(json,path,value,...) | | |
| json_set(json,path,value,...) | | |
| jsonb_set(json,path,value,...) | | |
| json_type(json) | | |
| json_type(json,path) | | |
| json_valid(json) | | |
| json_valid(json,flags) | | |
| json_quote(value) | | |
| json_group_array(value) | | |
| jsonb_group_array(value) | | |
| json_group_object(label,value) | | |
| jsonb_group_object(name,value) | | |
| json_each(json) | | |
| json_each(json,path) | | |
| json_tree(json) | | |
| json_tree(json,path) | | |
| Function | Status | Comment |
|------------------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------|
| json(json) | Partial | |
| jsonb(json) | | |
| json_array(value1,value2,...) | Yes | |
| jsonb_array(value1,value2,...) | | |
| json_array_length(json) | Yes | |
| json_array_length(json,path) | Yes | |
| json_error_position(json) | | |
| json_extract(json,path,...) | Partial | Does not fully support unicode literal syntax and does not allow numbers > 2^127 - 1 (which SQLite truncates to i32), does not support BLOBs |
| jsonb_extract(json,path,...) | | |
| json -> path | | |
| json ->> path | | |
| json_insert(json,path,value,...) | | |
| jsonb_insert(json,path,value,...) | | |
| json_object(label1,value1,...) | | |
| jsonb_object(label1,value1,...) | | |
| json_patch(json1,json2) | | |
| jsonb_patch(json1,json2) | | |
| json_pretty(json) | | |
| json_remove(json,path,...) | | |
| jsonb_remove(json,path,...) | | |
| json_replace(json,path,value,...) | | |
| jsonb_replace(json,path,value,...) | | |
| json_set(json,path,value,...) | | |
| jsonb_set(json,path,value,...) | | |
| json_type(json) | | |
| json_type(json,path) | | |
| json_valid(json) | | |
| json_valid(json,flags) | | |
| json_quote(value) | | |
| json_group_array(value) | | |
| jsonb_group_array(value) | | |
| json_group_object(label,value) | | |
| jsonb_group_object(name,value) | | |
| json_each(json) | | |
| json_each(json,path) | | |
| json_tree(json) | | |
| json_tree(json,path) | | |
## SQLite API
@@ -390,7 +394,7 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
| ReadCookie | No |
| Real | Yes |
| RealAffinity | Yes |
| Remainder | No |
| Remainder | Yes |
| ResetCount | No |
| ResultRow | Yes |
| Return | Yes |
@@ -448,3 +452,13 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
| Variable | No |
| VerifyCookie | No |
| Yield | Yes |
| LibSql Compatibility / Extensions | | |
|-----------------------------------|-----|---------------------------------------------------------------|
| **UUID** | | UUID's in limbo are `blobs` by default |
| uuid4() | Yes | uuid version 4 |
| uuid4_str() | Yes | uuid v4 string alias `gen_random_uuid()` for PG compatibility |
| uuid7(X?) | Yes | uuid version 7, Optional arg for seconds since epoch |
| uuid7_timestamp_ms(X) | Yes | Convert a uuid v7 to milliseconds since epoch |
| uuid_str(X) | Yes | Convert a valid uuid to string |
| uuid_blob(X) | Yes | Convert a valid uuid to blob |

11
Cargo.lock generated
View File

@@ -1138,6 +1138,7 @@ dependencies = [
"jsonb",
"julian_day_converter",
"libc",
"limbo_macros",
"log",
"mimalloc",
"mockall",
@@ -1148,6 +1149,7 @@ dependencies = [
"pprof",
"rand",
"regex",
"regex-syntax",
"rstest",
"rusqlite",
"rustix",
@@ -1156,13 +1158,19 @@ dependencies = [
"sqlite3-parser",
"tempfile",
"thiserror 1.0.69",
"uuid",
]
[[package]]
name = "limbo_macros"
version = "0.0.10"
[[package]]
name = "limbo_sim"
version = "0.0.10"
dependencies = [
"anarchist-readable-name-generator-lib",
"clap",
"env_logger 0.10.2",
"limbo_core",
"log",
@@ -2272,6 +2280,9 @@ name = "uuid"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
dependencies = [
"getrandom",
]
[[package]]
name = "vcpkg"

2
Cargo.toml
View File

@@ -9,7 +9,7 @@ members = [
"sqlite3",
"core",
"simulator",
"test",
"test", "macros",
]
exclude = ["perf/latency/limbo"]

61
bindings/python/src/lib.rs
View File

@@ -104,7 +104,10 @@ impl Cursor {
// TODO: use stmt_is_dml to set rowcount
if stmt_is_dml {
todo!()
return Err(PyErr::new::<NotSupportedError, _>(
"DML statements (INSERT/UPDATE/DELETE) are not fully supported in this version",
)
.into());
}
Ok(Cursor {
@@ -125,21 +128,26 @@ impl Cursor {
match smt_lock.step().map_err(|e| {
PyErr::new::<OperationalError, _>(format!("Step error: {:?}", e))
})? {
limbo_core::RowResult::Row(row) => {
limbo_core::StepResult::Row(row) => {
let py_row = row_to_py(py, &row);
return Ok(Some(py_row));
}
limbo_core::RowResult::IO => {
limbo_core::StepResult::IO => {
self.conn.io.run_once().map_err(|e| {
PyErr::new::<OperationalError, _>(format!("IO error: {:?}", e))
})?;
}
limbo_core::RowResult::Interrupt => {
limbo_core::StepResult::Interrupt => {
return Ok(None);
}
limbo_core::RowResult::Done => {
limbo_core::StepResult::Done => {
return Ok(None);
}
limbo_core::StepResult::Busy => {
return Err(
PyErr::new::<OperationalError, _>("Busy error".to_string()).into()
);
}
}
}
} else {
@@ -159,21 +167,26 @@ impl Cursor {
match smt_lock.step().map_err(|e| {
PyErr::new::<OperationalError, _>(format!("Step error: {:?}", e))
})? {
limbo_core::RowResult::Row(row) => {
limbo_core::StepResult::Row(row) => {
let py_row = row_to_py(py, &row);
results.push(py_row);
}
limbo_core::RowResult::IO => {
limbo_core::StepResult::IO => {
self.conn.io.run_once().map_err(|e| {
PyErr::new::<OperationalError, _>(format!("IO error: {:?}", e))
})?;
}
limbo_core::RowResult::Interrupt => {
limbo_core::StepResult::Interrupt => {
return Ok(results);
}
limbo_core::RowResult::Done => {
limbo_core::StepResult::Done => {
return Ok(results);
}
limbo_core::StepResult::Busy => {
return Err(
PyErr::new::<OperationalError, _>("Busy error".to_string()).into()
);
}
}
}
} else {
@@ -181,18 +194,24 @@ impl Cursor {
}
}
pub fn close(&self) -> Result<()> {
todo!()
pub fn close(&self) -> PyResult<()> {
Err(PyErr::new::<NotSupportedError, _>(
"close() is not supported in this version",
))
}
#[pyo3(signature = (sql, parameters=None))]
pub fn executemany(&self, sql: &str, parameters: Option<Py<PyList>>) {
todo!()
pub fn executemany(&self, sql: &str, parameters: Option<Py<PyList>>) -> PyResult<()> {
Err(PyErr::new::<NotSupportedError, _>(
"executemany() is not supported in this version",
))
}
#[pyo3(signature = (size=None))]
pub fn fetchmany(&self, size: Option<i64>) {
todo!()
pub fn fetchmany(&self, size: Option<i64>) -> PyResult<Option<Vec<PyObject>>> {
Err(PyErr::new::<NotSupportedError, _>(
"fetchmany() is not supported in this version",
))
}
}
@@ -228,12 +247,16 @@ impl Connection {
drop(self.conn.clone());
}
pub fn commit(&self) {
todo!()
pub fn commit(&self) -> PyResult<()> {
Err(PyErr::new::<NotSupportedError, _>(
"Transactions are not supported in this version",
))
}
pub fn rollback(&self) {
todo!()
pub fn rollback(&self) -> PyResult<()> {
Err(PyErr::new::<NotSupportedError, _>(
"Transactions are not supported in this version",
))
}
}

26
bindings/wasm/lib.rs
View File

@@ -5,7 +5,7 @@ use std::cell::RefCell;
use std::rc::Rc;
use std::sync::Arc;
use wasm_bindgen::prelude::*;
#[allow(dead_code)]
#[wasm_bindgen]
pub struct Database {
db: Arc<limbo_core::Database>,
@@ -64,7 +64,7 @@ pub struct Statement {
#[wasm_bindgen]
impl Statement {
fn new(inner: RefCell<limbo_core::Statement>, raw: bool) -> Self {
Statement { inner, raw }
Self { inner, raw }
}
#[wasm_bindgen]
@@ -75,7 +75,7 @@ impl Statement {
pub fn get(&self) -> JsValue {
match self.inner.borrow_mut().step() {
Ok(limbo_core::RowResult::Row(row)) => {
Ok(limbo_core::StepResult::Row(row)) => {
let row_array = js_sys::Array::new();
for value in row.values {
let value = to_js_value(value);
@@ -83,9 +83,10 @@ impl Statement {
}
JsValue::from(row_array)
}
Ok(limbo_core::RowResult::IO)
| Ok(limbo_core::RowResult::Done)
| Ok(limbo_core::RowResult::Interrupt) => JsValue::UNDEFINED,
Ok(limbo_core::StepResult::IO)
| Ok(limbo_core::StepResult::Done)
| Ok(limbo_core::StepResult::Interrupt)
| Ok(limbo_core::StepResult::Busy) => JsValue::UNDEFINED,
Err(e) => panic!("Error: {:?}", e),
}
}
@@ -94,7 +95,7 @@ impl Statement {
let array = js_sys::Array::new();
loop {
match self.inner.borrow_mut().step() {
Ok(limbo_core::RowResult::Row(row)) => {
Ok(limbo_core::StepResult::Row(row)) => {
let row_array = js_sys::Array::new();
for value in row.values {
let value = to_js_value(value);
@@ -102,9 +103,10 @@ impl Statement {
}
array.push(&row_array);
}
Ok(limbo_core::RowResult::IO) => {}
Ok(limbo_core::RowResult::Interrupt) => break,
Ok(limbo_core::RowResult::Done) => break,
Ok(limbo_core::StepResult::IO) => {}
Ok(limbo_core::StepResult::Interrupt) => break,
Ok(limbo_core::StepResult::Done) => break,
Ok(limbo_core::StepResult::Busy) => break,
Err(e) => panic!("Error: {:?}", e),
}
}
@@ -148,7 +150,7 @@ pub struct File {
#[allow(dead_code)]
impl File {
fn new(vfs: VFS, fd: i32) -> Self {
File { vfs, fd }
Self { vfs, fd }
}
}
@@ -261,7 +263,7 @@ pub struct DatabaseStorage {
impl DatabaseStorage {
pub fn new(file: Rc<dyn limbo_core::File>) -> Self {
DatabaseStorage { file }
Self { file }
}
}

87
cli/app.rs
View File

@@ -1,6 +1,6 @@
use crate::opcodes_dictionary::OPCODE_DESCRIPTIONS;
use cli_table::{Cell, Table};
use limbo_core::{Database, LimboError, RowResult, Value};
use limbo_core::{Database, LimboError, StepResult, Value};
use clap::{Parser, ValueEnum};
use std::{
@@ -160,7 +160,7 @@ impl From<&Opts> for Settings {
null_value: String::new(),
output_mode: opts.output_mode,
echo: false,
is_stdout: opts.output == "",
is_stdout: opts.output.is_empty(),
output_filename: opts.output.clone(),
db_file: opts
.database
@@ -192,7 +192,6 @@ impl std::fmt::Display for Settings {
}
impl Limbo {
#[allow(clippy::arc_with_non_send_sync)]
pub fn new() -> anyhow::Result<Self> {
let opts = Opts::parse();
let db_file = opts
@@ -229,13 +228,13 @@ impl Limbo {
app.writeln("Enter \".help\" for usage hints.")?;
app.display_in_memory()?;
}
return Ok(app);
Ok(app)
}
fn handle_first_input(&mut self, cmd: &str) {
if cmd.trim().starts_with('.') {
self.handle_dot_command(&cmd);
} else if let Err(e) = self.query(&cmd) {
self.handle_dot_command(cmd);
} else if let Err(e) = self.query(cmd) {
eprintln!("{}", e);
}
std::process::exit(0);
@@ -293,7 +292,7 @@ impl Limbo {
let db = Database::open_file(self.io.clone(), path)?;
self.conn = db.connect();
self.opts.db_file = ":memory:".to_string();
return Ok(());
Ok(())
}
path => {
let io: Arc<dyn limbo_core::IO> = Arc::new(limbo_core::PlatformIO::new()?);
@@ -301,7 +300,7 @@ impl Limbo {
let db = Database::open_file(self.io.clone(), path)?;
self.conn = db.connect();
self.opts.db_file = path.to_string();
return Ok(());
Ok(())
}
}
}
@@ -317,11 +316,9 @@ impl Limbo {
self.opts.is_stdout = false;
self.opts.output_mode = OutputMode::Raw;
self.opts.output_filename = path.to_string();
return Ok(());
}
Err(e) => {
return Err(e.to_string());
Ok(())
}
Err(e) => Err(e.to_string()),
}
}
@@ -333,7 +330,7 @@ impl Limbo {
fn set_mode(&mut self, mode: OutputMode) -> Result<(), String> {
if mode == OutputMode::Pretty && !self.opts.is_stdout {
return Err("pretty output can only be written to a tty".to_string());
Err("pretty output can only be written to a tty".to_string())
} else {
self.opts.output_mode = mode;
Ok(())
@@ -498,7 +495,7 @@ impl Limbo {
}
match rows.next_row() {
Ok(RowResult::Row(row)) => {
Ok(StepResult::Row(row)) => {
for (i, value) in row.values.iter().enumerate() {
if i > 0 {
let _ = self.writer.write(b"|");
@@ -518,11 +515,15 @@ impl Limbo {
}
let _ = self.writeln("");
}
Ok(RowResult::IO) => {
Ok(StepResult::IO) => {
self.io.run_once()?;
}
Ok(RowResult::Interrupt) => break,
Ok(RowResult::Done) => {
Ok(StepResult::Interrupt) => break,
Ok(StepResult::Done) => {
break;
}
Ok(StepResult::Busy) => {
let _ = self.writeln("database is busy");
break;
}
Err(err) => {
@@ -539,7 +540,7 @@ impl Limbo {
let mut table_rows: Vec<Vec<_>> = vec![];
loop {
match rows.next_row() {
Ok(RowResult::Row(row)) => {
Ok(StepResult::Row(row)) => {
table_rows.push(
row.values
.iter()
@@ -555,11 +556,15 @@ impl Limbo {
.collect(),
);
}
Ok(RowResult::IO) => {
Ok(StepResult::IO) => {
self.io.run_once()?;
}
Ok(RowResult::Interrupt) => break,
Ok(RowResult::Done) => break,
Ok(StepResult::Interrupt) => break,
Ok(StepResult::Done) => break,
Ok(StepResult::Busy) => {
let _ = self.writeln("database is busy");
break;
}
Err(err) => {
let _ = self.write_fmt(format_args!("{}", err));
break;
@@ -599,17 +604,21 @@ impl Limbo {
let mut found = false;
loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
if let Some(Value::Text(schema)) = row.values.first() {
let _ = self.write_fmt(format_args!("{};", schema));
found = true;
}
}
RowResult::IO => {
StepResult::IO => {
self.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => {
let _ = self.writeln("database is busy");
break;
}
}
}
if !found {
@@ -652,31 +661,33 @@ impl Limbo {
let mut tables = String::new();
loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
if let Some(Value::Text(table)) = row.values.first() {
tables.push_str(table);
tables.push(' ');
}
}
RowResult::IO => {
StepResult::IO => {
self.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => {
let _ = self.writeln("database is busy");
break;
}
}
}
if tables.len() > 0 {
if !tables.is_empty() {
let _ = self.writeln(tables.trim_end());
} else if let Some(pattern) = pattern {
let _ = self.write_fmt(format_args!(
"Error: Tables with pattern '{}' not found.",
pattern
));
} else {
if let Some(pattern) = pattern {
let _ = self.write_fmt(format_args!(
"Error: Tables with pattern '{}' not found.",
pattern
));
} else {
let _ = self.writeln("No tables found in the database.");
}
let _ = self.writeln("No tables found in the database.");
}
}
Ok(None) => {

2
cli/main.rs
View File

@@ -1,10 +1,10 @@
#![allow(clippy::arc_with_non_send_sync)]
mod app;
mod opcodes_dictionary;
use rustyline::{error::ReadlineError, DefaultEditor};
use std::sync::atomic::Ordering;
#[allow(clippy::arc_with_non_send_sync)]
fn main() -> anyhow::Result<()> {
env_logger::init();
let mut app = app::Limbo::new()?;

8
core/Cargo.toml
View File

@@ -14,13 +14,14 @@ name = "limbo_core"
path = "lib.rs"
[features]
default = ["fs", "json"]
default = ["fs", "json", "uuid"]
fs = []
json = [
"dep:jsonb",
"dep:pest",
"dep:pest_derive",
]
uuid = ["dep:uuid"]
[target.'cfg(target_os = "linux")'.dependencies]
io-uring = "0.6.1"
@@ -43,7 +44,8 @@ sieve-cache = "0.1.4"
sqlite3-parser = { path = "../vendored/sqlite3-parser" }
thiserror = "1.0.61"
getrandom = { version = "0.2.15", features = ["js"] }
regex = "1.10.5"
regex = "1.11.1"
regex-syntax = { version = "0.8.5", default-features = false, features = ["unicode"] }
chrono = "0.4.38"
julian_day_converter = "0.3.2"
jsonb = { version = "0.4.4", optional = true }
@@ -53,6 +55,8 @@ pest = { version = "2.0", optional = true }
pest_derive = { version = "2.0", optional = true }
rand = "0.8.5"
bumpalo = { version = "3.16.0", features = ["collections", "boxed"] }
limbo_macros = { path = "../macros" }
uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true }
[target.'cfg(not(target_family = "windows"))'.dev-dependencies]
pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] }

33
core/benches/benchmark.rs
View File

@@ -40,16 +40,19 @@ fn limbo_bench(criterion: &mut Criterion) {
b.iter(|| {
let mut rows = stmt.query().unwrap();
match rows.next_row().unwrap() {
limbo_core::RowResult::Row(row) => {
limbo_core::StepResult::Row(row) => {
assert_eq!(row.get::<i64>(0).unwrap(), 1);
}
limbo_core::RowResult::IO => {
limbo_core::StepResult::IO => {
io.run_once().unwrap();
}
limbo_core::RowResult::Interrupt => {
limbo_core::StepResult::Interrupt => {
unreachable!();
}
limbo_core::RowResult::Done => {
limbo_core::StepResult::Done => {
unreachable!();
}
limbo_core::StepResult::Busy => {
unreachable!();
}
}
@@ -65,18 +68,21 @@ fn limbo_bench(criterion: &mut Criterion) {
b.iter(|| {
let mut rows = stmt.query().unwrap();
match rows.next_row().unwrap() {
limbo_core::RowResult::Row(row) => {
limbo_core::StepResult::Row(row) => {
assert_eq!(row.get::<i64>(0).unwrap(), 1);
}
limbo_core::RowResult::IO => {
limbo_core::StepResult::IO => {
io.run_once().unwrap();
}
limbo_core::RowResult::Interrupt => {
limbo_core::StepResult::Interrupt => {
unreachable!();
}
limbo_core::RowResult::Done => {
limbo_core::StepResult::Done => {
unreachable!();
}
limbo_core::StepResult::Busy => {
unreachable!()
}
}
stmt.reset();
});
@@ -91,18 +97,21 @@ fn limbo_bench(criterion: &mut Criterion) {
b.iter(|| {
let mut rows = stmt.query().unwrap();
match rows.next_row().unwrap() {
limbo_core::RowResult::Row(row) => {
limbo_core::StepResult::Row(row) => {
assert_eq!(row.get::<i64>(0).unwrap(), 1);
}
limbo_core::RowResult::IO => {
limbo_core::StepResult::IO => {
io.run_once().unwrap();
}
limbo_core::RowResult::Interrupt => {
limbo_core::StepResult::Interrupt => {
unreachable!();
}
limbo_core::RowResult::Done => {
limbo_core::StepResult::Done => {
unreachable!();
}
limbo_core::StepResult::Busy => {
unreachable!()
}
}
stmt.reset();
});

32
core/ext/mod.rs Normal file
View File

@@ -0,0 +1,32 @@
#[cfg(feature = "uuid")]
mod uuid;
#[cfg(feature = "uuid")]
pub use uuid::{exec_ts_from_uuid7, exec_uuid, exec_uuidblob, exec_uuidstr, UuidFunc};
#[derive(Debug, Clone, PartialEq)]
pub enum ExtFunc {
#[cfg(feature = "uuid")]
Uuid(UuidFunc),
}
#[allow(unreachable_patterns)] // TODO: remove when more extension funcs added
impl std::fmt::Display for ExtFunc {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
#[cfg(feature = "uuid")]
Self::Uuid(uuidfn) => write!(f, "{}", uuidfn),
_ => write!(f, "unknown"),
}
}
}
#[allow(unreachable_patterns)]
impl ExtFunc {
pub fn resolve_function(name: &str, num_args: usize) -> Option<ExtFunc> {
match name {
#[cfg(feature = "uuid")]
name => UuidFunc::resolve_function(name, num_args),
_ => None,
}
}
}

341
core/ext/uuid.rs Normal file
View File

@@ -0,0 +1,341 @@
use super::ExtFunc;
use crate::{
types::{LimboText, OwnedValue},
LimboError,
};
use std::rc::Rc;
use uuid::{ContextV7, Timestamp, Uuid};
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum UuidFunc {
Uuid4Str,
Uuid4,
Uuid7,
Uuid7TS,
UuidStr,
UuidBlob,
}
impl UuidFunc {
pub fn resolve_function(name: &str, num_args: usize) -> Option<ExtFunc> {
match name {
"uuid4_str" => Some(ExtFunc::Uuid(Self::Uuid4Str)),
"uuid4" => Some(ExtFunc::Uuid(Self::Uuid4)),
"uuid7" if num_args < 2 => Some(ExtFunc::Uuid(Self::Uuid7)),
"uuid_str" if num_args == 1 => Some(ExtFunc::Uuid(Self::UuidStr)),
"uuid_blob" if num_args == 1 => Some(ExtFunc::Uuid(Self::UuidBlob)),
"uuid7_timestamp_ms" if num_args == 1 => Some(ExtFunc::Uuid(Self::Uuid7TS)),
// postgres_compatability
"gen_random_uuid" => Some(ExtFunc::Uuid(Self::Uuid4Str)),
_ => None,
}
}
}
impl std::fmt::Display for UuidFunc {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Uuid4Str => write!(f, "uuid4_str"),
Self::Uuid4 => write!(f, "uuid4"),
Self::Uuid7 => write!(f, "uuid7"),
Self::Uuid7TS => write!(f, "uuid7_timestamp_ms"),
Self::UuidStr => write!(f, "uuid_str"),
Self::UuidBlob => write!(f, "uuid_blob"),
}
}
}
pub fn exec_uuid(var: &UuidFunc, sec: Option<&OwnedValue>) -> crate::Result<OwnedValue> {
match var {
UuidFunc::Uuid4 => Ok(OwnedValue::Blob(Rc::new(
Uuid::new_v4().into_bytes().to_vec(),
))),
UuidFunc::Uuid4Str => Ok(OwnedValue::Text(LimboText::new(Rc::new(
Uuid::new_v4().to_string(),
)))),
UuidFunc::Uuid7 => {
let uuid = match sec {
Some(OwnedValue::Integer(ref seconds)) => {
let ctx = ContextV7::new();
if *seconds < 0 {
// not valid unix timestamp, error or null?
return Ok(OwnedValue::Null);
}
Uuid::new_v7(Timestamp::from_unix(ctx, *seconds as u64, 0))
}
_ => Uuid::now_v7(),
};
Ok(OwnedValue::Blob(Rc::new(uuid.into_bytes().to_vec())))
}
_ => unreachable!(),
}
}
pub fn exec_uuidstr(reg: &OwnedValue) -> crate::Result<OwnedValue> {
match reg {
OwnedValue::Blob(blob) => {
let uuid = Uuid::from_slice(blob).map_err(|e| LimboError::ParseError(e.to_string()))?;
Ok(OwnedValue::Text(LimboText::new(Rc::new(uuid.to_string()))))
}
OwnedValue::Text(ref val) => {
let uuid =
Uuid::parse_str(&val.value).map_err(|e| LimboError::ParseError(e.to_string()))?;
Ok(OwnedValue::Text(LimboText::new(Rc::new(uuid.to_string()))))
}
OwnedValue::Null => Ok(OwnedValue::Null),
_ => Err(LimboError::ParseError(
"Invalid argument type for UUID function".to_string(),
)),
}
}
pub fn exec_uuidblob(reg: &OwnedValue) -> crate::Result<OwnedValue> {
match reg {
OwnedValue::Text(val) => {
let uuid =
Uuid::parse_str(&val.value).map_err(|e| LimboError::ParseError(e.to_string()))?;
Ok(OwnedValue::Blob(Rc::new(uuid.as_bytes().to_vec())))
}
OwnedValue::Blob(blob) => {
let uuid = Uuid::from_slice(blob).map_err(|e| LimboError::ParseError(e.to_string()))?;
Ok(OwnedValue::Blob(Rc::new(uuid.as_bytes().to_vec())))
}
OwnedValue::Null => Ok(OwnedValue::Null),
_ => Err(LimboError::ParseError(
"Invalid argument type for UUID function".to_string(),
)),
}
}
pub fn exec_ts_from_uuid7(reg: &OwnedValue) -> OwnedValue {
let uuid = match reg {
OwnedValue::Blob(blob) => {
Uuid::from_slice(blob).map_err(|e| LimboError::ParseError(e.to_string()))
}
OwnedValue::Text(val) => {
Uuid::parse_str(&val.value).map_err(|e| LimboError::ParseError(e.to_string()))
}
_ => Err(LimboError::ParseError(
"Invalid argument type for UUID function".to_string(),
)),
};
match uuid {
Ok(uuid) => OwnedValue::Integer(uuid_to_unix(uuid.as_bytes()) as i64),
// display error? sqlean seems to set value to null
Err(_) => OwnedValue::Null,
}
}
#[inline(always)]
fn uuid_to_unix(uuid: &[u8; 16]) -> u64 {
((uuid[0] as u64) << 40)
| ((uuid[1] as u64) << 32)
| ((uuid[2] as u64) << 24)
| ((uuid[3] as u64) << 16)
| ((uuid[4] as u64) << 8)
| (uuid[5] as u64)
}
#[cfg(test)]
#[cfg(feature = "uuid")]
pub mod test {
use super::UuidFunc;
use crate::types::OwnedValue;
#[test]
fn test_exec_uuid_v4blob() {
use super::exec_uuid;
use uuid::Uuid;
let func = UuidFunc::Uuid4;
let owned_val = exec_uuid(&func, None);
match owned_val {
Ok(OwnedValue::Blob(blob)) => {
assert_eq!(blob.len(), 16);
let uuid = Uuid::from_slice(&blob);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 4);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
}
#[test]
fn test_exec_uuid_v4str() {
use super::{exec_uuid, UuidFunc};
use uuid::Uuid;
let func = UuidFunc::Uuid4Str;
let owned_val = exec_uuid(&func, None);
match owned_val {
Ok(OwnedValue::Text(v4str)) => {
assert_eq!(v4str.value.len(), 36);
let uuid = Uuid::parse_str(&v4str.value);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 4);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
}
#[test]
fn test_exec_uuid_v7_now() {
use super::{exec_uuid, UuidFunc};
use uuid::Uuid;
let func = UuidFunc::Uuid7;
let owned_val = exec_uuid(&func, None);
match owned_val {
Ok(OwnedValue::Blob(blob)) => {
assert_eq!(blob.len(), 16);
let uuid = Uuid::from_slice(&blob);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 7);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
}
#[test]
fn test_exec_uuid_v7_with_input() {
use super::{exec_uuid, UuidFunc};
use uuid::Uuid;
let func = UuidFunc::Uuid7;
let owned_val = exec_uuid(&func, Some(&OwnedValue::Integer(946702800)));
match owned_val {
Ok(OwnedValue::Blob(blob)) => {
assert_eq!(blob.len(), 16);
let uuid = Uuid::from_slice(&blob);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 7);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
}
#[test]
fn test_exec_uuid_v7_now_to_timestamp() {
use super::{exec_ts_from_uuid7, exec_uuid, UuidFunc};
use uuid::Uuid;
let func = UuidFunc::Uuid7;
let owned_val = exec_uuid(&func, None);
match owned_val {
Ok(OwnedValue::Blob(ref blob)) => {
assert_eq!(blob.len(), 16);
let uuid = Uuid::from_slice(blob);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 7);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
let result = exec_ts_from_uuid7(&owned_val.expect("uuid7"));
if let OwnedValue::Integer(ref ts) = result {
let unixnow = (std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs()
* 1000) as i64;
assert!(*ts >= unixnow - 1000);
}
}
#[test]
fn test_exec_uuid_v7_to_timestamp() {
use super::{exec_ts_from_uuid7, exec_uuid, UuidFunc};
use uuid::Uuid;
let func = UuidFunc::Uuid7;
let owned_val = exec_uuid(&func, Some(&OwnedValue::Integer(946702800)));
match owned_val {
Ok(OwnedValue::Blob(ref blob)) => {
assert_eq!(blob.len(), 16);
let uuid = Uuid::from_slice(blob);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 7);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
let result = exec_ts_from_uuid7(&owned_val.expect("uuid7"));
assert_eq!(result, OwnedValue::Integer(946702800 * 1000));
if let OwnedValue::Integer(ts) = result {
let time = chrono::DateTime::from_timestamp(ts / 1000, 0);
assert_eq!(
time.unwrap(),
"2000-01-01T05:00:00Z"
.parse::<chrono::DateTime<chrono::Utc>>()
.unwrap()
);
}
}
#[test]
fn test_exec_uuid_v4_str_to_blob() {
use super::{exec_uuid, exec_uuidblob, UuidFunc};
use uuid::Uuid;
let owned_val = exec_uuidblob(
&exec_uuid(&UuidFunc::Uuid4Str, None).expect("uuid v4 string to generate"),
);
match owned_val {
Ok(OwnedValue::Blob(blob)) => {
assert_eq!(blob.len(), 16);
let uuid = Uuid::from_slice(&blob);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 4);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
}
#[test]
fn test_exec_uuid_v7_str_to_blob() {
use super::{exec_uuid, exec_uuidblob, exec_uuidstr, UuidFunc};
use uuid::Uuid;
// convert a v7 blob to a string then back to a blob
let owned_val = exec_uuidblob(
&exec_uuidstr(&exec_uuid(&UuidFunc::Uuid7, None).expect("uuid v7 blob to generate"))
.expect("uuid v7 string to generate"),
);
match owned_val {
Ok(OwnedValue::Blob(blob)) => {
assert_eq!(blob.len(), 16);
let uuid = Uuid::from_slice(&blob);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 7);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
}
#[test]
fn test_exec_uuid_v4_blob_to_str() {
use super::{exec_uuid, exec_uuidstr, UuidFunc};
use uuid::Uuid;
// convert a v4 blob to a string
let owned_val =
exec_uuidstr(&exec_uuid(&UuidFunc::Uuid4, None).expect("uuid v7 blob to generate"));
match owned_val {
Ok(OwnedValue::Text(v4str)) => {
assert_eq!(v4str.value.len(), 36);
let uuid = Uuid::parse_str(&v4str.value);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 4);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
}
#[test]
fn test_exec_uuid_v7_blob_to_str() {
use super::{exec_uuid, exec_uuidstr};
use uuid::Uuid;
// convert a v7 blob to a string
let owned_val = exec_uuidstr(
&exec_uuid(&UuidFunc::Uuid7, Some(&OwnedValue::Integer(123456789)))
.expect("uuid v7 blob to generate"),
);
match owned_val {
Ok(OwnedValue::Text(v7str)) => {
assert_eq!(v7str.value.len(), 36);
let uuid = Uuid::parse_str(&v7str.value);
assert!(uuid.is_ok());
assert_eq!(uuid.unwrap().get_version_num(), 7);
}
_ => panic!("exec_uuid did not return a Blob variant"),
}
}
}

390
core/function.rs
View File

@@ -1,12 +1,13 @@
use crate::ext::ExtFunc;
use std::fmt;
use std::fmt::Display;
#[cfg(feature = "json")]
#[derive(Debug, Clone, PartialEq)]
pub enum JsonFunc {
Json,
JsonArray,
JsonExtract,
JsonArrayLength,
}
#[cfg(feature = "json")]
@@ -19,6 +20,7 @@ impl Display for JsonFunc {
JsonFunc::Json => "json".to_string(),
JsonFunc::JsonArray => "json_array".to_string(),
JsonFunc::JsonExtract => "json_extract".to_string(),
Self::JsonArrayLength => "json_array_length".to_string(),
}
)
}
@@ -39,14 +41,14 @@ pub enum AggFunc {
impl AggFunc {
pub fn to_string(&self) -> &str {
match self {
AggFunc::Avg => "avg",
AggFunc::Count => "count",
AggFunc::GroupConcat => "group_concat",
AggFunc::Max => "max",
AggFunc::Min => "min",
AggFunc::StringAgg => "string_agg",
AggFunc::Sum => "sum",
AggFunc::Total => "total",
Self::Avg => "avg",
Self::Count => "count",
Self::GroupConcat => "group_concat",
Self::Max => "max",
Self::Min => "min",
Self::StringAgg => "string_agg",
Self::Sum => "sum",
Self::Total => "total",
}
}
}
@@ -98,46 +100,46 @@ pub enum ScalarFunc {
impl Display for ScalarFunc {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let str = match self {
ScalarFunc::Cast => "cast".to_string(),
ScalarFunc::Char => "char".to_string(),
ScalarFunc::Coalesce => "coalesce".to_string(),
ScalarFunc::Concat => "concat".to_string(),
ScalarFunc::ConcatWs => "concat_ws".to_string(),
ScalarFunc::Glob => "glob".to_string(),
ScalarFunc::IfNull => "ifnull".to_string(),
ScalarFunc::Iif => "iif".to_string(),
ScalarFunc::Instr => "instr".to_string(),
ScalarFunc::Like => "like(2)".to_string(),
ScalarFunc::Abs => "abs".to_string(),
ScalarFunc::Upper => "upper".to_string(),
ScalarFunc::Lower => "lower".to_string(),
ScalarFunc::Random => "random".to_string(),
ScalarFunc::RandomBlob => "randomblob".to_string(),
ScalarFunc::Trim => "trim".to_string(),
ScalarFunc::LTrim => "ltrim".to_string(),
ScalarFunc::RTrim => "rtrim".to_string(),
ScalarFunc::Round => "round".to_string(),
ScalarFunc::Length => "length".to_string(),
ScalarFunc::OctetLength => "octet_length".to_string(),
ScalarFunc::Min => "min".to_string(),
ScalarFunc::Max => "max".to_string(),
ScalarFunc::Nullif => "nullif".to_string(),
ScalarFunc::Sign => "sign".to_string(),
ScalarFunc::Substr => "substr".to_string(),
ScalarFunc::Substring => "substring".to_string(),
ScalarFunc::Soundex => "soundex".to_string(),
ScalarFunc::Date => "date".to_string(),
ScalarFunc::Time => "time".to_string(),
ScalarFunc::Typeof => "typeof".to_string(),
ScalarFunc::Unicode => "unicode".to_string(),
ScalarFunc::Quote => "quote".to_string(),
ScalarFunc::SqliteVersion => "sqlite_version".to_string(),
ScalarFunc::UnixEpoch => "unixepoch".to_string(),
ScalarFunc::Hex => "hex".to_string(),
ScalarFunc::Unhex => "unhex".to_string(),
ScalarFunc::ZeroBlob => "zeroblob".to_string(),
ScalarFunc::LastInsertRowid => "last_insert_rowid".to_string(),
ScalarFunc::Replace => "replace".to_string(),
Self::Cast => "cast".to_string(),
Self::Char => "char".to_string(),
Self::Coalesce => "coalesce".to_string(),
Self::Concat => "concat".to_string(),
Self::ConcatWs => "concat_ws".to_string(),
Self::Glob => "glob".to_string(),
Self::IfNull => "ifnull".to_string(),
Self::Iif => "iif".to_string(),
Self::Instr => "instr".to_string(),
Self::Like => "like(2)".to_string(),
Self::Abs => "abs".to_string(),
Self::Upper => "upper".to_string(),
Self::Lower => "lower".to_string(),
Self::Random => "random".to_string(),
Self::RandomBlob => "randomblob".to_string(),
Self::Trim => "trim".to_string(),
Self::LTrim => "ltrim".to_string(),
Self::RTrim => "rtrim".to_string(),
Self::Round => "round".to_string(),
Self::Length => "length".to_string(),
Self::OctetLength => "octet_length".to_string(),
Self::Min => "min".to_string(),
Self::Max => "max".to_string(),
Self::Nullif => "nullif".to_string(),
Self::Sign => "sign".to_string(),
Self::Substr => "substr".to_string(),
Self::Substring => "substring".to_string(),
Self::Soundex => "soundex".to_string(),
Self::Date => "date".to_string(),
Self::Time => "time".to_string(),
Self::Typeof => "typeof".to_string(),
Self::Unicode => "unicode".to_string(),
Self::Quote => "quote".to_string(),
Self::SqliteVersion => "sqlite_version".to_string(),
Self::UnixEpoch => "unixepoch".to_string(),
Self::Hex => "hex".to_string(),
Self::Unhex => "unhex".to_string(),
Self::ZeroBlob => "zeroblob".to_string(),
Self::LastInsertRowid => "last_insert_rowid".to_string(),
Self::Replace => "replace".to_string(),
};
write!(f, "{}", str)
}
@@ -186,37 +188,34 @@ pub enum MathFuncArity {
impl MathFunc {
pub fn arity(&self) -> MathFuncArity {
match self {
MathFunc::Pi => MathFuncArity::Nullary,
Self::Pi => MathFuncArity::Nullary,
Self::Acos
| Self::Acosh
| Self::Asin
| Self::Asinh
| Self::Atan
| Self::Atanh
| Self::Ceil
| Self::Ceiling
| Self::Cos
| Self::Cosh
| Self::Degrees
| Self::Exp
| Self::Floor
| Self::Ln
| Self::Log10
| Self::Log2
| Self::Radians
| Self::Sin
| Self::Sinh
| Self::Sqrt
| Self::Tan
| Self::Tanh
| Self::Trunc => MathFuncArity::Unary,
MathFunc::Acos
| MathFunc::Acosh
| MathFunc::Asin
| MathFunc::Asinh
| MathFunc::Atan
| MathFunc::Atanh
| MathFunc::Ceil
| MathFunc::Ceiling
| MathFunc::Cos
| MathFunc::Cosh
| MathFunc::Degrees
| MathFunc::Exp
| MathFunc::Floor
| MathFunc::Ln
| MathFunc::Log10
| MathFunc::Log2
| MathFunc::Radians
| MathFunc::Sin
| MathFunc::Sinh
| MathFunc::Sqrt
| MathFunc::Tan
| MathFunc::Tanh
| MathFunc::Trunc => MathFuncArity::Unary,
Self::Atan2 | Self::Mod | Self::Pow | Self::Power => MathFuncArity::Binary,
MathFunc::Atan2 | MathFunc::Mod | MathFunc::Pow | MathFunc::Power => {
MathFuncArity::Binary
}
MathFunc::Log => MathFuncArity::UnaryOrBinary,
Self::Log => MathFuncArity::UnaryOrBinary,
}
}
}
@@ -224,57 +223,59 @@ impl MathFunc {
impl Display for MathFunc {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let str = match self {
MathFunc::Acos => "acos".to_string(),
MathFunc::Acosh => "acosh".to_string(),
MathFunc::Asin => "asin".to_string(),
MathFunc::Asinh => "asinh".to_string(),
MathFunc::Atan => "atan".to_string(),
MathFunc::Atan2 => "atan2".to_string(),
MathFunc::Atanh => "atanh".to_string(),
MathFunc::Ceil => "ceil".to_string(),
MathFunc::Ceiling => "ceiling".to_string(),
MathFunc::Cos => "cos".to_string(),
MathFunc::Cosh => "cosh".to_string(),
MathFunc::Degrees => "degrees".to_string(),
MathFunc::Exp => "exp".to_string(),
MathFunc::Floor => "floor".to_string(),
MathFunc::Ln => "ln".to_string(),
MathFunc::Log => "log".to_string(),
MathFunc::Log10 => "log10".to_string(),
MathFunc::Log2 => "log2".to_string(),
MathFunc::Mod => "mod".to_string(),
MathFunc::Pi => "pi".to_string(),
MathFunc::Pow => "pow".to_string(),
MathFunc::Power => "power".to_string(),
MathFunc::Radians => "radians".to_string(),
MathFunc::Sin => "sin".to_string(),
MathFunc::Sinh => "sinh".to_string(),
MathFunc::Sqrt => "sqrt".to_string(),
MathFunc::Tan => "tan".to_string(),
MathFunc::Tanh => "tanh".to_string(),
MathFunc::Trunc => "trunc".to_string(),
Self::Acos => "acos".to_string(),
Self::Acosh => "acosh".to_string(),
Self::Asin => "asin".to_string(),
Self::Asinh => "asinh".to_string(),
Self::Atan => "atan".to_string(),
Self::Atan2 => "atan2".to_string(),
Self::Atanh => "atanh".to_string(),
Self::Ceil => "ceil".to_string(),
Self::Ceiling => "ceiling".to_string(),
Self::Cos => "cos".to_string(),
Self::Cosh => "cosh".to_string(),
Self::Degrees => "degrees".to_string(),
Self::Exp => "exp".to_string(),
Self::Floor => "floor".to_string(),
Self::Ln => "ln".to_string(),
Self::Log => "log".to_string(),
Self::Log10 => "log10".to_string(),
Self::Log2 => "log2".to_string(),
Self::Mod => "mod".to_string(),
Self::Pi => "pi".to_string(),
Self::Pow => "pow".to_string(),
Self::Power => "power".to_string(),
Self::Radians => "radians".to_string(),
Self::Sin => "sin".to_string(),
Self::Sinh => "sinh".to_string(),
Self::Sqrt => "sqrt".to_string(),
Self::Tan => "tan".to_string(),
Self::Tanh => "tanh".to_string(),
Self::Trunc => "trunc".to_string(),
};
write!(f, "{}", str)
}
}
#[derive(Debug)]
#[derive(Debug, Clone, PartialEq)]
pub enum Func {
Agg(AggFunc),
Scalar(ScalarFunc),
Math(MathFunc),
#[cfg(feature = "json")]
Json(JsonFunc),
Extension(ExtFunc),
}
impl Display for Func {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Func::Agg(agg_func) => write!(f, "{}", agg_func.to_string()),
Func::Scalar(scalar_func) => write!(f, "{}", scalar_func),
Func::Math(math_func) => write!(f, "{}", math_func),
Self::Agg(agg_func) => write!(f, "{}", agg_func.to_string()),
Self::Scalar(scalar_func) => write!(f, "{}", scalar_func),
Self::Math(math_func) => write!(f, "{}", math_func),
#[cfg(feature = "json")]
Func::Json(json_func) => write!(f, "{}", json_func),
Self::Json(json_func) => write!(f, "{}", json_func),
Self::Extension(ext_func) => write!(f, "{}", ext_func),
}
}
}
@@ -286,91 +287,96 @@ pub struct FuncCtx {
}
impl Func {
pub fn resolve_function(name: &str, arg_count: usize) -> Result<Func, ()> {
pub fn resolve_function(name: &str, arg_count: usize) -> Result<Self, ()> {
match name {
"avg" => Ok(Func::Agg(AggFunc::Avg)),
"count" => Ok(Func::Agg(AggFunc::Count)),
"group_concat" => Ok(Func::Agg(AggFunc::GroupConcat)),
"max" if arg_count == 0 || arg_count == 1 => Ok(Func::Agg(AggFunc::Max)),
"max" if arg_count > 1 => Ok(Func::Scalar(ScalarFunc::Max)),
"min" if arg_count == 0 || arg_count == 1 => Ok(Func::Agg(AggFunc::Min)),
"min" if arg_count > 1 => Ok(Func::Scalar(ScalarFunc::Min)),
"nullif" if arg_count == 2 => Ok(Func::Scalar(ScalarFunc::Nullif)),
"string_agg" => Ok(Func::Agg(AggFunc::StringAgg)),
"sum" => Ok(Func::Agg(AggFunc::Sum)),
"total" => Ok(Func::Agg(AggFunc::Total)),
"char" => Ok(Func::Scalar(ScalarFunc::Char)),
"coalesce" => Ok(Func::Scalar(ScalarFunc::Coalesce)),
"concat" => Ok(Func::Scalar(ScalarFunc::Concat)),
"concat_ws" => Ok(Func::Scalar(ScalarFunc::ConcatWs)),
"glob" => Ok(Func::Scalar(ScalarFunc::Glob)),
"ifnull" => Ok(Func::Scalar(ScalarFunc::IfNull)),
"iif" => Ok(Func::Scalar(ScalarFunc::Iif)),
"instr" => Ok(Func::Scalar(ScalarFunc::Instr)),
"like" => Ok(Func::Scalar(ScalarFunc::Like)),
"abs" => Ok(Func::Scalar(ScalarFunc::Abs)),
"upper" => Ok(Func::Scalar(ScalarFunc::Upper)),
"lower" => Ok(Func::Scalar(ScalarFunc::Lower)),
"random" => Ok(Func::Scalar(ScalarFunc::Random)),
"randomblob" => Ok(Func::Scalar(ScalarFunc::RandomBlob)),
"trim" => Ok(Func::Scalar(ScalarFunc::Trim)),
"ltrim" => Ok(Func::Scalar(ScalarFunc::LTrim)),
"rtrim" => Ok(Func::Scalar(ScalarFunc::RTrim)),
"round" => Ok(Func::Scalar(ScalarFunc::Round)),
"length" => Ok(Func::Scalar(ScalarFunc::Length)),
"octet_length" => Ok(Func::Scalar(ScalarFunc::OctetLength)),
"sign" => Ok(Func::Scalar(ScalarFunc::Sign)),
"substr" => Ok(Func::Scalar(ScalarFunc::Substr)),
"substring" => Ok(Func::Scalar(ScalarFunc::Substring)),
"date" => Ok(Func::Scalar(ScalarFunc::Date)),
"time" => Ok(Func::Scalar(ScalarFunc::Time)),
"typeof" => Ok(Func::Scalar(ScalarFunc::Typeof)),
"last_insert_rowid" => Ok(Func::Scalar(ScalarFunc::LastInsertRowid)),
"unicode" => Ok(Func::Scalar(ScalarFunc::Unicode)),
"quote" => Ok(Func::Scalar(ScalarFunc::Quote)),
"sqlite_version" => Ok(Func::Scalar(ScalarFunc::SqliteVersion)),
"replace" => Ok(Func::Scalar(ScalarFunc::Replace)),
"avg" => Ok(Self::Agg(AggFunc::Avg)),
"count" => Ok(Self::Agg(AggFunc::Count)),
"group_concat" => Ok(Self::Agg(AggFunc::GroupConcat)),
"max" if arg_count == 0 || arg_count == 1 => Ok(Self::Agg(AggFunc::Max)),
"max" if arg_count > 1 => Ok(Self::Scalar(ScalarFunc::Max)),
"min" if arg_count == 0 || arg_count == 1 => Ok(Self::Agg(AggFunc::Min)),
"min" if arg_count > 1 => Ok(Self::Scalar(ScalarFunc::Min)),
"nullif" if arg_count == 2 => Ok(Self::Scalar(ScalarFunc::Nullif)),
"string_agg" => Ok(Self::Agg(AggFunc::StringAgg)),
"sum" => Ok(Self::Agg(AggFunc::Sum)),
"total" => Ok(Self::Agg(AggFunc::Total)),
"char" => Ok(Self::Scalar(ScalarFunc::Char)),
"coalesce" => Ok(Self::Scalar(ScalarFunc::Coalesce)),
"concat" => Ok(Self::Scalar(ScalarFunc::Concat)),
"concat_ws" => Ok(Self::Scalar(ScalarFunc::ConcatWs)),
"glob" => Ok(Self::Scalar(ScalarFunc::Glob)),
"ifnull" => Ok(Self::Scalar(ScalarFunc::IfNull)),
"iif" => Ok(Self::Scalar(ScalarFunc::Iif)),
"instr" => Ok(Self::Scalar(ScalarFunc::Instr)),
"like" => Ok(Self::Scalar(ScalarFunc::Like)),
"abs" => Ok(Self::Scalar(ScalarFunc::Abs)),
"upper" => Ok(Self::Scalar(ScalarFunc::Upper)),
"lower" => Ok(Self::Scalar(ScalarFunc::Lower)),
"random" => Ok(Self::Scalar(ScalarFunc::Random)),
"randomblob" => Ok(Self::Scalar(ScalarFunc::RandomBlob)),
"trim" => Ok(Self::Scalar(ScalarFunc::Trim)),
"ltrim" => Ok(Self::Scalar(ScalarFunc::LTrim)),
"rtrim" => Ok(Self::Scalar(ScalarFunc::RTrim)),
"round" => Ok(Self::Scalar(ScalarFunc::Round)),
"length" => Ok(Self::Scalar(ScalarFunc::Length)),
"octet_length" => Ok(Self::Scalar(ScalarFunc::OctetLength)),
"sign" => Ok(Self::Scalar(ScalarFunc::Sign)),
"substr" => Ok(Self::Scalar(ScalarFunc::Substr)),
"substring" => Ok(Self::Scalar(ScalarFunc::Substring)),
"date" => Ok(Self::Scalar(ScalarFunc::Date)),
"time" => Ok(Self::Scalar(ScalarFunc::Time)),
"typeof" => Ok(Self::Scalar(ScalarFunc::Typeof)),
"last_insert_rowid" => Ok(Self::Scalar(ScalarFunc::LastInsertRowid)),
"unicode" => Ok(Self::Scalar(ScalarFunc::Unicode)),
"quote" => Ok(Self::Scalar(ScalarFunc::Quote)),
"sqlite_version" => Ok(Self::Scalar(ScalarFunc::SqliteVersion)),
"replace" => Ok(Self::Scalar(ScalarFunc::Replace)),
#[cfg(feature = "json")]
"json" => Ok(Func::Json(JsonFunc::Json)),
"json" => Ok(Self::Json(JsonFunc::Json)),
#[cfg(feature = "json")]
"json_array" => Ok(Func::Json(JsonFunc::JsonArray)),
"json_array_length" => Ok(Self::Json(JsonFunc::JsonArrayLength)),
#[cfg(feature = "json")]
"json_array" => Ok(Self::Json(JsonFunc::JsonArray)),
#[cfg(feature = "json")]
"json_extract" => Ok(Func::Json(JsonFunc::JsonExtract)),
"unixepoch" => Ok(Func::Scalar(ScalarFunc::UnixEpoch)),
"hex" => Ok(Func::Scalar(ScalarFunc::Hex)),
"unhex" => Ok(Func::Scalar(ScalarFunc::Unhex)),
"zeroblob" => Ok(Func::Scalar(ScalarFunc::ZeroBlob)),
"soundex" => Ok(Func::Scalar(ScalarFunc::Soundex)),
"acos" => Ok(Func::Math(MathFunc::Acos)),
"acosh" => Ok(Func::Math(MathFunc::Acosh)),
"asin" => Ok(Func::Math(MathFunc::Asin)),
"asinh" => Ok(Func::Math(MathFunc::Asinh)),
"atan" => Ok(Func::Math(MathFunc::Atan)),
"atan2" => Ok(Func::Math(MathFunc::Atan2)),
"atanh" => Ok(Func::Math(MathFunc::Atanh)),
"ceil" => Ok(Func::Math(MathFunc::Ceil)),
"ceiling" => Ok(Func::Math(MathFunc::Ceiling)),
"cos" => Ok(Func::Math(MathFunc::Cos)),
"cosh" => Ok(Func::Math(MathFunc::Cosh)),
"degrees" => Ok(Func::Math(MathFunc::Degrees)),
"exp" => Ok(Func::Math(MathFunc::Exp)),
"floor" => Ok(Func::Math(MathFunc::Floor)),
"ln" => Ok(Func::Math(MathFunc::Ln)),
"log" => Ok(Func::Math(MathFunc::Log)),
"log10" => Ok(Func::Math(MathFunc::Log10)),
"log2" => Ok(Func::Math(MathFunc::Log2)),
"mod" => Ok(Func::Math(MathFunc::Mod)),
"pi" => Ok(Func::Math(MathFunc::Pi)),
"pow" => Ok(Func::Math(MathFunc::Pow)),
"power" => Ok(Func::Math(MathFunc::Power)),
"radians" => Ok(Func::Math(MathFunc::Radians)),
"sin" => Ok(Func::Math(MathFunc::Sin)),
"sinh" => Ok(Func::Math(MathFunc::Sinh)),
"sqrt" => Ok(Func::Math(MathFunc::Sqrt)),
"tan" => Ok(Func::Math(MathFunc::Tan)),
"tanh" => Ok(Func::Math(MathFunc::Tanh)),
"trunc" => Ok(Func::Math(MathFunc::Trunc)),
_ => Err(()),
"unixepoch" => Ok(Self::Scalar(ScalarFunc::UnixEpoch)),
"hex" => Ok(Self::Scalar(ScalarFunc::Hex)),
"unhex" => Ok(Self::Scalar(ScalarFunc::Unhex)),
"zeroblob" => Ok(Self::Scalar(ScalarFunc::ZeroBlob)),
"soundex" => Ok(Self::Scalar(ScalarFunc::Soundex)),
"acos" => Ok(Self::Math(MathFunc::Acos)),
"acosh" => Ok(Self::Math(MathFunc::Acosh)),
"asin" => Ok(Self::Math(MathFunc::Asin)),
"asinh" => Ok(Self::Math(MathFunc::Asinh)),
"atan" => Ok(Self::Math(MathFunc::Atan)),
"atan2" => Ok(Self::Math(MathFunc::Atan2)),
"atanh" => Ok(Self::Math(MathFunc::Atanh)),
"ceil" => Ok(Self::Math(MathFunc::Ceil)),
"ceiling" => Ok(Self::Math(MathFunc::Ceiling)),
"cos" => Ok(Self::Math(MathFunc::Cos)),
"cosh" => Ok(Self::Math(MathFunc::Cosh)),
"degrees" => Ok(Self::Math(MathFunc::Degrees)),
"exp" => Ok(Self::Math(MathFunc::Exp)),
"floor" => Ok(Self::Math(MathFunc::Floor)),
"ln" => Ok(Self::Math(MathFunc::Ln)),
"log" => Ok(Self::Math(MathFunc::Log)),
"log10" => Ok(Self::Math(MathFunc::Log10)),
"log2" => Ok(Self::Math(MathFunc::Log2)),
"mod" => Ok(Self::Math(MathFunc::Mod)),
"pi" => Ok(Self::Math(MathFunc::Pi)),
"pow" => Ok(Self::Math(MathFunc::Pow)),
"power" => Ok(Self::Math(MathFunc::Power)),
"radians" => Ok(Self::Math(MathFunc::Radians)),
"sin" => Ok(Self::Math(MathFunc::Sin)),
"sinh" => Ok(Self::Math(MathFunc::Sinh)),
"sqrt" => Ok(Self::Math(MathFunc::Sqrt)),
"tan" => Ok(Self::Math(MathFunc::Tan)),
"tanh" => Ok(Self::Math(MathFunc::Tanh)),
"trunc" => Ok(Self::Math(MathFunc::Trunc)),
_ => match ExtFunc::resolve_function(name, arg_count) {
Some(ext_func) => Ok(Self::Extension(ext_func)),
None => Err(()),
},
}
}
}

6
core/io/generic.rs
View File

@@ -15,7 +15,11 @@ impl GenericIO {
impl IO for GenericIO {
fn open_file(&self, path: &str, flags: OpenFlags, _direct: bool) -> Result<Rc<dyn File>> {
trace!("open_file(path = {})", path);
let file = std::fs::File::open(path)?;
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(matches!(flags, OpenFlags::Create))
.open(path)?;
Ok(Rc::new(GenericFile {
file: RefCell::new(file),
}))

6
core/io/mod.rs
View File

@@ -51,9 +51,9 @@ pub struct ReadCompletion {
impl Completion {
pub fn complete(&self, result: i32) {
match self {
Completion::Read(r) => r.complete(),
Completion::Write(w) => w.complete(result),
Completion::Sync(s) => s.complete(result), // fix
Self::Read(r) => r.complete(),
Self::Write(w) => w.complete(result),
Self::Sync(s) => s.complete(result), // fix
}
}
}

2
core/json/de.rs
View File

@@ -37,7 +37,7 @@ impl<'de> Deserializer<'de> {
}
fn from_pair(pair: Pair<'de, Rule>) -> Self {
Deserializer { pair: Some(pair) }
Self { pair: Some(pair) }
}
}

12
core/json/error.rs
View File

@@ -43,7 +43,7 @@ impl From<pest::error::Error<Rule>> for Error {
pest::error::LineColLocation::Pos((l, c)) => (l, c),
pest::error::LineColLocation::Span((l, c), (_, _)) => (l, c),
};
Error::Message {
Self::Message {
msg: err.to_string(),
location: Some(Location { line, column }),
}
@@ -52,7 +52,7 @@ impl From<pest::error::Error<Rule>> for Error {
impl From<std::io::Error> for Error {
fn from(err: std::io::Error) -> Self {
Error::Message {
Self::Message {
msg: err.to_string(),
location: None,
}
@@ -61,7 +61,7 @@ impl From<std::io::Error> for Error {
impl From<std::str::Utf8Error> for Error {
fn from(err: std::str::Utf8Error) -> Self {
Error::Message {
Self::Message {
msg: err.to_string(),
location: None,
}
@@ -70,7 +70,7 @@ impl From<std::str::Utf8Error> for Error {
impl ser::Error for Error {
fn custom<T: Display>(msg: T) -> Self {
Error::Message {
Self::Message {
msg: msg.to_string(),
location: None,
}
@@ -79,7 +79,7 @@ impl ser::Error for Error {
impl de::Error for Error {
fn custom<T: Display>(msg: T) -> Self {
Error::Message {
Self::Message {
msg: msg.to_string(),
location: None,
}
@@ -89,7 +89,7 @@ impl de::Error for Error {
impl Display for Error {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Error::Message { ref msg, .. } => write!(formatter, "{}", msg),
Self::Message { ref msg, .. } => write!(formatter, "{}", msg),
}
}
}

143
core/json/mod.rs
View File

@@ -117,6 +117,32 @@ pub fn json_array(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
Ok(OwnedValue::Text(LimboText::json(Rc::new(s))))
}
pub fn json_array_length(
json_value: &OwnedValue,
json_path: Option<&OwnedValue>,
) -> crate::Result<OwnedValue> {
let path = match json_path {
Some(OwnedValue::Text(t)) => Some(t.value.to_string()),
Some(OwnedValue::Integer(i)) => Some(i.to_string()),
Some(OwnedValue::Float(f)) => Some(f.to_string()),
_ => None::<String>,
};
let json = get_json_value(json_value)?;
let arr_val = if let Some(path) = path {
&json_extract_single(&json, path.as_str())?
} else {
&json
};
match arr_val {
Val::Array(val) => (Ok(OwnedValue::Integer(val.len() as i64))),
Val::Null => Ok(OwnedValue::Null),
_ => Ok(OwnedValue::Integer(0)),
}
}
pub fn json_extract(value: &OwnedValue, paths: &[OwnedValue]) -> crate::Result<OwnedValue> {
if let OwnedValue::Null = value {
return Ok(OwnedValue::Null);
@@ -389,6 +415,123 @@ mod tests {
}
}
#[test]
fn test_json_array_length() {
let input = OwnedValue::build_text(Rc::new("[1,2,3,4]".to_string()));
let result = json_array_length(&input, None).unwrap();
if let OwnedValue::Integer(res) = result {
assert_eq!(res, 4);
} else {
panic!("Expected OwnedValue::Integer");
}
}
#[test]
fn test_json_array_length_empty() {
let input = OwnedValue::build_text(Rc::new("[]".to_string()));
let result = json_array_length(&input, None).unwrap();
if let OwnedValue::Integer(res) = result {
assert_eq!(res, 0);
} else {
panic!("Expected OwnedValue::Integer");
}
}
#[test]
fn test_json_array_length_root() {
let input = OwnedValue::build_text(Rc::new("[1,2,3,4]".to_string()));
let result = json_array_length(
&input,
Some(&OwnedValue::build_text(Rc::new("$".to_string()))),
)
.unwrap();
if let OwnedValue::Integer(res) = result {
assert_eq!(res, 4);
} else {
panic!("Expected OwnedValue::Integer");
}
}
#[test]
fn test_json_array_length_not_array() {
let input = OwnedValue::build_text(Rc::new("{one: [1,2,3,4]}".to_string()));
let result = json_array_length(&input, None).unwrap();
if let OwnedValue::Integer(res) = result {
assert_eq!(res, 0);
} else {
panic!("Expected OwnedValue::Integer");
}
}
#[test]
fn test_json_array_length_via_prop() {
let input = OwnedValue::build_text(Rc::new("{one: [1,2,3,4]}".to_string()));
let result = json_array_length(
&input,
Some(&OwnedValue::build_text(Rc::new("$.one".to_string()))),
)
.unwrap();
if let OwnedValue::Integer(res) = result {
assert_eq!(res, 4);
} else {
panic!("Expected OwnedValue::Integer");
}
}
#[test]
fn test_json_array_length_via_index() {
let input = OwnedValue::build_text(Rc::new("[[1,2,3,4]]".to_string()));
let result = json_array_length(
&input,
Some(&OwnedValue::build_text(Rc::new("$[0]".to_string()))),
)
.unwrap();
if let OwnedValue::Integer(res) = result {
assert_eq!(res, 4);
} else {
panic!("Expected OwnedValue::Integer");
}
}
#[test]
fn test_json_array_length_via_index_not_array() {
let input = OwnedValue::build_text(Rc::new("[1,2,3,4]".to_string()));
let result = json_array_length(
&input,
Some(&OwnedValue::build_text(Rc::new("$[2]".to_string()))),
)
.unwrap();
if let OwnedValue::Integer(res) = result {
assert_eq!(res, 0);
} else {
panic!("Expected OwnedValue::Integer");
}
}
#[test]
fn test_json_array_length_via_index_bad_prop() {
let input = OwnedValue::build_text(Rc::new("{one: [1,2,3,4]}".to_string()));
let result = json_array_length(
&input,
Some(&OwnedValue::build_text(Rc::new("$.two".to_string()))),
)
.unwrap();
assert_eq!(OwnedValue::Null, result);
}
#[test]
fn test_json_array_length_simple_json_subtype() {
let input = OwnedValue::build_text(Rc::new("[1,2,3]".to_string()));
let wrapped = get_json(&input).unwrap();
let result = json_array_length(&wrapped, None).unwrap();
if let OwnedValue::Integer(res) = result {
assert_eq!(res, 3);
} else {
panic!("Expected OwnedValue::Integer");
}
}
#[test]
fn test_json_extract_missing_path() {
let result = json_extract(

46
core/lib.rs
View File

@@ -1,9 +1,11 @@
mod error;
mod ext;
mod function;
mod io;
#[cfg(feature = "json")]
mod json;
mod pseudo;
mod result;
mod schema;
mod storage;
mod translate;
@@ -34,12 +36,12 @@ pub use storage::wal::WalFile;
pub use storage::wal::WalFileShared;
use util::parse_schema_rows;
use translate::optimizer::optimize_plan;
use translate::planner::prepare_select_plan;
pub use error::LimboError;
pub type Result<T> = std::result::Result<T, error::LimboError>;
use crate::translate::optimizer::optimize_plan;
pub use io::OpenFlags;
#[cfg(feature = "fs")]
pub use io::PlatformIO;
@@ -65,11 +67,10 @@ pub struct Database {
pager: Rc<Pager>,
schema: Rc<RefCell<Schema>>,
header: Rc<RefCell<DatabaseHeader>>,
transaction_state: RefCell<TransactionState>,
// Shared structures of a Database are the parts that are common to multiple threads that might
// create DB connections.
shared_page_cache: Arc<RwLock<DumbLruPageCache>>,
shared_wal: Arc<RwLock<WalFileShared>>,
_shared_page_cache: Arc<RwLock<DumbLruPageCache>>,
_shared_wal: Arc<RwLock<WalFileShared>>,
}
impl Database {
@@ -95,6 +96,7 @@ impl Database {
Self::open(io, page_io, wal, wal_shared, buffer_pool)
}
#[allow(clippy::arc_with_non_send_sync)]
pub fn open(
io: Arc<dyn IO>,
page_io: Rc<dyn DatabaseStorage>,
@@ -108,13 +110,13 @@ impl Database {
let version = db_header.borrow().version_number;
version.to_string()
});
let shared_page_cache = Arc::new(RwLock::new(DumbLruPageCache::new(10)));
let _shared_page_cache = Arc::new(RwLock::new(DumbLruPageCache::new(10)));
let pager = Rc::new(Pager::finish_open(
db_header.clone(),
page_io,
wal,
io.clone(),
shared_page_cache.clone(),
_shared_page_cache.clone(),
buffer_pool,
)?);
let bootstrap_schema = Rc::new(RefCell::new(Schema::new()));
@@ -122,7 +124,8 @@ impl Database {
pager: pager.clone(),
schema: bootstrap_schema.clone(),
header: db_header.clone(),
db: Weak::new(),
transaction_state: RefCell::new(TransactionState::None),
_db: Weak::new(),
last_insert_rowid: Cell::new(0),
});
let mut schema = Schema::new();
@@ -134,9 +137,8 @@ impl Database {
pager,
schema,
header,
transaction_state: RefCell::new(TransactionState::None),
shared_page_cache,
shared_wal,
_shared_page_cache,
_shared_wal: shared_wal,
}))
}
@@ -146,7 +148,8 @@ impl Database {
schema: self.schema.clone(),
header: self.header.clone(),
last_insert_rowid: Cell::new(0),
db: Arc::downgrade(self),
_db: Arc::downgrade(self),
transaction_state: RefCell::new(TransactionState::None),
})
}
}
@@ -204,7 +207,8 @@ pub struct Connection {
pager: Rc<Pager>,
schema: Rc<RefCell<Schema>>,
header: Rc<RefCell<DatabaseHeader>>,
db: Weak<Database>, // backpointer to the database holding this connection
_db: Weak<Database>, // backpointer to the database holding this connection
transaction_state: RefCell<TransactionState>,
last_insert_rowid: Cell<u64>,
}
@@ -266,7 +270,7 @@ impl Connection {
Cmd::ExplainQueryPlan(stmt) => {
match stmt {
ast::Stmt::Select(select) => {
let plan = prepare_select_plan(&*self.schema.borrow(), select)?;
let plan = prepare_select_plan(&self.schema.borrow(), select)?;
let plan = optimize_plan(plan)?;
println!("{}", plan);
}
@@ -371,13 +375,14 @@ impl Statement {
self.state.interrupt();
}
pub fn step(&mut self) -> Result<RowResult<'_>> {
pub fn step(&mut self) -> Result<StepResult<'_>> {
let result = self.program.step(&mut self.state, self.pager.clone())?;
match result {
vdbe::StepResult::Row(row) => Ok(RowResult::Row(Row { values: row.values })),
vdbe::StepResult::IO => Ok(RowResult::IO),
vdbe::StepResult::Done => Ok(RowResult::Done),
vdbe::StepResult::Interrupt => Ok(RowResult::Interrupt),
vdbe::StepResult::Row(row) => Ok(StepResult::Row(Row { values: row.values })),
vdbe::StepResult::IO => Ok(StepResult::IO),
vdbe::StepResult::Done => Ok(StepResult::Done),
vdbe::StepResult::Interrupt => Ok(StepResult::Interrupt),
vdbe::StepResult::Busy => Ok(StepResult::Busy),
}
}
@@ -389,11 +394,12 @@ impl Statement {
pub fn reset(&self) {}
}
pub enum RowResult<'a> {
pub enum StepResult<'a> {
Row(Row<'a>),
IO,
Done,
Interrupt,
Busy,
}
pub struct Row<'a> {
@@ -416,7 +422,7 @@ impl Rows {
Self { stmt }
}
pub fn next_row(&mut self) -> Result<RowResult<'_>> {
pub fn next_row(&mut self) -> Result<StepResult<'_>> {
self.stmt.step()
}
}

4
core/pseudo.rs
View File

@@ -79,6 +79,10 @@ impl Cursor for PseudoCursor {
Ok(CursorResult::Ok(()))
}
fn delete(&mut self) -> Result<CursorResult<()>> {
unimplemented!()
}
fn get_null_flag(&self) -> bool {
false
}

6
core/result.rs Normal file
View File

@@ -0,0 +1,6 @@
/// Common results that different functions can return in limbo.
pub enum LimboResult {
/// Couldn't acquire a lock
Busy,
Ok,
}

62
core/schema.rs
View File

@@ -57,39 +57,39 @@ impl Table {
pub fn get_rowid_alias_column(&self) -> Option<(usize, &Column)> {
match self {
Table::BTree(table) => table.get_rowid_alias_column(),
Table::Index(_) => None,
Table::Pseudo(_) => None,
Self::BTree(table) => table.get_rowid_alias_column(),
Self::Index(_) => None,
Self::Pseudo(_) => None,
}
}
pub fn column_is_rowid_alias(&self, col: &Column) -> bool {
match self {
Table::BTree(table) => table.column_is_rowid_alias(col),
Table::Index(_) => false,
Table::Pseudo(_) => false,
Self::Index(_) => false,
Self::Pseudo(_) => false,
}
}
pub fn get_name(&self) -> &str {
match self {
Table::BTree(table) => &table.name,
Table::Index(index) => &index.name,
Table::Pseudo(_) => "",
Self::BTree(table) => &table.name,
Self::Index(index) => &index.name,
Self::Pseudo(_) => "",
}
}
pub fn column_index_to_name(&self, index: usize) -> Option<&str> {
match self {
Table::BTree(table) => match table.columns.get(index) {
Self::BTree(table) => match table.columns.get(index) {
Some(column) => Some(&column.name),
None => None,
},
Table::Index(i) => match i.columns.get(index) {
Self::Index(i) => match i.columns.get(index) {
Some(column) => Some(&column.name),
None => None,
},
Table::Pseudo(table) => match table.columns.get(index) {
Self::Pseudo(table) => match table.columns.get(index) {
Some(_) => None,
None => None,
},
@@ -98,33 +98,33 @@ impl Table {
pub fn get_column(&self, name: &str) -> Option<(usize, &Column)> {
match self {
Table::BTree(table) => table.get_column(name),
Table::Index(_) => unimplemented!(),
Table::Pseudo(table) => table.get_column(name),
Self::BTree(table) => table.get_column(name),
Self::Index(_) => unimplemented!(),
Self::Pseudo(table) => table.get_column(name),
}
}
pub fn get_column_at(&self, index: usize) -> &Column {
match self {
Table::BTree(table) => table.columns.get(index).unwrap(),
Table::Index(_) => unimplemented!(),
Table::Pseudo(table) => table.columns.get(index).unwrap(),
Self::BTree(table) => table.columns.get(index).unwrap(),
Self::Index(_) => unimplemented!(),
Self::Pseudo(table) => table.columns.get(index).unwrap(),
}
}
pub fn columns(&self) -> &Vec<Column> {
match self {
Table::BTree(table) => &table.columns,
Table::Index(_) => unimplemented!(),
Table::Pseudo(table) => &table.columns,
Self::BTree(table) => &table.columns,
Self::Index(_) => unimplemented!(),
Self::Pseudo(table) => &table.columns,
}
}
pub fn has_rowid(&self) -> bool {
match self {
Table::BTree(table) => table.has_rowid,
Table::Index(_) => unimplemented!(),
Table::Pseudo(_) => unimplemented!(),
Self::BTree(table) => table.has_rowid,
Self::Index(_) => unimplemented!(),
Self::Pseudo(_) => unimplemented!(),
}
}
}
@@ -132,8 +132,8 @@ impl Table {
impl PartialEq for Table {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Table::BTree(a), Table::BTree(b)) => Rc::ptr_eq(a, b),
(Table::Pseudo(a), Table::Pseudo(b)) => Rc::ptr_eq(a, b),
(Self::BTree(a), Self::BTree(b)) => Rc::ptr_eq(a, b),
(Self::Pseudo(a), Self::Pseudo(b)) => Rc::ptr_eq(a, b),
_ => false,
}
}
@@ -386,12 +386,12 @@ pub enum Type {
impl fmt::Display for Type {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {
Type::Null => "NULL",
Type::Text => "TEXT",
Type::Numeric => "NUMERIC",
Type::Integer => "INTEGER",
Type::Real => "REAL",
Type::Blob => "BLOB",
Self::Null => "NULL",
Self::Text => "TEXT",
Self::Numeric => "NUMERIC",
Self::Integer => "INTEGER",
Self::Real => "REAL",
Self::Blob => "BLOB",
};
write!(f, "{}", s)
}

472
core/storage/btree.rs
View File

@@ -20,22 +20,37 @@ use super::sqlite3_ondisk::{
/*
These are offsets of fields in the header of a b-tree page.
*/
const BTREE_HEADER_OFFSET_TYPE: usize = 0; /* type of btree page -> u8 */
const BTREE_HEADER_OFFSET_FREEBLOCK: usize = 1; /* pointer to first freeblock -> u16 */
const BTREE_HEADER_OFFSET_CELL_COUNT: usize = 3; /* number of cells in the page -> u16 */
const BTREE_HEADER_OFFSET_CELL_CONTENT: usize = 5; /* pointer to first byte of cell allocated content from top -> u16 */
const BTREE_HEADER_OFFSET_FRAGMENTED: usize = 7; /* number of fragmented bytes -> u8 */
const BTREE_HEADER_OFFSET_RIGHTMOST: usize = 8; /* if internalnode, pointer right most pointer (saved separately from cells) -> u32 */
/*
** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than
** this will be declared corrupt. This value is calculated based on a
** maximum database size of 2^31 pages a minimum fanout of 2 for a
** root-node and 3 for all other internal nodes.
**
** If a tree that appears to be taller than this is encountered, it is
** assumed that the database is corrupt.
*/
/// type of btree page -> u8
const PAGE_HEADER_OFFSET_PAGE_TYPE: usize = 0;
/// pointer to first freeblock -> u16
/// The second field of the b-tree page header is the offset of the first freeblock, or zero if there are no freeblocks on the page.
/// A freeblock is a structure used to identify unallocated space within a b-tree page.
/// Freeblocks are organized as a chain.
///
/// To be clear, freeblocks do not mean the regular unallocated free space to the left of the cell content area pointer, but instead
/// blocks of at least 4 bytes WITHIN the cell content area that are not in use due to e.g. deletions.
const PAGE_HEADER_OFFSET_FIRST_FREEBLOCK: usize = 1;
/// number of cells in the page -> u16
const PAGE_HEADER_OFFSET_CELL_COUNT: usize = 3;
/// pointer to first byte of cell allocated content from top -> u16
/// SQLite strives to place cells as far toward the end of the b-tree page as it can,
/// in order to leave space for future growth of the cell pointer array.
/// = the cell content area pointer moves leftward as cells are added to the page
const PAGE_HEADER_OFFSET_CELL_CONTENT_AREA: usize = 5;
/// number of fragmented bytes -> u8
/// Fragments are isolated groups of 1, 2, or 3 unused bytes within the cell content area.
const PAGE_HEADER_OFFSET_FRAGMENTED_BYTES_COUNT: usize = 7;
/// if internalnode, pointer right most pointer (saved separately from cells) -> u32
const PAGE_HEADER_OFFSET_RIGHTMOST_PTR: usize = 8;
/// Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than
/// this will be declared corrupt. This value is calculated based on a
/// maximum database size of 2^31 pages a minimum fanout of 2 for a
/// root-node and 3 for all other internal nodes.
///
/// If a tree that appears to be taller than this is encountered, it is
/// assumed that the database is corrupt.
pub const BTCURSOR_MAX_DEPTH: usize = 20;
/// Evaluate a Result<CursorResult<T>>, if IO return IO.
@@ -57,6 +72,8 @@ macro_rules! return_if_locked {
}};
}
/// State machine of a write operation.
/// May involve balancing due to overflow.
#[derive(Debug)]
enum WriteState {
Start,
@@ -67,11 +84,16 @@ enum WriteState {
}
struct WriteInfo {
/// State of the write operation state machine.
state: WriteState,
/// Pages allocated during the write operation due to balancing.
new_pages: RefCell<Vec<PageRef>>,
/// Scratch space used during balancing.
scratch_cells: RefCell<Vec<&'static [u8]>>,
/// Bookkeeping of the rightmost pointer so the PAGE_HEADER_OFFSET_RIGHTMOST_PTR can be updated.
rightmost_pointer: RefCell<Option<u32>>,
page_copy: RefCell<Option<PageContent>>, // this holds the copy a of a page needed for buffer references
/// Copy of the current page needed for buffer references.
page_copy: RefCell<Option<PageContent>>,
}
pub struct BTreeCursor {
@@ -142,6 +164,8 @@ impl BTreeCursor {
}
}
/// Check if the table is empty.
/// This is done by checking if the root page has no cells.
fn is_empty_table(&mut self) -> Result<CursorResult<bool>> {
let page = self.pager.read_page(self.root_page)?;
return_if_locked!(page);
@@ -150,16 +174,18 @@ impl BTreeCursor {
Ok(CursorResult::Ok(cell_count == 0))
}
/// Move the cursor to the previous record and return it.
/// Used in backwards iteration.
fn get_prev_record(&mut self) -> Result<CursorResult<(Option<u64>, Option<OwnedRecord>)>> {
loop {
let page = self.stack.top();
let cell_idx = self.stack.current_index();
let cell_idx = self.stack.current_cell_index();
// moved to current page begin
// moved to beginning of current page
// todo: find a better way to flag moved to end or begin of page
if self.stack.curr_idx_out_of_begin() {
if self.stack.current_cell_index_less_than_min() {
loop {
if self.stack.current_index() > 0 {
if self.stack.current_cell_index() > 0 {
self.stack.retreat();
break;
}
@@ -198,8 +224,8 @@ impl BTreeCursor {
let cell = contents.cell_get(
cell_idx,
self.pager.clone(),
self.max_local(contents.page_type()),
self.min_local(contents.page_type()),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)?;
@@ -228,13 +254,15 @@ impl BTreeCursor {
}
}
/// Move the cursor to the next record and return it.
/// Used in forwards iteration, which is the default.
fn get_next_record(
&mut self,
predicate: Option<(SeekKey<'_>, SeekOp)>,
) -> Result<CursorResult<(Option<u64>, Option<OwnedRecord>)>> {
loop {
let mem_page_rc = self.stack.top();
let cell_idx = self.stack.current_index() as usize;
let cell_idx = self.stack.current_cell_index() as usize;
debug!("current id={} cell={}", mem_page_rc.get().id, cell_idx);
return_if_locked!(mem_page_rc);
@@ -286,8 +314,8 @@ impl BTreeCursor {
let cell = contents.cell_get(
cell_idx,
self.pager.clone(),
self.max_local(contents.page_type()),
self.min_local(contents.page_type()),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)?;
match &cell {
@@ -386,6 +414,9 @@ impl BTreeCursor {
}
}
/// Move the cursor to the record that matches the seek key and seek operation.
/// This may be used to seek to a specific record in a point query (e.g. SELECT * FROM table WHERE col = 10)
/// or e.g. find the first record greater than the seek key in a range query (e.g. SELECT * FROM table WHERE col > 10).
fn seek(
&mut self,
key: SeekKey<'_>,
@@ -403,8 +434,8 @@ impl BTreeCursor {
let cell = contents.cell_get(
cell_idx,
self.pager.clone(),
self.max_local(contents.page_type()),
self.min_local(contents.page_type()),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)?;
match &cell {
@@ -476,12 +507,14 @@ impl BTreeCursor {
Ok(CursorResult::Ok((None, None)))
}
/// Move the cursor to the root page of the btree.
fn move_to_root(&mut self) {
let mem_page = self.pager.read_page(self.root_page).unwrap();
self.stack.clear();
self.stack.push(mem_page);
}
/// Move the cursor to the rightmost record in the btree.
fn move_to_rightmost(&mut self) -> Result<CursorResult<()>> {
self.move_to_root();
@@ -553,8 +586,8 @@ impl BTreeCursor {
match &contents.cell_get(
cell_idx,
self.pager.clone(),
self.max_local(contents.page_type()),
self.min_local(contents.page_type()),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)? {
BTreeCell::TableInteriorCell(TableInteriorCell {
@@ -634,6 +667,8 @@ impl BTreeCursor {
}
}
/// Insert a record into the btree.
/// If the insert operation overflows the page, it will be split and the btree will be balanced.
fn insert_into_page(
&mut self,
key: &OwnedValue,
@@ -700,10 +735,15 @@ impl BTreeCursor {
}
}
/* insert to position and shift other pointers */
/// Insert a record into a cell.
/// If the cell overflows, an overflow cell is created.
/// insert_into_cell() is called from insert_into_page(),
/// and the overflow cell count is used to determine if the page overflows,
/// i.e. whether we need to balance the btree after the insert.
fn insert_into_cell(&self, page: &mut PageContent, payload: &[u8], cell_idx: usize) {
let free = self.compute_free_space(page, RefCell::borrow(&self.database_header));
let enough_space = payload.len() + 2 <= free as usize;
const CELL_POINTER_SIZE_BYTES: usize = 2;
let enough_space = payload.len() + CELL_POINTER_SIZE_BYTES <= free as usize;
if !enough_space {
// add to overflow cell
page.overflow_cells.push(OverflowCell {
@@ -714,61 +754,77 @@ impl BTreeCursor {
}
// TODO: insert into cell payload in internal page
let pc = self.allocate_cell_space(page, payload.len() as u16);
let new_cell_data_pointer = self.allocate_cell_space(page, payload.len() as u16);
let buf = page.as_ptr();
// copy data
buf[pc as usize..pc as usize + payload.len()].copy_from_slice(payload);
buf[new_cell_data_pointer as usize..new_cell_data_pointer as usize + payload.len()]
.copy_from_slice(payload);
// memmove(pIns+2, pIns, 2*(pPage->nCell - i));
let (pointer_area_pc_by_idx, _) = page.cell_get_raw_pointer_region();
let pointer_area_pc_by_idx = pointer_area_pc_by_idx + (2 * cell_idx);
let (cell_pointer_array_start, _) = page.cell_pointer_array_offset_and_size();
let cell_pointer_cur_idx = cell_pointer_array_start + (CELL_POINTER_SIZE_BYTES * cell_idx);
// move previous pointers forward and insert new pointer there
let n_cells_forward = 2 * (page.cell_count() - cell_idx);
if n_cells_forward > 0 {
// move existing pointers forward by CELL_POINTER_SIZE_BYTES...
let n_cells_forward = page.cell_count() - cell_idx;
let n_bytes_forward = CELL_POINTER_SIZE_BYTES * n_cells_forward;
if n_bytes_forward > 0 {
buf.copy_within(
pointer_area_pc_by_idx..pointer_area_pc_by_idx + n_cells_forward,
pointer_area_pc_by_idx + 2,
cell_pointer_cur_idx..cell_pointer_cur_idx + n_bytes_forward,
cell_pointer_cur_idx + CELL_POINTER_SIZE_BYTES,
);
}
page.write_u16(pointer_area_pc_by_idx - page.offset, pc);
// ...and insert new cell pointer at the current index
page.write_u16(cell_pointer_cur_idx - page.offset, new_cell_data_pointer);
// update first byte of content area
page.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, pc);
// update first byte of content area (cell data always appended to the left, so cell content area pointer moves to point to the new cell data)
page.write_u16(PAGE_HEADER_OFFSET_CELL_CONTENT_AREA, new_cell_data_pointer);
// update cell count
let new_n_cells = (page.cell_count() + 1) as u16;
page.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, new_n_cells);
page.write_u16(PAGE_HEADER_OFFSET_CELL_COUNT, new_n_cells);
}
/// Free the range of bytes that a cell occupies.
/// This function also updates the freeblock list in the page.
/// Freeblocks are used to keep track of free space in the page,
/// and are organized as a linked list.
fn free_cell_range(&self, page: &mut PageContent, offset: u16, len: u16) {
// if the freeblock list is empty, we set this block as the first freeblock in the page header.
if page.first_freeblock() == 0 {
// insert into empty list
page.write_u16(offset as usize, 0);
page.write_u16(offset as usize + 2, len);
page.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, offset);
page.write_u16(offset as usize, 0); // next freeblock = null
page.write_u16(offset as usize + 2, len); // size of this freeblock
page.write_u16(PAGE_HEADER_OFFSET_FIRST_FREEBLOCK, offset); // first freeblock in page = this block
return;
}
let first_block = page.first_freeblock();
// if the freeblock list is not empty, and the offset is less than the first freeblock,
// we insert this block at the head of the list
if offset < first_block {
// insert into head of list
page.write_u16(offset as usize, first_block);
page.write_u16(offset as usize + 2, len);
page.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, offset);
page.write_u16(offset as usize, first_block); // next freeblock = previous first freeblock
page.write_u16(offset as usize + 2, len); // size of this freeblock
page.write_u16(PAGE_HEADER_OFFSET_FIRST_FREEBLOCK, offset); // first freeblock in page = this block
return;
}
// if we clear space that is at the start of the cell content area,
// we need to update the cell content area pointer forward to account for the removed space
// FIXME: is offset ever < cell_content_area? cell content area grows leftwards and the pointer
// is to the start of the last allocated cell. should we assert!(offset >= page.cell_content_area())
// and change this to if offset == page.cell_content_area()?
if offset <= page.cell_content_area() {
// extend boundary of content area
page.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, page.first_freeblock());
page.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, offset + len);
// FIXME: remove the line directly below this, it does not change anything.
page.write_u16(PAGE_HEADER_OFFSET_FIRST_FREEBLOCK, page.first_freeblock());
page.write_u16(PAGE_HEADER_OFFSET_CELL_CONTENT_AREA, offset + len);
return;
}
// if the freeblock list is not empty, and the offset is greater than the first freeblock,
// then we need to do some more calculation to figure out where to insert the freeblock
// in the freeblock linked list.
let maxpc = {
let db_header = self.database_header.borrow();
let usable_space = (db_header.page_size - db_header.unused_space as u16) as usize;
let usable_space = (db_header.page_size - db_header.reserved_space as u16) as usize;
usable_space as u16
};
@@ -799,17 +855,23 @@ impl BTreeCursor {
}
}
/// Drop a cell from a page.
/// This is done by freeing the range of bytes that the cell occupies.
fn drop_cell(&self, page: &mut PageContent, cell_idx: usize) {
let (cell_start, cell_len) = page.cell_get_raw_region(
cell_idx,
self.max_local(page.page_type()),
self.min_local(page.page_type()),
self.payload_overflow_threshold_max(page.page_type()),
self.payload_overflow_threshold_min(page.page_type()),
self.usable_space(),
);
self.free_cell_range(page, cell_start as u16, cell_len as u16);
page.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, page.cell_count() as u16 - 1);
page.write_u16(PAGE_HEADER_OFFSET_CELL_COUNT, page.cell_count() as u16 - 1);
}
/// Balance a leaf page.
/// Balancing is done when a page overflows.
/// see e.g. https://en.wikipedia.org/wiki/B-tree
///
/// This is a naive algorithm that doesn't try to distribute cells evenly by content.
/// It will try to split the page in half by keys not by content.
/// Sqlite tries to have a page at least 40% full.
@@ -852,8 +914,8 @@ impl BTreeCursor {
for cell_idx in 0..page_copy.cell_count() {
let (start, len) = page_copy.cell_get_raw_region(
cell_idx,
self.max_local(page_copy.page_type()),
self.min_local(page_copy.page_type()),
self.payload_overflow_threshold_max(page_copy.page_type()),
self.payload_overflow_threshold_min(page_copy.page_type()),
self.usable_space(),
);
let buf = page_copy.as_ptr();
@@ -930,14 +992,14 @@ impl BTreeCursor {
assert_eq!(parent_contents.overflow_cells.len(), 0);
// Right page pointer is u32 in right most pointer, and in cell is u32 too, so we can use a *u32 to hold where we want to change this value
let mut right_pointer = BTREE_HEADER_OFFSET_RIGHTMOST;
let mut right_pointer = PAGE_HEADER_OFFSET_RIGHTMOST_PTR;
for cell_idx in 0..parent_contents.cell_count() {
let cell = parent_contents
.cell_get(
cell_idx,
self.pager.clone(),
self.max_local(page_type.clone()),
self.min_local(page_type.clone()),
self.payload_overflow_threshold_max(page_type.clone()),
self.payload_overflow_threshold_min(page_type.clone()),
self.usable_space(),
)
.unwrap();
@@ -950,8 +1012,8 @@ impl BTreeCursor {
if found {
let (start, _len) = parent_contents.cell_get_raw_region(
cell_idx,
self.max_local(page_type.clone()),
self.min_local(page_type.clone()),
self.payload_overflow_threshold_max(page_type.clone()),
self.payload_overflow_threshold_min(page_type.clone()),
self.usable_space(),
);
right_pointer = start;
@@ -967,17 +1029,20 @@ impl BTreeCursor {
assert!(page.is_dirty());
let contents = page.get().contents.as_mut().unwrap();
contents.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, 0);
contents.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, 0);
contents.write_u16(PAGE_HEADER_OFFSET_FIRST_FREEBLOCK, 0);
contents.write_u16(PAGE_HEADER_OFFSET_CELL_COUNT, 0);
let db_header = RefCell::borrow(&self.database_header);
let cell_content_area_start =
db_header.page_size - db_header.unused_space as u16;
contents.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, cell_content_area_start);
db_header.page_size - db_header.reserved_space as u16;
contents.write_u16(
PAGE_HEADER_OFFSET_CELL_CONTENT_AREA,
cell_content_area_start,
);
contents.write_u8(BTREE_HEADER_OFFSET_FRAGMENTED, 0);
contents.write_u8(PAGE_HEADER_OFFSET_FRAGMENTED_BYTES_COUNT, 0);
if !contents.is_leaf() {
contents.write_u32(BTREE_HEADER_OFFSET_RIGHTMOST, 0);
contents.write_u32(PAGE_HEADER_OFFSET_RIGHTMOST_PTR, 0);
}
}
@@ -1035,8 +1100,8 @@ impl BTreeCursor {
.cell_get(
contents.cell_count() - 1,
self.pager.clone(),
self.max_local(contents.page_type()),
self.min_local(contents.page_type()),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)
.unwrap();
@@ -1045,13 +1110,13 @@ impl BTreeCursor {
_ => unreachable!(),
};
self.drop_cell(contents, contents.cell_count() - 1);
contents.write_u32(BTREE_HEADER_OFFSET_RIGHTMOST, last_cell_pointer);
contents.write_u32(PAGE_HEADER_OFFSET_RIGHTMOST_PTR, last_cell_pointer);
}
// last page right most pointer points to previous right most pointer before splitting
let last_page = new_pages.last().unwrap();
let last_page_contents = last_page.get().contents.as_mut().unwrap();
last_page_contents.write_u32(
BTREE_HEADER_OFFSET_RIGHTMOST,
PAGE_HEADER_OFFSET_RIGHTMOST_PTR,
self.write_info.rightmost_pointer.borrow().unwrap(),
);
}
@@ -1069,8 +1134,8 @@ impl BTreeCursor {
&contents.page_type(),
0,
self.pager.clone(),
self.max_local(contents.page_type()),
self.min_local(contents.page_type()),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)
.unwrap();
@@ -1119,6 +1184,9 @@ impl BTreeCursor {
}
}
/// Balance the root page.
/// This is done when the root page overflows, and we need to create a new root page.
/// See e.g. https://en.wikipedia.org/wiki/B-tree
fn balance_root(&mut self) {
/* todo: balance deeper, create child and copy contents of root there. Then split root */
/* if we are in root page then we just need to create a new root and push key there */
@@ -1145,8 +1213,8 @@ impl BTreeCursor {
}
// point new root right child to previous root
new_root_page_contents
.write_u32(BTREE_HEADER_OFFSET_RIGHTMOST, new_root_page_id as u32);
new_root_page_contents.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, 0);
.write_u32(PAGE_HEADER_OFFSET_RIGHTMOST_PTR, new_root_page_id as u32);
new_root_page_contents.write_u16(PAGE_HEADER_OFFSET_CELL_COUNT, 0);
}
/* swap splitted page buffer with new root buffer so we don't have to update page idx */
@@ -1164,7 +1232,7 @@ impl BTreeCursor {
if is_page_1 {
// Remove header from child and set offset to 0
let contents = child.get().contents.as_mut().unwrap();
let (cell_pointer_offset, _) = contents.cell_get_raw_pointer_region();
let (cell_pointer_offset, _) = contents.cell_pointer_array_offset_and_size();
// change cell pointers
for cell_idx in 0..contents.cell_count() {
let cell_pointer_offset = cell_pointer_offset + (2 * cell_idx) - offset;
@@ -1195,12 +1263,16 @@ impl BTreeCursor {
}
}
/// Allocate a new page to the btree via the pager.
/// This marks the page as dirty and writes the page header.
fn allocate_page(&self, page_type: PageType, offset: usize) -> PageRef {
let page = self.pager.allocate_page().unwrap();
btree_init_page(&page, page_type, &self.database_header.borrow(), offset);
page
}
/// Allocate a new overflow page.
/// This is done when a cell overflows and new space is needed.
fn allocate_overflow_page(&self) -> PageRef {
let page = self.pager.allocate_page().unwrap();
@@ -1212,13 +1284,11 @@ impl BTreeCursor {
page
}
/*
Allocate space for a cell on a page.
*/
/// Allocate space for a cell on a page.
fn allocate_cell_space(&self, page_ref: &PageContent, amount: u16) -> u16 {
let amount = amount as usize;
let (cell_offset, _) = page_ref.cell_get_raw_pointer_region();
let (cell_offset, _) = page_ref.cell_pointer_array_offset_and_size();
let gap = cell_offset + 2 * page_ref.cell_count();
let mut top = page_ref.cell_content_area() as usize;
@@ -1236,33 +1306,31 @@ impl BTreeCursor {
if gap + 2 + amount > top {
// defragment
self.defragment_page(page_ref, RefCell::borrow(&self.database_header));
top = page_ref.read_u16(BTREE_HEADER_OFFSET_CELL_CONTENT) as usize;
top = page_ref.read_u16(PAGE_HEADER_OFFSET_CELL_CONTENT_AREA) as usize;
}
let db_header = RefCell::borrow(&self.database_header);
top -= amount;
page_ref.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, top as u16);
page_ref.write_u16(PAGE_HEADER_OFFSET_CELL_CONTENT_AREA, top as u16);
let usable_space = (db_header.page_size - db_header.unused_space as u16) as usize;
let usable_space = (db_header.page_size - db_header.reserved_space as u16) as usize;
assert!(top + amount <= usable_space);
top as u16
}
/// Defragment a page. This means packing all the cells to the end of the page.
fn defragment_page(&self, page: &PageContent, db_header: Ref<DatabaseHeader>) {
log::debug!("defragment_page");
let cloned_page = page.clone();
// TODO(pere): usable space should include offset probably
let usable_space = (db_header.page_size - db_header.unused_space as u16) as u64;
let usable_space = (db_header.page_size - db_header.reserved_space as u16) as u64;
let mut cbrk = usable_space;
// TODO: implement fast algorithm
let last_cell = usable_space - 4;
let first_cell = {
let (start, end) = cloned_page.cell_get_raw_pointer_region();
start + end
};
let first_cell = cloned_page.unallocated_region_start() as u64;
if cloned_page.cell_count() > 0 {
let page_type = page.page_type();
@@ -1330,42 +1398,54 @@ impl BTreeCursor {
let write_buf = page.as_ptr();
// set new first byte of cell content
page.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, cbrk as u16);
page.write_u16(PAGE_HEADER_OFFSET_CELL_CONTENT_AREA, cbrk as u16);
// set free block to 0, unused spaced can be retrieved from gap between cell pointer end and content start
page.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, 0);
page.write_u16(PAGE_HEADER_OFFSET_FIRST_FREEBLOCK, 0);
// set unused space to 0
let first_cell = cloned_page.cell_content_area() as u64;
assert!(first_cell <= cbrk);
write_buf[first_cell as usize..cbrk as usize].fill(0);
}
// Free blocks can be zero, meaning the "real free space" that can be used to allocate is expected to be between first cell byte
// and end of cell pointer area.
/// Free blocks can be zero, meaning the "real free space" that can be used to allocate is expected to be between first cell byte
/// and end of cell pointer area.
#[allow(unused_assignments)]
fn compute_free_space(&self, page: &PageContent, db_header: Ref<DatabaseHeader>) -> u16 {
// TODO(pere): maybe free space is not calculated correctly with offset
let buf = page.as_ptr();
let usable_space = (db_header.page_size - db_header.unused_space as u16) as usize;
let mut first_byte_in_cell_content = page.cell_content_area();
if first_byte_in_cell_content == 0 {
first_byte_in_cell_content = u16::MAX;
// Usable space, not the same as free space, simply means:
// space that is not reserved for extensions by sqlite. Usually reserved_space is 0.
let usable_space = (db_header.page_size - db_header.reserved_space as u16) as usize;
let mut cell_content_area_start = page.cell_content_area();
// A zero value for the cell content area pointer is interpreted as 65536.
// See https://www.sqlite.org/fileformat.html
// The max page size for a sqlite database is 64kiB i.e. 65536 bytes.
// 65536 is u16::MAX + 1, and since cell content grows from right to left, this means
// the cell content area pointer is at the end of the page,
// i.e.
// 1. the page size is 64kiB
// 2. there are no cells on the page
// 3. there is no reserved space at the end of the page
if cell_content_area_start == 0 {
cell_content_area_start = u16::MAX;
}
let fragmented_free_bytes = page.num_frag_free_bytes();
let free_block_pointer = page.first_freeblock();
let ncell = page.cell_count();
// The amount of free space is the sum of:
// #1. the size of the unallocated region
// #2. fragments (isolated 1-3 byte chunks of free space within the cell content area)
// #3. freeblocks (linked list of blocks of at least 4 bytes within the cell content area that are not in use due to e.g. deletions)
// 8 + 4 == header end
let child_pointer_size = if page.is_leaf() { 0 } else { 4 };
let first_cell = (page.offset + 8 + child_pointer_size + (2 * ncell)) as u16;
let mut free_space_bytes =
page.unallocated_region_size() as usize + page.num_frag_free_bytes() as usize;
let mut nfree = fragmented_free_bytes as usize + first_byte_in_cell_content as usize;
let mut pc = free_block_pointer as usize;
if pc > 0 {
if pc < first_byte_in_cell_content as usize {
// corrupt
// #3 is computed by iterating over the freeblocks linked list
let mut cur_freeblock_ptr = page.first_freeblock() as usize;
let page_buf = page.as_ptr();
if cur_freeblock_ptr > 0 {
if cur_freeblock_ptr < cell_content_area_start as usize {
// Freeblocks exist in the cell content area e.g. after deletions
// They should never exist in the unused area of the page.
todo!("corrupted page");
}
@@ -1373,32 +1453,51 @@ impl BTreeCursor {
let mut size = 0;
loop {
// TODO: check corruption icellast
next = u16::from_be_bytes(buf[pc..pc + 2].try_into().unwrap()) as usize;
size = u16::from_be_bytes(buf[pc + 2..pc + 4].try_into().unwrap()) as usize;
nfree += size;
if next <= pc + size + 3 {
next = u16::from_be_bytes(
page_buf[cur_freeblock_ptr..cur_freeblock_ptr + 2]
.try_into()
.unwrap(),
) as usize; // first 2 bytes in freeblock = next freeblock pointer
size = u16::from_be_bytes(
page_buf[cur_freeblock_ptr + 2..cur_freeblock_ptr + 4]
.try_into()
.unwrap(),
) as usize; // next 2 bytes in freeblock = size of current freeblock
free_space_bytes += size;
// Freeblocks are in order from left to right on the page,
// so next pointer should > current pointer + its size, or 0 if no next block exists.
if next <= cur_freeblock_ptr + size + 3 {
break;
}
pc = next;
cur_freeblock_ptr = next;
}
if next > 0 {
todo!("corrupted page ascending order");
}
// Next should always be 0 (NULL) at this point since we have reached the end of the freeblocks linked list
assert!(
next == 0,
"corrupted page: freeblocks list not in ascending order"
);
if pc + size > usable_space {
todo!("corrupted page last freeblock extends last page end");
}
assert!(
cur_freeblock_ptr + size <= usable_space,
"corrupted page: last freeblock extends last page end"
);
}
assert!(
free_space_bytes <= usable_space,
"corrupted page: free space is greater than usable space"
);
// if( nFree>usableSize || nFree<iCellFirst ){
// return SQLITE_CORRUPT_PAGE(pPage);
// }
// don't count header and cell pointers?
nfree -= first_cell as usize;
nfree as u16
free_space_bytes as u16
}
/// Fill in the cell payload with the record.
/// If the record is too large to fit in the cell, it will spill onto overflow pages.
fn fill_cell_payload(
&self,
page_type: PageType,
@@ -1423,25 +1522,26 @@ impl BTreeCursor {
write_varint_to_vec(record_buf.len() as u64, cell_payload);
}
let max_local = self.max_local(page_type.clone());
let payload_overflow_threshold_max = self.payload_overflow_threshold_max(page_type.clone());
log::debug!(
"fill_cell_payload(record_size={}, max_local={})",
"fill_cell_payload(record_size={}, payload_overflow_threshold_max={})",
record_buf.len(),
max_local
payload_overflow_threshold_max
);
if record_buf.len() <= max_local {
if record_buf.len() <= payload_overflow_threshold_max {
// enough allowed space to fit inside a btree page
cell_payload.extend_from_slice(record_buf.as_slice());
cell_payload.resize(cell_payload.len() + 4, 0);
return;
}
log::debug!("fill_cell_payload(overflow)");
let min_local = self.min_local(page_type);
let mut space_left = min_local + (record_buf.len() - min_local) % (self.usable_space() - 4);
let payload_overflow_threshold_min = self.payload_overflow_threshold_min(page_type);
// see e.g. https://github.com/sqlite/sqlite/blob/9591d3fe93936533c8c3b0dc4d025ac999539e11/src/dbstat.c#L371
let mut space_left = payload_overflow_threshold_min
+ (record_buf.len() - payload_overflow_threshold_min) % (self.usable_space() - 4);
if space_left > max_local {
space_left = min_local;
if space_left > payload_overflow_threshold_max {
space_left = payload_overflow_threshold_min;
}
// cell_size must be equal to first value of space_left as this will be the bytes copied to non-overflow page.
@@ -1487,31 +1587,54 @@ impl BTreeCursor {
assert_eq!(cell_size, cell_payload.len());
}
fn max_local(&self, page_type: PageType) -> usize {
let usable_space = self.usable_space();
/// Returns the maximum payload size (X) that can be stored directly on a b-tree page without spilling to overflow pages.
///
/// For table leaf pages: X = usable_size - 35
/// For index pages: X = ((usable_size - 12) * 64/255) - 23
///
/// The usable size is the total page size less the reserved space at the end of each page.
/// These thresholds are designed to:
/// - Give a minimum fanout of 4 for index b-trees
/// - Ensure enough payload is on the b-tree page that the record header can usually be accessed
/// without consulting an overflow page
fn payload_overflow_threshold_max(&self, page_type: PageType) -> usize {
let usable_size = self.usable_space();
match page_type {
PageType::IndexInterior | PageType::TableInterior => {
(usable_space - 12) * 64 / 255 - 23
PageType::IndexInterior | PageType::IndexLeaf => {
((usable_size - 12) * 64 / 255) - 23 // Index page formula
}
PageType::TableInterior | PageType::TableLeaf => {
usable_size - 35 // Table leaf page formula
}
PageType::IndexLeaf | PageType::TableLeaf => usable_space - 35,
}
}
fn min_local(&self, page_type: PageType) -> usize {
let usable_space = self.usable_space();
match page_type {
PageType::IndexInterior | PageType::TableInterior => {
(usable_space - 12) * 32 / 255 - 23
}
PageType::IndexLeaf | PageType::TableLeaf => (usable_space - 12) * 32 / 255 - 23,
}
/// Returns the minimum payload size (M) that must be stored on the b-tree page before spilling to overflow pages is allowed.
///
/// For all page types: M = ((usable_size - 12) * 32/255) - 23
///
/// When payload size P exceeds max_local():
/// - If K = M + ((P-M) % (usable_size-4)) <= max_local(): store K bytes on page
/// - Otherwise: store M bytes on page
///
/// The remaining bytes are stored on overflow pages in both cases.
fn payload_overflow_threshold_min(&self, _page_type: PageType) -> usize {
let usable_size = self.usable_space();
// Same formula for all page types
((usable_size - 12) * 32 / 255) - 23
}
/// The "usable size" of a database page is the page size specified by the 2-byte integer at offset 16
/// in the header, minus the "reserved" space size recorded in the 1-byte integer at offset 20 in the header.
/// The usable size of a page might be an odd number. However, the usable size is not allowed to be less than 480.
/// In other words, if the page size is 512, then the reserved space size cannot exceed 32.
fn usable_space(&self) -> usize {
let db_header = RefCell::borrow(&self.database_header);
(db_header.page_size - db_header.unused_space as u16) as usize
(db_header.page_size - db_header.reserved_space as u16) as usize
}
/// Find the index of the cell in the page that contains the given rowid.
/// BTree tables only.
fn find_cell(&self, page: &PageContent, int_key: u64) -> usize {
let mut cell_idx = 0;
let cell_count = page.cell_count();
@@ -1520,8 +1643,8 @@ impl BTreeCursor {
.cell_get(
cell_idx,
self.pager.clone(),
self.max_local(page.page_type()),
self.min_local(page.page_type()),
self.payload_overflow_threshold_max(page.page_type()),
self.payload_overflow_threshold_min(page.page_type()),
self.usable_space(),
)
.unwrap()
@@ -1545,6 +1668,8 @@ impl BTreeCursor {
}
impl PageStack {
/// Push a new page onto the stack.
/// This effectively means traversing to a child page.
fn push(&self, page: PageRef) {
debug!(
"pagestack::push(current={}, new_page_id={})",
@@ -1561,6 +1686,8 @@ impl PageStack {
self.cell_indices.borrow_mut()[current as usize] = 0;
}
/// Pop a page off the stack.
/// This effectively means traversing back up to a parent page.
fn pop(&self) {
let current = *self.current_page.borrow();
debug!("pagestack::pop(current={})", current);
@@ -1569,6 +1696,8 @@ impl PageStack {
*self.current_page.borrow_mut() -= 1;
}
/// Get the top page on the stack.
/// This is the page that is currently being traversed.
fn top(&self) -> PageRef {
let current = *self.current_page.borrow();
let page = self.stack.borrow()[current as usize]
@@ -1583,6 +1712,7 @@ impl PageStack {
page
}
/// Get the parent page of the current page.
fn parent(&self) -> PageRef {
let current = *self.current_page.borrow();
self.stack.borrow()[current as usize - 1]
@@ -1597,13 +1727,15 @@ impl PageStack {
}
/// Cell index of the current page
fn current_index(&self) -> i32 {
fn current_cell_index(&self) -> i32 {
let current = self.current();
self.cell_indices.borrow()[current]
}
fn curr_idx_out_of_begin(&self) -> bool {
let cell_idx = self.current_index();
/// Check if the current cell index is less than 0.
/// This means we have been iterating backwards and have reached the start of the page.
fn current_cell_index_less_than_min(&self) -> bool {
let cell_idx = self.current_cell_index();
cell_idx < 0
}
@@ -1639,7 +1771,7 @@ fn find_free_cell(page_ref: &PageContent, db_header: Ref<DatabaseHeader>, amount
let buf = page_ref.as_ptr();
let usable_space = (db_header.page_size - db_header.unused_space as u16) as usize;
let usable_space = (db_header.page_size - db_header.reserved_space as u16) as usize;
let maxpc = usable_space - amount;
let mut found = false;
while pc <= maxpc {
@@ -1753,6 +1885,11 @@ impl Cursor for BTreeCursor {
Ok(CursorResult::Ok(()))
}
fn delete(&mut self) -> Result<CursorResult<()>> {
println!("rowid: {:?}", self.rowid.borrow());
Ok(CursorResult::Ok(()))
}
fn set_null_flag(&mut self, flag: bool) {
self.null_flag = flag;
}
@@ -1785,8 +1922,8 @@ impl Cursor for BTreeCursor {
let equals = match &contents.cell_get(
cell_idx,
self.pager.clone(),
self.max_local(contents.page_type()),
self.min_local(contents.page_type()),
self.payload_overflow_threshold_max(contents.page_type()),
self.payload_overflow_threshold_min(contents.page_type()),
self.usable_space(),
)? {
BTreeCell::TableLeafCell(l) => l._rowid == int_key,
@@ -1823,15 +1960,18 @@ pub fn btree_init_page(
let contents = contents.contents.as_mut().unwrap();
contents.offset = offset;
let id = page_type as u8;
contents.write_u8(BTREE_HEADER_OFFSET_TYPE, id);
contents.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, 0);
contents.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, 0);
contents.write_u8(PAGE_HEADER_OFFSET_PAGE_TYPE, id);
contents.write_u16(PAGE_HEADER_OFFSET_FIRST_FREEBLOCK, 0);
contents.write_u16(PAGE_HEADER_OFFSET_CELL_COUNT, 0);
let cell_content_area_start = db_header.page_size - db_header.unused_space as u16;
contents.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, cell_content_area_start);
let cell_content_area_start = db_header.page_size - db_header.reserved_space as u16;
contents.write_u16(
PAGE_HEADER_OFFSET_CELL_CONTENT_AREA,
cell_content_area_start,
);
contents.write_u8(BTREE_HEADER_OFFSET_FRAGMENTED, 0);
contents.write_u32(BTREE_HEADER_OFFSET_RIGHTMOST, 0);
contents.write_u8(PAGE_HEADER_OFFSET_FRAGMENTED_BYTES_COUNT, 0);
contents.write_u32(PAGE_HEADER_OFFSET_RIGHTMOST_PTR, 0);
}
fn to_static_buf(buf: &[u8]) -> &'static [u8] {

3
core/storage/mod.rs
View File

@@ -10,11 +10,12 @@
//! for reading and writing pages to the database file, either local or
//! remote. The `Wal` struct is responsible for managing the write-ahead log
//! for the database, also either local or remote.
pub(crate) mod btree;
pub(crate) mod buffer_pool;
pub(crate) mod database;
pub(crate) mod page_cache;
#[allow(clippy::arc_with_non_send_sync)]
pub(crate) mod pager;
pub(crate) mod sqlite3_ondisk;
#[allow(clippy::arc_with_non_send_sync)]
pub(crate) mod wal;

40
core/storage/pager.rs
View File

@@ -1,9 +1,10 @@
use crate::result::LimboResult;
use crate::storage::buffer_pool::BufferPool;
use crate::storage::database::DatabaseStorage;
use crate::storage::sqlite3_ondisk::{self, DatabaseHeader, PageContent};
use crate::storage::wal::Wal;
use crate::{Buffer, Result};
use log::{debug, trace};
use log::trace;
use std::cell::{RefCell, UnsafeCell};
use std::collections::HashSet;
use std::rc::Rc;
@@ -11,7 +12,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, RwLock};
use super::page_cache::{DumbLruPageCache, PageCacheKey};
use super::wal::CheckpointStatus;
use super::wal::{CheckpointMode, CheckpointStatus};
pub struct PageInner {
pub flags: AtomicUsize,
@@ -39,8 +40,8 @@ const PAGE_DIRTY: usize = 0b1000;
const PAGE_LOADED: usize = 0b10000;
impl Page {
pub fn new(id: usize) -> Page {
Page {
pub fn new(id: usize) -> Self {
Self {
inner: UnsafeCell::new(PageInner {
flags: AtomicUsize::new(0),
contents: None,
@@ -49,6 +50,7 @@ impl Page {
}
}
#[allow(clippy::mut_from_ref)]
pub fn get(&self) -> &mut PageInner {
unsafe { &mut *self.inner.get() }
}
@@ -196,14 +198,12 @@ impl Pager {
})
}
pub fn begin_read_tx(&self) -> Result<()> {
self.wal.borrow_mut().begin_read_tx()?;
Ok(())
pub fn begin_read_tx(&self) -> Result<LimboResult> {
self.wal.borrow_mut().begin_read_tx()
}
pub fn begin_write_tx(&self) -> Result<()> {
self.wal.borrow_mut().begin_write_tx()?;
Ok(())
pub fn begin_write_tx(&self) -> Result<LimboResult> {
self.wal.borrow_mut().begin_write_tx()
}
pub fn end_tx(&self) -> Result<CheckpointStatus> {
@@ -378,7 +378,11 @@ impl Pager {
match state {
CheckpointState::Checkpoint => {
let in_flight = self.checkpoint_inflight.clone();
match self.wal.borrow_mut().checkpoint(self, in_flight)? {
match self.wal.borrow_mut().checkpoint(
self,
in_flight,
CheckpointMode::Passive,
)? {
CheckpointStatus::IO => return Ok(CheckpointStatus::IO),
CheckpointStatus::Done => {
self.checkpoint_state.replace(CheckpointState::SyncDbFile);
@@ -414,13 +418,13 @@ impl Pager {
// WARN: used for testing purposes
pub fn clear_page_cache(&self) {
loop {
match self
.wal
.borrow_mut()
.checkpoint(self, Rc::new(RefCell::new(0)))
{
match self.wal.borrow_mut().checkpoint(
self,
Rc::new(RefCell::new(0)),
CheckpointMode::Passive,
) {
Ok(CheckpointStatus::IO) => {
self.io.run_once();
let _ = self.io.run_once();
}
Ok(CheckpointStatus::Done) => {
break;
@@ -482,7 +486,7 @@ impl Pager {
pub fn usable_size(&self) -> usize {
let db_header = self.db_header.borrow();
(db_header.page_size - db_header.unused_space as u16) as usize
(db_header.page_size - db_header.reserved_space as u16) as usize
}
}

289
core/storage/sqlite3_ondisk.rs
View File

@@ -64,30 +64,84 @@ const DEFAULT_CACHE_SIZE: i32 = -2000;
// Minimum number of pages that cache can hold.
pub const MIN_PAGE_CACHE_SIZE: usize = 10;
/// The database header.
/// The first 100 bytes of the database file comprise the database file header.
/// The database file header is divided into fields as shown by the table below.
/// All multibyte fields in the database file header are stored with the most significant byte first (big-endian).
#[derive(Debug, Clone)]
pub struct DatabaseHeader {
/// The header string: "SQLite format 3\0"
magic: [u8; 16],
/// The database page size in bytes. Must be a power of two between 512 and 32768 inclusive,
/// or the value 1 representing a page size of 65536.
pub page_size: u16,
/// File format write version. 1 for legacy; 2 for WAL.
write_version: u8,
/// File format read version. 1 for legacy; 2 for WAL.
read_version: u8,
pub unused_space: u8,
/// Bytes of unused "reserved" space at the end of each page. Usually 0.
/// SQLite has the ability to set aside a small number of extra bytes at the end of every page for use by extensions.
/// These extra bytes are used, for example, by the SQLite Encryption Extension to store a nonce and/or
/// cryptographic checksum associated with each page.
pub reserved_space: u8,
/// Maximum embedded payload fraction. Must be 64.
max_embed_frac: u8,
/// Minimum embedded payload fraction. Must be 32.
min_embed_frac: u8,
/// Leaf payload fraction. Must be 32.
min_leaf_frac: u8,
/// File change counter, incremented when database is modified.
change_counter: u32,
/// Size of the database file in pages. The "in-header database size".
pub database_size: u32,
/// Page number of the first freelist trunk page.
freelist_trunk_page: u32,
/// Total number of freelist pages.
freelist_pages: u32,
/// The schema cookie. Incremented when the database schema changes.
schema_cookie: u32,
/// The schema format number. Supported formats are 1, 2, 3, and 4.
schema_format: u32,
pub default_cache_size: i32,
vacuum: u32,
/// Default page cache size.
pub default_page_cache_size: i32,
/// The page number of the largest root b-tree page when in auto-vacuum or
/// incremental-vacuum modes, or zero otherwise.
vacuum_mode_largest_root_page: u32,
/// The database text encoding. 1=UTF-8, 2=UTF-16le, 3=UTF-16be.
text_encoding: u32,
/// The "user version" as read and set by the user_version pragma.
user_version: u32,
incremental_vacuum: u32,
/// True (non-zero) for incremental-vacuum mode. False (zero) otherwise.
incremental_vacuum_enabled: u32,
/// The "Application ID" set by PRAGMA application_id.
application_id: u32,
reserved: [u8; 20],
/// Reserved for expansion. Must be zero.
reserved_for_expansion: [u8; 20],
/// The version-valid-for number.
version_valid_for: u32,
/// SQLITE_VERSION_NUMBER
pub version_number: u32,
}
@@ -98,28 +152,62 @@ pub const WAL_FRAME_HEADER_SIZE: usize = 24;
pub const WAL_MAGIC_LE: u32 = 0x377f0682;
pub const WAL_MAGIC_BE: u32 = 0x377f0683;
/// The Write-Ahead Log (WAL) header.
/// The first 32 bytes of a WAL file comprise the WAL header.
/// The WAL header is divided into the following fields stored in big-endian order.
#[derive(Debug, Default, Clone)]
#[repr(C)] // This helps with encoding because rust does not respect the order in structs, so in
// this case we want to keep the order
pub struct WalHeader {
/// Magic number. 0x377f0682 or 0x377f0683
/// If the LSB is 0, checksums are native byte order, else checksums are serialized
pub magic: u32,
/// WAL format version. Currently 3007000
pub file_format: u32,
/// Database page size in bytes. Power of two between 512 and 32768 inclusive
pub page_size: u32,
/// Checkpoint sequence number. Increases with each checkpoint
pub checkpoint_seq: u32,
/// Random value used for the first salt in checksum calculations
pub salt_1: u32,
/// Random value used for the second salt in checksum calculations
pub salt_2: u32,
/// First checksum value in the wal-header
pub checksum_1: u32,
/// Second checksum value in the wal-header
pub checksum_2: u32,
}
/// Immediately following the wal-header are zero or more frames.
/// Each frame consists of a 24-byte frame-header followed by <page-size> bytes of page data.
/// The frame-header is six big-endian 32-bit unsigned integer values, as follows:
#[allow(dead_code)]
#[derive(Debug, Default)]
pub struct WalFrameHeader {
/// Page number
page_number: u32,
/// For commit records, the size of the database file in pages after the commit.
/// For all other records, zero.
db_size: u32,
/// Salt-1 copied from the WAL header
salt_1: u32,
/// Salt-2 copied from the WAL header
salt_2: u32,
/// Checksum-1: Cumulative checksum up through and including this page
checksum_1: u32,
/// Checksum-2: Second half of the cumulative checksum
checksum_2: u32,
}
@@ -130,7 +218,7 @@ impl Default for DatabaseHeader {
page_size: 4096,
write_version: 2,
read_version: 2,
unused_space: 0,
reserved_space: 0,
max_embed_frac: 64,
min_embed_frac: 32,
min_leaf_frac: 32,
@@ -140,13 +228,13 @@ impl Default for DatabaseHeader {
freelist_pages: 0,
schema_cookie: 0,
schema_format: 4, // latest format, new sqlite3 databases use this format
default_cache_size: 500, // pages
vacuum: 0,
default_page_cache_size: 500, // pages
vacuum_mode_largest_root_page: 0,
text_encoding: 1, // utf-8
user_version: 1,
incremental_vacuum: 0,
incremental_vacuum_enabled: 0,
application_id: 0,
reserved: [0; 20],
reserved_for_expansion: [0; 20],
version_valid_for: 3047000,
version_number: 3047000,
}
@@ -180,7 +268,7 @@ fn finish_read_database_header(
header.page_size = u16::from_be_bytes([buf[16], buf[17]]);
header.write_version = buf[18];
header.read_version = buf[19];
header.unused_space = buf[20];
header.reserved_space = buf[20];
header.max_embed_frac = buf[21];
header.min_embed_frac = buf[22];
header.min_leaf_frac = buf[23];
@@ -190,16 +278,16 @@ fn finish_read_database_header(
header.freelist_pages = u32::from_be_bytes([buf[36], buf[37], buf[38], buf[39]]);
header.schema_cookie = u32::from_be_bytes([buf[40], buf[41], buf[42], buf[43]]);
header.schema_format = u32::from_be_bytes([buf[44], buf[45], buf[46], buf[47]]);
header.default_cache_size = i32::from_be_bytes([buf[48], buf[49], buf[50], buf[51]]);
if header.default_cache_size == 0 {
header.default_cache_size = DEFAULT_CACHE_SIZE;
header.default_page_cache_size = i32::from_be_bytes([buf[48], buf[49], buf[50], buf[51]]);
if header.default_page_cache_size == 0 {
header.default_page_cache_size = DEFAULT_CACHE_SIZE;
}
header.vacuum = u32::from_be_bytes([buf[52], buf[53], buf[54], buf[55]]);
header.vacuum_mode_largest_root_page = u32::from_be_bytes([buf[52], buf[53], buf[54], buf[55]]);
header.text_encoding = u32::from_be_bytes([buf[56], buf[57], buf[58], buf[59]]);
header.user_version = u32::from_be_bytes([buf[60], buf[61], buf[62], buf[63]]);
header.incremental_vacuum = u32::from_be_bytes([buf[64], buf[65], buf[66], buf[67]]);
header.incremental_vacuum_enabled = u32::from_be_bytes([buf[64], buf[65], buf[66], buf[67]]);
header.application_id = u32::from_be_bytes([buf[68], buf[69], buf[70], buf[71]]);
header.reserved.copy_from_slice(&buf[72..92]);
header.reserved_for_expansion.copy_from_slice(&buf[72..92]);
header.version_valid_for = u32::from_be_bytes([buf[92], buf[93], buf[94], buf[95]]);
header.version_number = u32::from_be_bytes([buf[96], buf[97], buf[98], buf[99]]);
Ok(())
@@ -258,7 +346,7 @@ fn write_header_to_buf(buf: &mut [u8], header: &DatabaseHeader) {
buf[16..18].copy_from_slice(&header.page_size.to_be_bytes());
buf[18] = header.write_version;
buf[19] = header.read_version;
buf[20] = header.unused_space;
buf[20] = header.reserved_space;
buf[21] = header.max_embed_frac;
buf[22] = header.min_embed_frac;
buf[23] = header.min_leaf_frac;
@@ -268,15 +356,15 @@ fn write_header_to_buf(buf: &mut [u8], header: &DatabaseHeader) {
buf[36..40].copy_from_slice(&header.freelist_pages.to_be_bytes());
buf[40..44].copy_from_slice(&header.schema_cookie.to_be_bytes());
buf[44..48].copy_from_slice(&header.schema_format.to_be_bytes());
buf[48..52].copy_from_slice(&header.default_cache_size.to_be_bytes());
buf[48..52].copy_from_slice(&header.default_page_cache_size.to_be_bytes());
buf[52..56].copy_from_slice(&header.vacuum.to_be_bytes());
buf[52..56].copy_from_slice(&header.vacuum_mode_largest_root_page.to_be_bytes());
buf[56..60].copy_from_slice(&header.text_encoding.to_be_bytes());
buf[60..64].copy_from_slice(&header.user_version.to_be_bytes());
buf[64..68].copy_from_slice(&header.incremental_vacuum.to_be_bytes());
buf[64..68].copy_from_slice(&header.incremental_vacuum_enabled.to_be_bytes());
buf[68..72].copy_from_slice(&header.application_id.to_be_bytes());
buf[72..92].copy_from_slice(&header.reserved);
buf[72..92].copy_from_slice(&header.reserved_for_expansion);
buf[92..96].copy_from_slice(&header.version_valid_for.to_be_bytes());
buf[96..100].copy_from_slice(&header.version_number.to_be_bytes());
}
@@ -387,18 +475,60 @@ impl PageContent {
buf[self.offset + pos..self.offset + pos + 4].copy_from_slice(&value.to_be_bytes());
}
/// The second field of the b-tree page header is the offset of the first freeblock, or zero if there are no freeblocks on the page.
/// A freeblock is a structure used to identify unallocated space within a b-tree page.
/// Freeblocks are organized as a chain.
///
/// To be clear, freeblocks do not mean the regular unallocated free space to the left of the cell content area pointer, but instead
/// blocks of at least 4 bytes WITHIN the cell content area that are not in use due to e.g. deletions.
pub fn first_freeblock(&self) -> u16 {
self.read_u16(1)
}
/// The number of cells on the page.
pub fn cell_count(&self) -> usize {
self.read_u16(3) as usize
}
/// The size of the cell pointer array in bytes.
/// 2 bytes per cell pointer
pub fn cell_pointer_array_size(&self) -> usize {
const CELL_POINTER_SIZE_BYTES: usize = 2;
self.cell_count() * CELL_POINTER_SIZE_BYTES
}
/// The start of the unallocated region.
/// Effectively: the offset after the page header + the cell pointer array.
pub fn unallocated_region_start(&self) -> usize {
let (cell_ptr_array_start, cell_ptr_array_size) = self.cell_pointer_array_offset_and_size();
cell_ptr_array_start + cell_ptr_array_size
}
pub fn unallocated_region_size(&self) -> usize {
self.cell_content_area() as usize - self.unallocated_region_start()
}
/// The start of the cell content area.
/// SQLite strives to place cells as far toward the end of the b-tree page as it can,
/// in order to leave space for future growth of the cell pointer array.
/// = the cell content area pointer moves leftward as cells are added to the page
pub fn cell_content_area(&self) -> u16 {
self.read_u16(5)
}
/// The size of the page header in bytes.
/// 8 bytes for leaf pages, 12 bytes for interior pages (due to storing rightmost child pointer)
pub fn header_size(&self) -> usize {
match self.page_type() {
PageType::IndexInterior => 12,
PageType::TableInterior => 12,
PageType::IndexLeaf => 8,
PageType::TableLeaf => 8,
}
}
/// The total number of bytes in all fragments is stored in the fifth field of the b-tree page header.
/// Fragments are isolated groups of 1, 2, or 3 unused bytes within the cell content area.
pub fn num_frag_free_bytes(&self) -> u8 {
self.read_u8(7)
}
@@ -416,22 +546,19 @@ impl PageContent {
&self,
idx: usize,
pager: Rc<Pager>,
max_local: usize,
min_local: usize,
payload_overflow_threshold_max: usize,
payload_overflow_threshold_min: usize,
usable_size: usize,
) -> Result<BTreeCell> {
log::debug!("cell_get(idx={})", idx);
let buf = self.as_ptr();
let ncells = self.cell_count();
let cell_start = match self.page_type() {
PageType::IndexInterior => 12,
PageType::TableInterior => 12,
PageType::IndexLeaf => 8,
PageType::TableLeaf => 8,
};
// the page header is 12 bytes for interior pages, 8 bytes for leaf pages
// this is because the 4 last bytes in the interior page's header are used for the rightmost pointer.
let cell_pointer_array_start = self.header_size();
assert!(idx < ncells, "cell_get: idx out of bounds");
let cell_pointer = cell_start + (idx * 2);
let cell_pointer = cell_pointer_array_start + (idx * 2);
let cell_pointer = self.read_u16(cell_pointer) as usize;
read_btree_cell(
@@ -439,48 +566,46 @@ impl PageContent {
&self.page_type(),
cell_pointer,
pager,
max_local,
min_local,
payload_overflow_threshold_max,
payload_overflow_threshold_min,
usable_size,
)
}
/// When using this fu
pub fn cell_get_raw_pointer_region(&self) -> (usize, usize) {
let cell_start = match self.page_type() {
PageType::IndexInterior => 12,
PageType::TableInterior => 12,
PageType::IndexLeaf => 8,
PageType::TableLeaf => 8,
};
(self.offset + cell_start, self.cell_count() * 2)
/// The cell pointer array of a b-tree page immediately follows the b-tree page header.
/// Let K be the number of cells on the btree.
/// The cell pointer array consists of K 2-byte integer offsets to the cell contents.
/// The cell pointers are arranged in key order with:
/// - left-most cell (the cell with the smallest key) first and
/// - the right-most cell (the cell with the largest key) last.
pub fn cell_pointer_array_offset_and_size(&self) -> (usize, usize) {
let header_size = self.header_size();
(self.offset + header_size, self.cell_pointer_array_size())
}
/* Get region of a cell's payload */
pub fn cell_get_raw_region(
&self,
idx: usize,
max_local: usize,
min_local: usize,
payload_overflow_threshold_max: usize,
payload_overflow_threshold_min: usize,
usable_size: usize,
) -> (usize, usize) {
let buf = self.as_ptr();
let ncells = self.cell_count();
let cell_start = match self.page_type() {
PageType::IndexInterior => 12,
PageType::TableInterior => 12,
PageType::IndexLeaf => 8,
PageType::TableLeaf => 8,
};
let cell_pointer_array_start = self.header_size();
assert!(idx < ncells, "cell_get: idx out of bounds");
let cell_pointer = cell_start + (idx * 2);
let cell_pointer = cell_pointer_array_start + (idx * 2); // pointers are 2 bytes each
let cell_pointer = self.read_u16(cell_pointer) as usize;
let start = cell_pointer;
let len = match self.page_type() {
PageType::IndexInterior => {
let (len_payload, n_payload) = read_varint(&buf[cell_pointer + 4..]).unwrap();
let (overflows, to_read) =
payload_overflows(len_payload as usize, max_local, min_local, usable_size);
let (overflows, to_read) = payload_overflows(
len_payload as usize,
payload_overflow_threshold_max,
payload_overflow_threshold_min,
usable_size,
);
if overflows {
4 + to_read + n_payload + 4
} else {
@@ -493,8 +618,12 @@ impl PageContent {
}
PageType::IndexLeaf => {
let (len_payload, n_payload) = read_varint(&buf[cell_pointer..]).unwrap();
let (overflows, to_read) =
payload_overflows(len_payload as usize, max_local, min_local, usable_size);
let (overflows, to_read) = payload_overflows(
len_payload as usize,
payload_overflow_threshold_max,
payload_overflow_threshold_min,
usable_size,
);
if overflows {
to_read + n_payload + 4
} else {
@@ -504,8 +633,12 @@ impl PageContent {
PageType::TableLeaf => {
let (len_payload, n_payload) = read_varint(&buf[cell_pointer..]).unwrap();
let (_, n_rowid) = read_varint(&buf[cell_pointer + n_payload..]).unwrap();
let (overflows, to_read) =
payload_overflows(len_payload as usize, max_local, min_local, usable_size);
let (overflows, to_read) = payload_overflows(
len_payload as usize,
payload_overflow_threshold_max,
payload_overflow_threshold_min,
usable_size,
);
if overflows {
to_read + n_payload + n_rowid
} else {
@@ -1170,28 +1303,46 @@ pub fn begin_write_wal_header(io: &Rc<dyn File>, header: &WalHeader) -> Result<(
Ok(())
}
/*
Checks if payload will overflow a cell based on max local and
it will return the min size that will be stored in that case,
including overflow pointer
*/
/// Checks if payload will overflow a cell based on the maximum allowed size.
/// It will return the min size that will be stored in that case,
/// including overflow pointer
/// see e.g. https://github.com/sqlite/sqlite/blob/9591d3fe93936533c8c3b0dc4d025ac999539e11/src/dbstat.c#L371
pub fn payload_overflows(
payload_size: usize,
max_local: usize,
min_local: usize,
payload_overflow_threshold_max: usize,
payload_overflow_threshold_min: usize,
usable_size: usize,
) -> (bool, usize) {
if payload_size <= max_local {
if payload_size <= payload_overflow_threshold_max {
return (false, 0);
}
let mut space_left = min_local + (payload_size - min_local) % (usable_size - 4);
if space_left > max_local {
space_left = min_local;
let mut space_left = payload_overflow_threshold_min
+ (payload_size - payload_overflow_threshold_min) % (usable_size - 4);
if space_left > payload_overflow_threshold_max {
space_left = payload_overflow_threshold_min;
}
(true, space_left + 4)
}
/// The checksum is computed by interpreting the input as an even number of unsigned 32-bit integers: x(0) through x(N).
/// The 32-bit integers are big-endian if the magic number in the first 4 bytes of the WAL header is 0x377f0683
/// and the integers are little-endian if the magic number is 0x377f0682.
/// The checksum values are always stored in the frame header in a big-endian format regardless of which byte order is used to compute the checksum.
///
/// The checksum algorithm only works for content which is a multiple of 8 bytes in length.
/// In other words, if the inputs are x(0) through x(N) then N must be odd.
/// The checksum algorithm is as follows:
///
/// s0 = s1 = 0
/// for i from 0 to n-1 step 2:
/// s0 += x(i) + s1;
/// s1 += x(i+1) + s0;
/// endfor
///
/// The outputs s0 and s1 are both weighted checksums using Fibonacci weights in reverse order.
/// (The largest Fibonacci weight occurs on the first element of the sequence being summed.)
/// The s1 value spans all 32-bit integer terms of the sequence whereas s0 omits the final term.
pub fn checksum_wal(
buf: &[u8],
_wal_header: &WalHeader,

306
core/storage/wal.rs
View File

@@ -1,10 +1,12 @@
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::RwLock;
use std::{cell::RefCell, rc::Rc, sync::Arc};
use log::{debug, trace};
use crate::io::{File, SyncCompletion, IO};
use crate::result::LimboResult;
use crate::storage::sqlite3_ondisk::{
begin_read_wal_frame, begin_write_wal_frame, WAL_FRAME_HEADER_SIZE, WAL_HEADER_SIZE,
};
@@ -14,23 +16,119 @@ use crate::{Completion, Page};
use self::sqlite3_ondisk::{checksum_wal, PageContent, WAL_MAGIC_BE, WAL_MAGIC_LE};
use super::buffer_pool::BufferPool;
use super::page_cache::PageCacheKey;
use super::pager::{PageRef, Pager};
use super::sqlite3_ondisk::{self, begin_write_btree_page, WalHeader};
pub const READMARK_NOT_USED: u32 = 0xffffffff;
pub const NO_LOCK: u32 = 0;
pub const SHARED_LOCK: u32 = 1;
pub const WRITE_LOCK: u32 = 2;
pub enum CheckpointMode {
Passive,
Full,
Restart,
Truncate,
}
#[derive(Debug)]
struct LimboRwLock {
lock: AtomicU32,
nreads: AtomicU32,
value: AtomicU32,
}
impl LimboRwLock {
/// Shared lock. Returns true if it was successful, false if it couldn't lock it
pub fn read(&mut self) -> bool {
let lock = self.lock.load(Ordering::SeqCst);
match lock {
NO_LOCK => {
let res = self.lock.compare_exchange(
lock,
SHARED_LOCK,
Ordering::SeqCst,
Ordering::SeqCst,
);
let ok = res.is_ok();
if ok {
self.nreads.fetch_add(1, Ordering::SeqCst);
}
ok
}
SHARED_LOCK => {
self.nreads.fetch_add(1, Ordering::SeqCst);
true
}
WRITE_LOCK => false,
_ => unreachable!(),
}
}
/// Locks exlusively. Returns true if it was successful, false if it couldn't lock it
pub fn write(&mut self) -> bool {
let lock = self.lock.load(Ordering::SeqCst);
match lock {
NO_LOCK => {
let res = self.lock.compare_exchange(
lock,
WRITE_LOCK,
Ordering::SeqCst,
Ordering::SeqCst,
);
res.is_ok()
}
SHARED_LOCK => {
// no op
false
}
WRITE_LOCK => true,
_ => unreachable!(),
}
}
/// Unlock the current held lock.
pub fn unlock(&mut self) {
let lock = self.lock.load(Ordering::SeqCst);
match lock {
NO_LOCK => {}
SHARED_LOCK => {
let prev = self.nreads.fetch_sub(1, Ordering::SeqCst);
if prev == 1 {
let res = self.lock.compare_exchange(
lock,
NO_LOCK,
Ordering::SeqCst,
Ordering::SeqCst,
);
assert!(res.is_ok());
}
}
WRITE_LOCK => {
let res =
self.lock
.compare_exchange(lock, NO_LOCK, Ordering::SeqCst, Ordering::SeqCst);
assert!(res.is_ok());
}
_ => unreachable!(),
}
}
}
/// Write-ahead log (WAL).
pub trait Wal {
/// Begin a read transaction.
fn begin_read_tx(&mut self) -> Result<()>;
fn begin_read_tx(&mut self) -> Result<LimboResult>;
/// Begin a write transaction.
fn begin_write_tx(&mut self) -> Result<()>;
fn begin_write_tx(&mut self) -> Result<LimboResult>;
/// End a read transaction.
fn end_read_tx(&self) -> Result<()>;
fn end_read_tx(&self) -> Result<LimboResult>;
/// End a write transaction.
fn end_write_tx(&self) -> Result<()>;
fn end_write_tx(&self) -> Result<LimboResult>;
/// Find the latest frame containing a page.
fn find_frame(&self, page_id: u64) -> Result<Option<u64>>;
@@ -51,6 +149,7 @@ pub trait Wal {
&mut self,
pager: &Pager,
write_counter: Rc<RefCell<usize>>,
mode: CheckpointMode,
) -> Result<CheckpointStatus>;
fn sync(&mut self) -> Result<CheckpointStatus>;
fn get_max_frame(&self) -> u64;
@@ -96,6 +195,7 @@ struct OngoingCheckpoint {
current_page: u64,
}
#[allow(dead_code)]
pub struct WalFile {
io: Arc<dyn crate::io::IO>,
buffer_pool: Rc<BufferPool>,
@@ -108,12 +208,19 @@ pub struct WalFile {
ongoing_checkpoint: OngoingCheckpoint,
checkpoint_threshold: usize,
// min and max frames for this connection
/// This is the index to the read_lock in WalFileShared that we are holding. This lock contains
/// the max frame for this connection.
max_frame_read_lock_index: usize,
/// Max frame allowed to lookup range=(minframe..max_frame)
max_frame: u64,
/// Start of range to look for frames range=(minframe..max_frame)
min_frame: u64,
}
// TODO(pere): lock only important parts + pin WalFileShared
/// WalFileShared is the part of a WAL that will be shared between threads. A wal has information
/// that needs to be communicated between threads so this struct does the job.
#[allow(dead_code)]
pub struct WalFileShared {
wal_header: Arc<RwLock<sqlite3_ondisk::WalHeader>>,
min_frame: u64,
@@ -130,20 +237,94 @@ pub struct WalFileShared {
pages_in_frames: Vec<u64>,
last_checksum: (u32, u32), // Check of last frame in WAL, this is a cumulative checksum over all frames in the WAL
file: Rc<dyn File>,
/// read_locks is a list of read locks that can coexist with the max_frame nubmer stored in
/// value. There is a limited amount because and unbounded amount of connections could be
/// fatal. Therefore, for now we copy how SQLite behaves with limited amounts of read max
/// frames that is equal to 5
read_locks: [LimboRwLock; 5],
/// There is only one write allowed in WAL mode. This lock takes care of ensuring there is only
/// one used.
write_lock: LimboRwLock,
}
impl Wal for WalFile {
/// Begin a read transaction.
fn begin_read_tx(&mut self) -> Result<()> {
let shared = self.shared.read().unwrap();
fn begin_read_tx(&mut self) -> Result<LimboResult> {
let mut shared = self.shared.write().unwrap();
let max_frame_in_wal = shared.max_frame;
self.min_frame = shared.nbackfills + 1;
self.max_frame = shared.max_frame;
Ok(())
let mut max_read_mark = 0;
let mut max_read_mark_index = -1;
// Find the largest mark we can find, ignore frames that are impossible to be in range and
// that are not set
for (index, lock) in shared.read_locks.iter().enumerate() {
let this_mark = lock.value.load(Ordering::SeqCst);
if this_mark > max_read_mark && this_mark <= max_frame_in_wal as u32 {
max_read_mark = this_mark;
max_read_mark_index = index as i64;
}
}
// If we didn't find any mark, then let's add a new one
if max_read_mark_index == -1 {
for (index, lock) in shared.read_locks.iter_mut().enumerate() {
let busy = !lock.write();
if !busy {
// If this was busy then it must mean >1 threads tried to set this read lock
lock.value.store(max_frame_in_wal as u32, Ordering::SeqCst);
max_read_mark = max_frame_in_wal as u32;
max_read_mark_index = index as i64;
lock.unlock();
break;
}
}
}
if max_read_mark_index == -1 {
return Ok(LimboResult::Busy);
}
let lock = &mut shared.read_locks[max_read_mark_index as usize];
let busy = !lock.read();
if busy {
return Ok(LimboResult::Busy);
}
self.max_frame_read_lock_index = max_read_mark_index as usize;
self.max_frame = max_read_mark as u64;
self.min_frame = shared.nbackfills + 1;
log::trace!(
"begin_read_tx(min_frame={}, max_frame={}, lock={})",
self.min_frame,
self.max_frame,
self.max_frame_read_lock_index
);
Ok(LimboResult::Ok)
}
/// End a read transaction.
fn end_read_tx(&self) -> Result<()> {
Ok(())
fn end_read_tx(&self) -> Result<LimboResult> {
let mut shared = self.shared.write().unwrap();
let read_lock = &mut shared.read_locks[self.max_frame_read_lock_index];
read_lock.unlock();
Ok(LimboResult::Ok)
}
/// Begin a write transaction
fn begin_write_tx(&mut self) -> Result<LimboResult> {
let mut shared = self.shared.write().unwrap();
let busy = !shared.write_lock.write();
if busy {
return Ok(LimboResult::Busy);
}
Ok(LimboResult::Ok)
}
/// End a write transaction
fn end_write_tx(&self) -> Result<LimboResult> {
let mut shared = self.shared.write().unwrap();
shared.write_lock.unlock();
Ok(LimboResult::Ok)
}
/// Find the latest frame containing a page.
@@ -186,7 +367,11 @@ impl Wal for WalFile {
) -> Result<()> {
let page_id = page.get().id;
let mut shared = self.shared.write().unwrap();
let frame_id = shared.max_frame;
let frame_id = if shared.max_frame == 0 {
1
} else {
shared.max_frame
};
let offset = self.frame_offset(frame_id);
trace!(
"append_frame(frame={}, offset={}, page_id={})",
@@ -221,16 +406,6 @@ impl Wal for WalFile {
Ok(())
}
/// Begin a write transaction
fn begin_write_tx(&mut self) -> Result<()> {
Ok(())
}
/// End a write transaction
fn end_write_tx(&self) -> Result<()> {
Ok(())
}
fn should_checkpoint(&self) -> bool {
let shared = self.shared.read().unwrap();
let frame_id = shared.max_frame as usize;
@@ -241,7 +416,12 @@ impl Wal for WalFile {
&mut self,
pager: &Pager,
write_counter: Rc<RefCell<usize>>,
mode: CheckpointMode,
) -> Result<CheckpointStatus> {
assert!(
matches!(mode, CheckpointMode::Passive),
"only passive mode supported for now"
);
'checkpoint_loop: loop {
let state = self.ongoing_checkpoint.state;
log::debug!("checkpoint(state={:?})", state);
@@ -249,9 +429,29 @@ impl Wal for WalFile {
CheckpointState::Start => {
// TODO(pere): check what frames are safe to checkpoint between many readers!
self.ongoing_checkpoint.min_frame = self.min_frame;
self.ongoing_checkpoint.max_frame = self.max_frame;
let mut shared = self.shared.write().unwrap();
let max_frame_in_wal = shared.max_frame as u32;
let mut max_safe_frame = shared.max_frame;
for read_lock in shared.read_locks.iter_mut() {
let this_mark = read_lock.value.load(Ordering::SeqCst);
if this_mark < max_safe_frame as u32 {
let busy = !read_lock.write();
if !busy {
read_lock.value.store(max_frame_in_wal, Ordering::SeqCst);
read_lock.unlock();
} else {
max_safe_frame = this_mark as u64;
}
}
}
self.ongoing_checkpoint.max_frame = max_safe_frame;
self.ongoing_checkpoint.current_page = 0;
self.ongoing_checkpoint.state = CheckpointState::ReadFrame;
log::trace!(
"checkpoint_start(min_frame={}, max_frame={})",
self.ongoing_checkpoint.max_frame,
self.ongoing_checkpoint.min_frame
);
}
CheckpointState::ReadFrame => {
let shared = self.shared.read().unwrap();
@@ -272,8 +472,9 @@ impl Wal for WalFile {
.expect("page must be in frame cache if it's in list");
for frame in frames.iter().rev() {
// TODO: do proper selection of frames to checkpoint
if *frame >= self.ongoing_checkpoint.min_frame {
if *frame >= self.ongoing_checkpoint.min_frame
&& *frame <= self.ongoing_checkpoint.max_frame
{
log::debug!(
"checkpoint page(state={:?}, page={}, frame={})",
state,
@@ -328,10 +529,18 @@ impl Wal for WalFile {
return Ok(CheckpointStatus::IO);
}
let mut shared = self.shared.write().unwrap();
shared.frame_cache.clear();
shared.pages_in_frames.clear();
shared.max_frame = 0;
shared.nbackfills = 0;
let everything_backfilled =
shared.max_frame == self.ongoing_checkpoint.max_frame;
if everything_backfilled {
// Here we know that we backfilled everything, therefore we can safely
// reset the wal.
shared.frame_cache.clear();
shared.pages_in_frames.clear();
shared.max_frame = 0;
shared.nbackfills = 0;
} else {
shared.nbackfills = self.ongoing_checkpoint.max_frame;
}
self.ongoing_checkpoint.state = CheckpointState::Start;
return Ok(CheckpointStatus::Done);
}
@@ -412,10 +621,11 @@ impl WalFile {
syncing: Rc::new(RefCell::new(false)),
checkpoint_threshold: 1000,
page_size,
max_frame: 0,
min_frame: 0,
buffer_pool,
sync_state: RefCell::new(SyncState::NotSyncing),
max_frame: 0,
min_frame: 0,
max_frame_read_lock_index: 0,
}
}
@@ -488,6 +698,38 @@ impl WalFileShared {
last_checksum: checksum,
file,
pages_in_frames: Vec::new(),
read_locks: [
LimboRwLock {
lock: AtomicU32::new(NO_LOCK),
nreads: AtomicU32::new(0),
value: AtomicU32::new(READMARK_NOT_USED),
},
LimboRwLock {
lock: AtomicU32::new(NO_LOCK),
nreads: AtomicU32::new(0),
value: AtomicU32::new(READMARK_NOT_USED),
},
LimboRwLock {
lock: AtomicU32::new(NO_LOCK),
nreads: AtomicU32::new(0),
value: AtomicU32::new(READMARK_NOT_USED),
},
LimboRwLock {
lock: AtomicU32::new(NO_LOCK),
nreads: AtomicU32::new(0),
value: AtomicU32::new(READMARK_NOT_USED),
},
LimboRwLock {
lock: AtomicU32::new(NO_LOCK),
nreads: AtomicU32::new(0),
value: AtomicU32::new(READMARK_NOT_USED),
},
],
write_lock: LimboRwLock {
lock: AtomicU32::new(NO_LOCK),
nreads: AtomicU32::new(0),
value: AtomicU32::new(READMARK_NOT_USED),
},
};
Ok(Arc::new(RwLock::new(shared)))
}

21
core/translate/delete.rs Normal file
View File

@@ -0,0 +1,21 @@
use crate::translate::emitter::emit_program;
use crate::translate::optimizer::optimize_plan;
use crate::translate::planner::prepare_delete_plan;
use crate::{schema::Schema, storage::sqlite3_ondisk::DatabaseHeader, vdbe::Program};
use crate::{Connection, Result};
use sqlite3_parser::ast::{Expr, Limit, QualifiedName};
use std::rc::Weak;
use std::{cell::RefCell, rc::Rc};
pub fn translate_delete(
schema: &Schema,
tbl_name: &QualifiedName,
where_clause: Option<Expr>,
limit: Option<Limit>,
database_header: Rc<RefCell<DatabaseHeader>>,
connection: Weak<Connection>,
) -> Result<Program> {
let delete_plan = prepare_delete_plan(schema, tbl_name, where_clause, limit)?;
let optimized_plan = optimize_plan(delete_plan)?;
emit_program(database_header, optimized_plan, connection)
}

238
core/translate/emitter.rs
View File

@@ -9,18 +9,18 @@ use sqlite3_parser::ast::{self};
use crate::schema::{Column, PseudoTable, Table};
use crate::storage::sqlite3_ondisk::DatabaseHeader;
use crate::translate::plan::{IterationDirection, Search};
use crate::translate::plan::{DeletePlan, IterationDirection, Plan, Search};
use crate::types::{OwnedRecord, OwnedValue};
use crate::util::exprs_are_equivalent;
use crate::vdbe::builder::ProgramBuilder;
use crate::vdbe::{BranchOffset, Insn, Program};
use crate::vdbe::{insn::Insn, BranchOffset, Program};
use crate::{Connection, Result};
use super::expr::{
translate_aggregation, translate_aggregation_groupby, translate_condition_expr, translate_expr,
ConditionMetadata,
};
use super::plan::{Aggregate, BTreeTableReference, Direction, GroupBy, Plan};
use super::plan::{Aggregate, BTreeTableReference, Direction, GroupBy, SelectPlan};
use super::plan::{ResultSetColumn, SourceOperator};
// Metadata for handling LEFT JOIN operations
@@ -101,6 +101,16 @@ pub struct Metadata {
pub result_columns_to_skip_in_orderby_sorter: Option<Vec<usize>>,
}
/// Used to distinguish database operations
#[allow(clippy::upper_case_acronyms, dead_code)]
#[derive(Debug, Clone)]
pub enum OperationMode {
SELECT,
INSERT,
UPDATE,
DELETE,
}
/// Initialize the program with basic setup and return initial metadata and labels
fn prologue() -> Result<(ProgramBuilder, Metadata, BranchOffset, BranchOffset)> {
let mut program = ProgramBuilder::new();
@@ -164,7 +174,18 @@ fn epilogue(
/// Takes a query plan and generates the corresponding bytecode program
pub fn emit_program(
database_header: Rc<RefCell<DatabaseHeader>>,
mut plan: Plan,
plan: Plan,
connection: Weak<Connection>,
) -> Result<Program> {
match plan {
Plan::Select(plan) => emit_program_for_select(database_header, plan, connection),
Plan::Delete(plan) => emit_program_for_delete(database_header, plan, connection),
}
}
fn emit_program_for_select(
database_header: Rc<RefCell<DatabaseHeader>>,
mut plan: SelectPlan,
connection: Weak<Connection>,
) -> Result<Program> {
let (mut program, mut metadata, init_label, start_offset) = prologue()?;
@@ -201,7 +222,12 @@ pub fn emit_program(
if let Some(ref mut group_by) = plan.group_by {
init_group_by(&mut program, group_by, &plan.aggregates, &mut metadata)?;
}
init_source(&mut program, &plan.source, &mut metadata)?;
init_source(
&mut program,
&plan.source,
&mut metadata,
&OperationMode::SELECT,
)?;
// Set up main query execution loop
open_loop(
@@ -272,6 +298,63 @@ pub fn emit_program(
Ok(program.build(database_header, connection))
}
fn emit_program_for_delete(
database_header: Rc<RefCell<DatabaseHeader>>,
mut plan: DeletePlan,
connection: Weak<Connection>,
) -> Result<Program> {
let (mut program, mut metadata, init_label, start_offset) = prologue()?;
// No rows will be read from source table loops if there is a constant false condition eg. WHERE 0
let skip_loops_label = if plan.contains_constant_false_condition {
let skip_loops_label = program.allocate_label();
program.emit_insn_with_label_dependency(
Insn::Goto {
target_pc: skip_loops_label,
},
skip_loops_label,
);
Some(skip_loops_label)
} else {
None
};
// Initialize cursors and other resources needed for query execution
init_source(
&mut program,
&plan.source,
&mut metadata,
&OperationMode::DELETE,
)?;
// Set up main query execution loop
open_loop(
&mut program,
&mut plan.source,
&plan.referenced_tables,
&mut metadata,
)?;
emit_delete_insns(&mut program, &plan.source, &plan.limit, &metadata)?;
// Clean up and close the main execution loop
close_loop(
&mut program,
&plan.source,
&mut metadata,
&plan.referenced_tables,
)?;
if let Some(skip_loops_label) = skip_loops_label {
program.resolve_label(skip_loops_label, program.offset());
}
// Finalize program
epilogue(&mut program, &mut metadata, init_label, start_offset)?;
Ok(program.build(database_header, connection))
}
/// Initialize resources needed for ORDER BY processing
fn init_order_by(
program: &mut ProgramBuilder,
@@ -385,6 +468,7 @@ fn init_source(
program: &mut ProgramBuilder,
source: &SourceOperator,
metadata: &mut Metadata,
mode: &OperationMode,
) -> Result<()> {
match source {
SourceOperator::Join {
@@ -402,10 +486,10 @@ fn init_source(
};
metadata.left_joins.insert(*id, lj_metadata);
}
init_source(program, left, metadata)?;
init_source(program, right, metadata)?;
init_source(program, left, metadata, mode)?;
init_source(program, right, metadata, mode)?;
return Ok(());
Ok(())
}
SourceOperator::Scan {
id,
@@ -419,13 +503,28 @@ fn init_source(
let root_page = table_reference.table.root_page;
let next_row_label = program.allocate_label();
metadata.next_row_labels.insert(*id, next_row_label);
program.emit_insn(Insn::OpenReadAsync {
cursor_id,
root_page,
});
program.emit_insn(Insn::OpenReadAwait);
return Ok(());
match mode {
OperationMode::SELECT => {
program.emit_insn(Insn::OpenReadAsync {
cursor_id,
root_page,
});
program.emit_insn(Insn::OpenReadAwait {});
}
OperationMode::DELETE => {
program.emit_insn(Insn::OpenWriteAsync {
cursor_id,
root_page,
});
program.emit_insn(Insn::OpenWriteAwait {});
}
_ => {
unimplemented!()
}
}
Ok(())
}
SourceOperator::Search {
id,
@@ -442,27 +541,54 @@ fn init_source(
metadata.next_row_labels.insert(*id, next_row_label);
program.emit_insn(Insn::OpenReadAsync {
cursor_id: table_cursor_id,
root_page: table_reference.table.root_page,
});
program.emit_insn(Insn::OpenReadAwait);
match mode {
OperationMode::SELECT => {
program.emit_insn(Insn::OpenReadAsync {
cursor_id: table_cursor_id,
root_page: table_reference.table.root_page,
});
program.emit_insn(Insn::OpenReadAwait {});
}
OperationMode::DELETE => {
program.emit_insn(Insn::OpenWriteAsync {
cursor_id: table_cursor_id,
root_page: table_reference.table.root_page,
});
program.emit_insn(Insn::OpenWriteAwait {});
}
_ => {
unimplemented!()
}
}
if let Search::IndexSearch { index, .. } = search {
let index_cursor_id = program
.alloc_cursor_id(Some(index.name.clone()), Some(Table::Index(index.clone())));
program.emit_insn(Insn::OpenReadAsync {
cursor_id: index_cursor_id,
root_page: index.root_page,
});
program.emit_insn(Insn::OpenReadAwait);
match mode {
OperationMode::SELECT => {
program.emit_insn(Insn::OpenReadAsync {
cursor_id: index_cursor_id,
root_page: index.root_page,
});
program.emit_insn(Insn::OpenReadAwait);
}
OperationMode::DELETE => {
program.emit_insn(Insn::OpenWriteAsync {
cursor_id: index_cursor_id,
root_page: index.root_page,
});
program.emit_insn(Insn::OpenWriteAwait {});
}
_ => {
unimplemented!()
}
}
}
return Ok(());
}
SourceOperator::Nothing => {
return Ok(());
Ok(())
}
SourceOperator::Nothing => Ok(()),
}
}
@@ -811,7 +937,7 @@ pub enum InnerLoopEmitTarget<'a> {
/// At this point the cursors for all tables have been opened and rewound.
fn inner_loop_emit(
program: &mut ProgramBuilder,
plan: &mut Plan,
plan: &mut SelectPlan,
metadata: &mut Metadata,
) -> Result<()> {
// if we have a group by, we emit a record into the group by sorter.
@@ -1121,6 +1247,60 @@ fn close_loop(
}
}
fn emit_delete_insns(
program: &mut ProgramBuilder,
source: &SourceOperator,
limit: &Option<usize>,
metadata: &Metadata,
) -> Result<()> {
let cursor_id = match source {
SourceOperator::Scan {
table_reference, ..
} => program.resolve_cursor_id(&table_reference.table_identifier),
SourceOperator::Search {
table_reference,
search,
..
} => match search {
Search::RowidEq { .. } | Search::RowidSearch { .. } => {
program.resolve_cursor_id(&table_reference.table_identifier)
}
Search::IndexSearch { index, .. } => program.resolve_cursor_id(&index.name),
},
_ => return Ok(()),
};
// Emit the instructions to delete the row
let key_reg = program.alloc_register();
program.emit_insn(Insn::RowId {
cursor_id,
dest: key_reg,
});
program.emit_insn(Insn::DeleteAsync { cursor_id });
program.emit_insn(Insn::DeleteAwait { cursor_id });
if let Some(limit) = limit {
let limit_reg = program.alloc_register();
program.emit_insn(Insn::Integer {
value: *limit as i64,
dest: limit_reg,
});
program.mark_last_insn_constant();
let jump_label_on_limit_reached = metadata
.termination_label_stack
.last()
.expect("termination_label_stack should not be empty.");
program.emit_insn_with_label_dependency(
Insn::DecrJumpZero {
reg: limit_reg,
target_pc: *jump_label_on_limit_reached,
},
*jump_label_on_limit_reached,
)
}
Ok(())
}
/// Emits the bytecode for processing a GROUP BY clause.
/// This is called when the main query execution loop has finished processing,
/// and we now have data in the GROUP BY sorter.

152
core/translate/expr.rs
View File

@@ -1,11 +1,13 @@
use sqlite3_parser::ast::{self, UnaryOperator};
#[cfg(feature = "uuid")]
use crate::ext::{ExtFunc, UuidFunc};
#[cfg(feature = "json")]
use crate::function::JsonFunc;
use crate::function::{AggFunc, Func, FuncCtx, MathFuncArity, ScalarFunc};
use crate::schema::Type;
use crate::util::normalize_ident;
use crate::vdbe::{builder::ProgramBuilder, BranchOffset, Insn};
use crate::util::{exprs_are_equivalent, normalize_ident};
use crate::vdbe::{builder::ProgramBuilder, insn::Insn, BranchOffset};
use crate::Result;
use super::plan::{Aggregate, BTreeTableReference};
@@ -554,10 +556,7 @@ pub fn translate_expr(
) -> Result<usize> {
if let Some(precomputed_exprs_to_registers) = precomputed_exprs_to_registers {
for (precomputed_expr, reg) in precomputed_exprs_to_registers.iter() {
// TODO: implement a custom equality check for expressions
// there are lots of examples where this breaks, even simple ones like
// sum(x) != SUM(x)
if expr == *precomputed_expr {
if exprs_are_equivalent(expr, precomputed_expr) {
program.emit_insn(Insn::Copy {
src_reg: *reg,
dst_reg: target_register,
@@ -694,6 +693,13 @@ pub fn translate_expr(
dest: target_register,
});
}
ast::Operator::Modulus => {
program.emit_insn(Insn::Remainder {
lhs: e1_reg,
rhs: e2_reg,
dest: target_register,
});
}
ast::Operator::BitwiseAnd => {
program.emit_insn(Insn::BitAnd {
lhs: e1_reg,
@@ -930,6 +936,51 @@ pub fn translate_expr(
});
Ok(target_register)
}
JsonFunc::JsonArrayLength => {
let args = if let Some(args) = args {
if args.len() > 2 {
crate::bail_parse_error!(
"{} function with wrong number of arguments",
j.to_string()
)
}
args
} else {
crate::bail_parse_error!(
"{} function with no arguments",
j.to_string()
);
};
let json_reg = program.alloc_register();
let path_reg = program.alloc_register();
translate_expr(
program,
referenced_tables,
&args[0],
json_reg,
precomputed_exprs_to_registers,
)?;
if args.len() == 2 {
translate_expr(
program,
referenced_tables,
&args[1],
path_reg,
precomputed_exprs_to_registers,
)?;
}
program.emit_insn(Insn::Function {
constant_mask: 0,
start_reg: json_reg,
dest: target_register,
func: func_ctx,
});
Ok(target_register)
}
},
Func::Scalar(srf) => {
match srf {
@@ -1572,7 +1623,7 @@ pub fn translate_expr(
program.emit_insn(Insn::Copy {
src_reg: output_register,
dst_reg: target_register,
amount: 1,
amount: 0,
});
Ok(target_register)
}
@@ -1629,6 +1680,93 @@ pub fn translate_expr(
}
}
}
Func::Extension(ext_func) => match ext_func {
#[cfg(feature = "uuid")]
ExtFunc::Uuid(ref uuid_fn) => match uuid_fn {
UuidFunc::UuidStr | UuidFunc::UuidBlob | UuidFunc::Uuid7TS => {
let args = if let Some(args) = args {
if args.len() != 1 {
crate::bail_parse_error!(
"{} function with not exactly 1 argument",
ext_func.to_string()
);
}
args
} else {
crate::bail_parse_error!(
"{} function with no arguments",
ext_func.to_string()
);
};
let regs = program.alloc_register();
translate_expr(
program,
referenced_tables,
&args[0],
regs,
precomputed_exprs_to_registers,
)?;
program.emit_insn(Insn::Function {
constant_mask: 0,
start_reg: regs,
dest: target_register,
func: func_ctx,
});
Ok(target_register)
}
UuidFunc::Uuid4 | UuidFunc::Uuid4Str => {
if args.is_some() {
crate::bail_parse_error!(
"{} function with arguments",
ext_func.to_string()
);
}
let regs = program.alloc_register();
program.emit_insn(Insn::Function {
constant_mask: 0,
start_reg: regs,
dest: target_register,
func: func_ctx,
});
Ok(target_register)
}
UuidFunc::Uuid7 => {
let args = match args {
Some(args) if args.len() > 1 => crate::bail_parse_error!(
"{} function with more than 1 argument",
ext_func.to_string()
),
Some(args) => args,
None => &vec![],
};
let mut start_reg = None;
if let Some(arg) = args.first() {
let reg = program.alloc_register();
start_reg = Some(reg);
translate_expr(
program,
referenced_tables,
arg,
reg,
precomputed_exprs_to_registers,
)?;
if let ast::Expr::Literal(_) = arg {
program.mark_last_insn_constant()
}
}
program.emit_insn(Insn::Function {
constant_mask: 0,
start_reg: start_reg.unwrap_or(target_register),
dest: target_register,
func: func_ctx,
});
Ok(target_register)
}
},
#[allow(unreachable_patterns)]
_ => unreachable!("{ext_func} not implemented yet"),
},
Func::Math(math_func) => match math_func.arity() {
MathFuncArity::Nullary => {
if args.is_some() {

434
core/translate/insert.rs
View File

@@ -2,32 +2,187 @@ use std::rc::Weak;
use std::{cell::RefCell, ops::Deref, rc::Rc};
use sqlite3_parser::ast::{
DistinctNames, InsertBody, QualifiedName, ResolveType, ResultColumn, With,
DistinctNames, Expr, InsertBody, QualifiedName, ResolveType, ResultColumn, With,
};
use crate::error::SQLITE_CONSTRAINT_PRIMARYKEY;
use crate::util::normalize_ident;
use crate::{
schema::{Schema, Table},
schema::{Column, Schema, Table},
storage::sqlite3_ondisk::DatabaseHeader,
translate::expr::translate_expr,
vdbe::{builder::ProgramBuilder, Insn, Program},
vdbe::{builder::ProgramBuilder, insn::Insn, Program},
};
use crate::{Connection, Result};
#[derive(Debug)]
/// Represents how a column should be populated during an INSERT.
/// Contains both the column definition and optionally the index into the VALUES tuple.
struct ColumnMapping<'a> {
/// Reference to the column definition from the table schema
column: &'a Column,
/// If Some(i), use the i-th value from the VALUES tuple
/// If None, use NULL (column was not specified in INSERT statement)
value_index: Option<usize>,
}
/// Resolves how each column in a table should be populated during an INSERT.
/// Returns a Vec of ColumnMapping, one for each column in the table's schema.
///
/// For each column, specifies:
/// 1. The column definition (type, constraints, etc)
/// 2. Where to get the value from:
/// - Some(i) -> use i-th value from the VALUES tuple
/// - None -> use NULL (column wasn't specified in INSERT)
///
/// Two cases are handled:
/// 1. No column list specified (INSERT INTO t VALUES ...):
/// - Values are assigned to columns in table definition order
/// - If fewer values than columns, remaining columns map to None
/// 2. Column list specified (INSERT INTO t (col1, col3) VALUES ...):
/// - Named columns map to their corresponding value index
/// - Unspecified columns map to None
fn resolve_columns_for_insert<'a>(
table: &'a Table,
columns: &Option<DistinctNames>,
values: &[Vec<Expr>],
) -> Result<Vec<ColumnMapping<'a>>> {
if values.is_empty() {
crate::bail_parse_error!("no values to insert");
}
let table_columns = table.columns();
// Case 1: No columns specified - map values to columns in order
if columns.is_none() {
let num_values = values[0].len();
if num_values > table_columns.len() {
crate::bail_parse_error!(
"table {} has {} columns but {} values were supplied",
table.get_name(),
table_columns.len(),
num_values
);
}
// Verify all value tuples have same length
for value in values.iter().skip(1) {
if value.len() != num_values {
crate::bail_parse_error!("all VALUES must have the same number of terms");
}
}
// Map each column to either its corresponding value index or None
return Ok(table_columns
.iter()
.enumerate()
.map(|(i, col)| ColumnMapping {
column: col,
value_index: if i < num_values { Some(i) } else { None },
})
.collect());
}
// Case 2: Columns specified - map named columns to their values
let mut mappings: Vec<_> = table_columns
.iter()
.map(|col| ColumnMapping {
column: col,
value_index: None,
})
.collect();
// Map each named column to its value index
for (value_index, column_name) in columns.as_ref().unwrap().iter().enumerate() {
let column_name = normalize_ident(column_name.0.as_str());
let table_index = table_columns
.iter()
.position(|c| c.name.eq_ignore_ascii_case(&column_name));
if table_index.is_none() {
crate::bail_parse_error!(
"table {} has no column named {}",
table.get_name(),
column_name
);
}
mappings[table_index.unwrap()].value_index = Some(value_index);
}
Ok(mappings)
}
/// Populates the column registers with values for a single row
fn populate_column_registers(
program: &mut ProgramBuilder,
value: &[Expr],
column_mappings: &[ColumnMapping],
column_registers_start: usize,
inserting_multiple_rows: bool,
rowid_reg: usize,
) -> Result<()> {
for (i, mapping) in column_mappings.iter().enumerate() {
let target_reg = column_registers_start + i;
// Column has a value in the VALUES tuple
if let Some(value_index) = mapping.value_index {
// When inserting a single row, SQLite writes the value provided for the rowid alias column (INTEGER PRIMARY KEY)
// directly into the rowid register and writes a NULL into the rowid alias column. Not sure why this only happens
// in the single row case, but let's copy it.
let write_directly_to_rowid_reg =
mapping.column.is_rowid_alias && !inserting_multiple_rows;
let reg = if write_directly_to_rowid_reg {
rowid_reg
} else {
target_reg
};
translate_expr(
program,
None,
value.get(value_index).expect("value index out of bounds"),
reg,
None,
)?;
if write_directly_to_rowid_reg {
program.emit_insn(Insn::SoftNull { reg: target_reg });
}
} else {
// Column was not specified - use NULL if it is nullable, otherwise error
// Rowid alias columns can be NULL because we will autogenerate a rowid in that case.
let is_nullable = !mapping.column.primary_key || mapping.column.is_rowid_alias;
if is_nullable {
program.emit_insn(Insn::Null {
dest: target_reg,
dest_end: None,
});
program.mark_last_insn_constant();
} else {
crate::bail_parse_error!("column {} is not nullable", mapping.column.name);
}
}
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub fn translate_insert(
schema: &Schema,
with: &Option<With>,
or_conflict: &Option<ResolveType>,
on_conflict: &Option<ResolveType>,
tbl_name: &QualifiedName,
_columns: &Option<DistinctNames>,
columns: &Option<DistinctNames>,
body: &InsertBody,
_returning: &Option<Vec<ResultColumn>>,
database_header: Rc<RefCell<DatabaseHeader>>,
connection: Weak<Connection>,
) -> Result<Program> {
assert!(with.is_none());
assert!(or_conflict.is_none());
if with.is_some() {
crate::bail_parse_error!("WITH clause is not supported");
}
if on_conflict.is_some() {
crate::bail_parse_error!("ON CONFLICT clause is not supported");
}
let mut program = ProgramBuilder::new();
let init_label = program.allocate_label();
program.emit_insn_with_label_dependency(
@@ -46,6 +201,10 @@ pub fn translate_insert(
None => crate::bail_corrupt_error!("Parse error: no such table: {}", table_name),
};
let table = Rc::new(Table::BTree(table));
if !table.has_rowid() {
crate::bail_parse_error!("INSERT into WITHOUT ROWID table is not supported");
}
let cursor_id = program.alloc_cursor_id(
Some(table_name.0.clone()),
Some(table.clone().deref().clone()),
@@ -55,18 +214,49 @@ pub fn translate_insert(
Table::Index(index) => index.root_page,
Table::Pseudo(_) => todo!(),
};
let values = match body {
InsertBody::Select(select, None) => match &select.body.select {
sqlite3_parser::ast::OneSelect::Values(values) => values,
_ => todo!(),
},
_ => todo!(),
};
let mut num_cols = table.columns().len();
if table.has_rowid() {
num_cols += 1;
}
// column_registers_start[0] == rowid if has rowid
let column_registers_start = program.alloc_registers(num_cols);
let column_mappings = resolve_columns_for_insert(&table, columns, values)?;
// Check if rowid was provided (through INTEGER PRIMARY KEY as a rowid alias)
let rowid_alias_index = table.columns().iter().position(|c| c.is_rowid_alias);
let has_user_provided_rowid = {
assert!(column_mappings.len() == table.columns().len());
if let Some(index) = rowid_alias_index {
column_mappings[index].value_index.is_some()
} else {
false
}
};
// Coroutine for values
let yield_reg = program.alloc_register();
let jump_on_definition_label = program.allocate_label();
{
// allocate a register for each column in the table. if not provided by user, they will simply be set as null.
// allocate an extra register for rowid regardless of whether user provided a rowid alias column.
let num_cols = table.columns().len();
let rowid_reg = program.alloc_registers(num_cols + 1);
let column_registers_start = rowid_reg + 1;
let rowid_alias_reg = {
if has_user_provided_rowid {
Some(column_registers_start + rowid_alias_index.unwrap())
} else {
None
}
};
let record_register = program.alloc_register();
let halt_label = program.allocate_label();
let mut loop_start_offset = 0;
let inserting_multiple_rows = values.len() > 1;
// Multiple rows - use coroutine for value population
if inserting_multiple_rows {
let yield_reg = program.alloc_register();
let jump_on_definition_label = program.allocate_label();
program.emit_insn_with_label_dependency(
Insn::InitCoroutine {
yield_reg,
@@ -75,134 +265,154 @@ pub fn translate_insert(
},
jump_on_definition_label,
);
match body {
InsertBody::Select(select, None) => match &select.body.select {
sqlite3_parser::ast::OneSelect::Select {
distinctness: _,
columns: _,
from: _,
where_clause: _,
group_by: _,
window_clause: _,
} => todo!(),
sqlite3_parser::ast::OneSelect::Values(values) => {
for value in values {
for (col, expr) in value.iter().enumerate() {
let mut col = col;
if table.has_rowid() {
col += 1;
}
translate_expr(
&mut program,
None,
expr,
column_registers_start + col,
None,
)?;
}
program.emit_insn(Insn::Yield {
yield_reg,
end_offset: 0,
});
}
}
},
InsertBody::DefaultValues => todo!("default values not yet supported"),
_ => todo!(),
for value in values {
populate_column_registers(
&mut program,
value,
&column_mappings,
column_registers_start,
true,
rowid_reg,
)?;
program.emit_insn(Insn::Yield {
yield_reg,
end_offset: 0,
});
}
program.emit_insn(Insn::EndCoroutine { yield_reg });
program.resolve_label(jump_on_definition_label, program.offset());
program.emit_insn(Insn::OpenWriteAsync {
cursor_id,
root_page,
});
program.emit_insn(Insn::OpenWriteAwait {});
// Main loop
// FIXME: rollback is not implemented. E.g. if you insert 2 rows and one fails to unique constraint violation,
// the other row will still be inserted.
loop_start_offset = program.offset();
program.emit_insn_with_label_dependency(
Insn::Yield {
yield_reg,
end_offset: halt_label,
},
halt_label,
);
} else {
// Single row - populate registers directly
program.emit_insn(Insn::OpenWriteAsync {
cursor_id,
root_page,
});
program.emit_insn(Insn::OpenWriteAwait {});
populate_column_registers(
&mut program,
&values[0],
&column_mappings,
column_registers_start,
false,
rowid_reg,
)?;
}
program.resolve_label(jump_on_definition_label, program.offset());
program.emit_insn(Insn::OpenWriteAsync {
cursor_id,
root_page,
});
program.emit_insn(Insn::OpenWriteAwait {});
// Main loop
let record_register = program.alloc_register();
let halt_label = program.allocate_label();
let loop_start_offset = program.offset();
program.emit_insn_with_label_dependency(
Insn::Yield {
yield_reg,
end_offset: halt_label,
},
halt_label,
);
if table.has_rowid() {
let row_id_reg = column_registers_start;
if let Some(rowid_alias_column) = table.get_rowid_alias_column() {
let key_reg = column_registers_start + 1 + rowid_alias_column.0;
// copy key to rowid
// Common record insertion logic for both single and multiple rows
let check_rowid_is_integer_label = rowid_alias_reg.and(Some(program.allocate_label()));
if let Some(reg) = rowid_alias_reg {
// for the row record, the rowid alias column (INTEGER PRIMARY KEY) is always set to NULL
// and its value is copied to the rowid register. in the case where a single row is inserted,
// the value is written directly to the rowid register (see populate_column_registers()).
// again, not sure why this only happens in the single row case, but let's mimic sqlite.
// in the single row case we save a Copy instruction, but in the multiple rows case we do
// it here in the loop.
if inserting_multiple_rows {
program.emit_insn(Insn::Copy {
src_reg: key_reg,
dst_reg: row_id_reg,
amount: 0,
src_reg: reg,
dst_reg: rowid_reg,
amount: 0, // TODO: rename 'amount' to something else; amount==0 means 1
});
program.emit_insn(Insn::SoftNull { reg: key_reg });
// for the row record, the rowid alias column is always set to NULL
program.emit_insn(Insn::SoftNull { reg });
}
let notnull_label = program.allocate_label();
// the user provided rowid value might itself be NULL. If it is, we create a new rowid on the next instruction.
program.emit_insn_with_label_dependency(
Insn::NotNull {
reg: row_id_reg,
target_pc: notnull_label,
reg: rowid_reg,
target_pc: check_rowid_is_integer_label.unwrap(),
},
notnull_label,
check_rowid_is_integer_label.unwrap(),
);
program.emit_insn(Insn::NewRowid {
cursor: cursor_id,
rowid_reg: row_id_reg,
prev_largest_reg: 0,
});
}
program.resolve_label(notnull_label, program.offset());
program.emit_insn(Insn::MustBeInt { reg: row_id_reg });
// Create new rowid if a) not provided by user or b) provided by user but is NULL
program.emit_insn(Insn::NewRowid {
cursor: cursor_id,
rowid_reg: rowid_reg,
prev_largest_reg: 0,
});
if let Some(must_be_int_label) = check_rowid_is_integer_label {
program.resolve_label(must_be_int_label, program.offset());
// If the user provided a rowid, it must be an integer.
program.emit_insn(Insn::MustBeInt { reg: rowid_reg });
}
// Check uniqueness constraint for rowid if it was provided by user.
// When the DB allocates it there are no need for separate uniqueness checks.
if has_user_provided_rowid {
let make_record_label = program.allocate_label();
program.emit_insn_with_label_dependency(
Insn::NotExists {
cursor: cursor_id,
rowid_reg: row_id_reg,
rowid_reg: rowid_reg,
target_pc: make_record_label,
},
make_record_label,
);
// TODO: rollback
let rowid_column_name = if let Some(index) = rowid_alias_index {
table.column_index_to_name(index).unwrap()
} else {
"rowid"
};
program.emit_insn(Insn::Halt {
err_code: SQLITE_CONSTRAINT_PRIMARYKEY,
description: format!(
"{}.{}",
table.get_name(),
table.column_index_to_name(0).unwrap()
),
description: format!("{}.{}", table.get_name(), rowid_column_name),
});
program.resolve_label(make_record_label, program.offset());
program.emit_insn(Insn::MakeRecord {
start_reg: column_registers_start + 1,
count: num_cols - 1,
dest_reg: record_register,
});
program.emit_insn(Insn::InsertAsync {
cursor: cursor_id,
key_reg: column_registers_start,
record_reg: record_register,
flag: 0,
});
program.emit_insn(Insn::InsertAwait { cursor_id });
}
program.emit_insn(Insn::Goto {
target_pc: loop_start_offset,
// Create and insert the record
program.emit_insn(Insn::MakeRecord {
start_reg: column_registers_start,
count: num_cols,
dest_reg: record_register,
});
program.emit_insn(Insn::InsertAsync {
cursor: cursor_id,
key_reg: rowid_reg,
record_reg: record_register,
flag: 0,
});
program.emit_insn(Insn::InsertAwait { cursor_id });
if inserting_multiple_rows {
// For multiple rows, loop back
program.emit_insn(Insn::Goto {
target_pc: loop_start_offset,
});
}
program.resolve_label(halt_label, program.offset());
program.emit_insn(Insn::Halt {
err_code: 0,
description: String::new(),
});
program.resolve_label(init_label, program.offset());
program.emit_insn(Insn::Transaction { write: true });
program.emit_constant_insns();

35
core/translate/mod.rs
View File

@@ -7,6 +7,7 @@
//! a SELECT statement will be translated into a sequence of instructions that
//! will read rows from the database and filter them according to a WHERE clause.
pub(crate) mod delete;
pub(crate) mod emitter;
pub(crate) mod expr;
pub(crate) mod insert;
@@ -15,20 +16,20 @@ pub(crate) mod plan;
pub(crate) mod planner;
pub(crate) mod select;
use std::cell::RefCell;
use std::fmt::Display;
use std::rc::{Rc, Weak};
use std::str::FromStr;
use crate::schema::Schema;
use crate::storage::pager::Pager;
use crate::storage::sqlite3_ondisk::{DatabaseHeader, MIN_PAGE_CACHE_SIZE};
use crate::vdbe::{builder::ProgramBuilder, Insn, Program};
use crate::translate::delete::translate_delete;
use crate::vdbe::{builder::ProgramBuilder, insn::Insn, Program};
use crate::{bail_parse_error, Connection, Result};
use insert::translate_insert;
use select::translate_select;
use sqlite3_parser::ast::fmt::ToTokens;
use sqlite3_parser::ast::{self, PragmaName};
use std::cell::RefCell;
use std::fmt::Display;
use std::rc::{Rc, Weak};
use std::str::FromStr;
/// Translate SQL statement into bytecode program.
pub fn translate(
@@ -68,7 +69,19 @@ pub fn translate(
ast::Stmt::CreateVirtualTable { .. } => {
bail_parse_error!("CREATE VIRTUAL TABLE not supported yet")
}
ast::Stmt::Delete { .. } => bail_parse_error!("DELETE not supported yet"),
ast::Stmt::Delete {
tbl_name,
where_clause,
limit,
..
} => translate_delete(
schema,
&tbl_name,
where_clause,
limit,
database_header,
connection,
),
ast::Stmt::Detach(_) => bail_parse_error!("DETACH not supported yet"),
ast::Stmt::DropIndex { .. } => bail_parse_error!("DROP INDEX not supported yet"),
ast::Stmt::DropTable { .. } => bail_parse_error!("DROP TABLE not supported yet"),
@@ -369,7 +382,6 @@ fn update_pragma(
query_pragma("journal_mode", header, program)?;
Ok(())
}
_ => todo!("pragma `{name}`"),
}
}
@@ -386,7 +398,7 @@ fn query_pragma(
match pragma {
PragmaName::CacheSize => {
program.emit_insn(Insn::Integer {
value: database_header.borrow().default_cache_size.into(),
value: database_header.borrow().default_page_cache_size.into(),
dest: register,
});
}
@@ -396,9 +408,6 @@ fn query_pragma(
dest: register,
});
}
_ => {
todo!("pragma `{name}`");
}
}
program.emit_insn(Insn::ResultRow {
@@ -424,7 +433,7 @@ fn update_cache_size(value: i64, header: Rc<RefCell<DatabaseHeader>>, pager: Rc<
}
// update in-memory header
header.borrow_mut().default_cache_size = cache_size_unformatted
header.borrow_mut().default_page_cache_size = cache_size_unformatted
.try_into()
.unwrap_or_else(|_| panic!("invalid value, too big for a i32 {}", value));

83
core/translate/optimizer.rs
View File

@@ -6,39 +6,68 @@ use crate::{schema::Index, Result};
use super::plan::{
get_table_ref_bitmask_for_ast_expr, get_table_ref_bitmask_for_operator, BTreeTableReference,
Direction, IterationDirection, Plan, Search, SourceOperator,
DeletePlan, Direction, IterationDirection, Plan, Search, SelectPlan, SourceOperator,
};
pub fn optimize_plan(plan: Plan) -> Result<Plan> {
match plan {
Plan::Select(plan) => optimize_select_plan(plan).map(Plan::Select),
Plan::Delete(plan) => optimize_delete_plan(plan).map(Plan::Delete),
}
}
/**
* Make a few passes over the plan to optimize it.
* TODO: these could probably be done in less passes,
* but having them separate makes them easier to understand
*/
pub fn optimize_plan(mut select_plan: Plan) -> Result<Plan> {
eliminate_between(&mut select_plan.source, &mut select_plan.where_clause)?;
fn optimize_select_plan(mut plan: SelectPlan) -> Result<SelectPlan> {
eliminate_between(&mut plan.source, &mut plan.where_clause)?;
if let ConstantConditionEliminationResult::ImpossibleCondition =
eliminate_constants(&mut select_plan.source, &mut select_plan.where_clause)?
eliminate_constants(&mut plan.source, &mut plan.where_clause)?
{
select_plan.contains_constant_false_condition = true;
return Ok(select_plan);
plan.contains_constant_false_condition = true;
return Ok(plan);
}
push_predicates(
&mut select_plan.source,
&mut select_plan.where_clause,
&select_plan.referenced_tables,
&mut plan.source,
&mut plan.where_clause,
&plan.referenced_tables,
)?;
use_indexes(
&mut select_plan.source,
&select_plan.referenced_tables,
&select_plan.available_indexes,
&mut plan.source,
&plan.referenced_tables,
&plan.available_indexes,
)?;
eliminate_unnecessary_orderby(
&mut select_plan.source,
&mut select_plan.order_by,
&select_plan.referenced_tables,
&select_plan.available_indexes,
&mut plan.source,
&mut plan.order_by,
&plan.referenced_tables,
&plan.available_indexes,
)?;
Ok(select_plan)
Ok(plan)
}
fn optimize_delete_plan(mut plan: DeletePlan) -> Result<DeletePlan> {
eliminate_between(&mut plan.source, &mut plan.where_clause)?;
if let ConstantConditionEliminationResult::ImpossibleCondition =
eliminate_constants(&mut plan.source, &mut plan.where_clause)?
{
plan.contains_constant_false_condition = true;
return Ok(plan);
}
use_indexes(
&mut plan.source,
&plan.referenced_tables,
&plan.available_indexes,
)?;
Ok(plan)
}
fn _operator_is_already_ordered_by(
@@ -574,7 +603,7 @@ pub trait Optimizable {
impl Optimizable for ast::Expr {
fn is_rowid_alias_of(&self, table_index: usize) -> bool {
match self {
ast::Expr::Column {
Self::Column {
table,
is_rowid_alias,
..
@@ -589,7 +618,7 @@ impl Optimizable for ast::Expr {
available_indexes: &[Rc<Index>],
) -> Result<Option<usize>> {
match self {
ast::Expr::Column { table, column, .. } => {
Self::Column { table, column, .. } => {
if *table != table_index {
return Ok(None);
}
@@ -607,7 +636,7 @@ impl Optimizable for ast::Expr {
}
Ok(None)
}
ast::Expr::Binary(lhs, op, rhs) => {
Self::Binary(lhs, op, rhs) => {
let lhs_index =
lhs.check_index_scan(table_index, referenced_tables, available_indexes)?;
if lhs_index.is_some() {
@@ -619,7 +648,7 @@ impl Optimizable for ast::Expr {
// swap lhs and rhs
let lhs_new = rhs.take_ownership();
let rhs_new = lhs.take_ownership();
*self = ast::Expr::Binary(Box::new(lhs_new), *op, Box::new(rhs_new));
*self = Self::Binary(Box::new(lhs_new), *op, Box::new(rhs_new));
return Ok(rhs_index);
}
Ok(None)
@@ -629,7 +658,7 @@ impl Optimizable for ast::Expr {
}
fn check_constant(&self) -> Result<Option<ConstantPredicate>> {
match self {
ast::Expr::Id(id) => {
Self::Id(id) => {
// true and false are special constants that are effectively aliases for 1 and 0
if id.0.eq_ignore_ascii_case("true") {
return Ok(Some(ConstantPredicate::AlwaysTrue));
@@ -639,7 +668,7 @@ impl Optimizable for ast::Expr {
}
return Ok(None);
}
ast::Expr::Literal(lit) => match lit {
Self::Literal(lit) => match lit {
ast::Literal::Null => Ok(Some(ConstantPredicate::AlwaysFalse)),
ast::Literal::Numeric(b) => {
if let Ok(int_value) = b.parse::<i64>() {
@@ -681,7 +710,7 @@ impl Optimizable for ast::Expr {
}
_ => Ok(None),
},
ast::Expr::Unary(op, expr) => {
Self::Unary(op, expr) => {
if *op == ast::UnaryOperator::Not {
let trivial = expr.check_constant()?;
return Ok(trivial.map(|t| match t {
@@ -697,7 +726,7 @@ impl Optimizable for ast::Expr {
Ok(None)
}
ast::Expr::InList { lhs: _, not, rhs } => {
Self::InList { lhs: _, not, rhs } => {
if rhs.is_none() {
return Ok(Some(if *not {
ConstantPredicate::AlwaysTrue
@@ -716,7 +745,7 @@ impl Optimizable for ast::Expr {
Ok(None)
}
ast::Expr::Binary(lhs, op, rhs) => {
Self::Binary(lhs, op, rhs) => {
let lhs_trivial = lhs.check_constant()?;
let rhs_trivial = rhs.check_constant()?;
match op {
@@ -920,6 +949,6 @@ impl TakeOwnership for ast::Expr {
impl TakeOwnership for SourceOperator {
fn take_ownership(&mut self) -> Self {
std::mem::replace(self, SourceOperator::Nothing)
std::mem::replace(self, Self::Nothing)
}
}

46
core/translate/plan.rs
View File

@@ -1,11 +1,11 @@
use core::fmt;
use sqlite3_parser::ast;
use std::{
fmt::{Display, Formatter},
rc::Rc,
};
use sqlite3_parser::ast;
use crate::translate::plan::Plan::{Delete, Select};
use crate::{
function::AggFunc,
schema::{BTreeTable, Column, Index},
@@ -27,7 +27,13 @@ pub struct GroupBy {
}
#[derive(Debug)]
pub struct Plan {
pub enum Plan {
Select(SelectPlan),
Delete(DeletePlan),
}
#[derive(Debug)]
pub struct SelectPlan {
/// A tree of sources (tables).
pub source: SourceOperator,
/// the columns inside SELECT ... FROM
@@ -50,9 +56,33 @@ pub struct Plan {
pub contains_constant_false_condition: bool,
}
#[allow(dead_code)]
#[derive(Debug)]
pub struct DeletePlan {
/// A tree of sources (tables).
pub source: SourceOperator,
/// the columns inside SELECT ... FROM
pub result_columns: Vec<ResultSetColumn>,
/// where clause split into a vec at 'AND' boundaries.
pub where_clause: Option<Vec<ast::Expr>>,
/// order by clause
pub order_by: Option<Vec<(ast::Expr, Direction)>>,
/// limit clause
pub limit: Option<usize>,
/// all the tables referenced in the query
pub referenced_tables: Vec<BTreeTableReference>,
/// all the indexes available
pub available_indexes: Vec<Rc<Index>>,
/// query contains a constant condition that is always false
pub contains_constant_false_condition: bool,
}
impl Display for Plan {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.source)
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
Select(select_plan) => write!(f, "{}", select_plan.source),
Delete(delete_plan) => write!(f, "{}", delete_plan.source),
}
}
}
@@ -176,6 +206,7 @@ pub struct BTreeTableReference {
/// An enum that represents a search operation that can be used to search for a row in a table using an index
/// (i.e. a primary key or a secondary index)
#[allow(clippy::enum_variant_names)]
#[derive(Clone, Debug)]
pub enum Search {
/// A rowid equality point lookup. This is a special case that uses the SeekRowid bytecode instruction and does not loop.
@@ -366,7 +397,7 @@ pub fn get_table_ref_bitmask_for_operator<'a>(
table_refs_mask |= 1
<< tables
.iter()
.position(|t| &t.table_identifier == &table_reference.table_identifier)
.position(|t| t.table_identifier == table_reference.table_identifier)
.unwrap();
}
SourceOperator::Search {
@@ -375,7 +406,7 @@ pub fn get_table_ref_bitmask_for_operator<'a>(
table_refs_mask |= 1
<< tables
.iter()
.position(|t| &t.table_identifier == &table_reference.table_identifier)
.position(|t| t.table_identifier == table_reference.table_identifier)
.unwrap();
}
SourceOperator::Nothing => {}
@@ -391,6 +422,7 @@ pub fn get_table_ref_bitmask_for_operator<'a>(
and predicate = "t1.a = t2.b"
then the return value will be (in bits): 011
*/
#[allow(clippy::only_used_in_recursion)]
pub fn get_table_ref_bitmask_for_ast_expr<'a>(
tables: &'a Vec<BTreeTableReference>,
predicate: &'a ast::Expr,

114
core/translate/planner.rs
View File

@@ -1,11 +1,14 @@
use super::{
optimizer::Optimizable,
plan::{
Aggregate, BTreeTableReference, Direction, GroupBy, Plan, ResultSetColumn, SourceOperator,
},
use super::plan::{
Aggregate, BTreeTableReference, DeletePlan, Direction, GroupBy, Plan, ResultSetColumn,
SelectPlan, SourceOperator,
};
use crate::{function::Func, schema::Schema, util::normalize_ident, Result};
use sqlite3_parser::ast::{self, FromClause, JoinType, ResultColumn};
use crate::{
function::Func,
schema::Schema,
util::{exprs_are_equivalent, normalize_ident},
Result,
};
use sqlite3_parser::ast::{self, Expr, FromClause, JoinType, Limit, QualifiedName, ResultColumn};
pub struct OperatorIdCounter {
id: usize,
@@ -23,7 +26,10 @@ impl OperatorIdCounter {
}
fn resolve_aggregates(expr: &ast::Expr, aggs: &mut Vec<Aggregate>) -> bool {
if aggs.iter().any(|a| a.original_expr == *expr) {
if aggs
.iter()
.any(|a| exprs_are_equivalent(&a.original_expr, expr))
{
return true;
}
match expr {
@@ -259,7 +265,7 @@ pub fn prepare_select_plan<'a>(schema: &Schema, select: ast::Select) -> Result<P
columns,
from,
where_clause,
mut group_by,
group_by,
..
} => {
let col_count = columns.len();
@@ -272,7 +278,7 @@ pub fn prepare_select_plan<'a>(schema: &Schema, select: ast::Select) -> Result<P
// Parse the FROM clause
let (source, referenced_tables) = parse_from(schema, from, &mut operator_id_counter)?;
let mut plan = Plan {
let mut plan = SelectPlan {
source,
result_columns: vec![],
where_clause: None,
@@ -286,14 +292,7 @@ pub fn prepare_select_plan<'a>(schema: &Schema, select: ast::Select) -> Result<P
};
// Parse the WHERE clause
if let Some(w) = where_clause {
let mut predicates = vec![];
break_predicate_at_and_boundaries(w, &mut predicates);
for expr in predicates.iter_mut() {
bind_column_references(expr, &plan.referenced_tables)?;
}
plan.where_clause = Some(predicates);
}
plan.where_clause = parse_where(where_clause, &plan.referenced_tables)?;
let mut aggregate_expressions = Vec::new();
for column in columns.clone() {
@@ -477,23 +476,58 @@ pub fn prepare_select_plan<'a>(schema: &Schema, select: ast::Select) -> Result<P
}
// Parse the LIMIT clause
if let Some(limit) = &select.limit {
plan.limit = match &limit.expr {
ast::Expr::Literal(ast::Literal::Numeric(n)) => {
let l = n.parse()?;
Some(l)
}
_ => todo!(),
}
}
plan.limit = select.limit.and_then(|limit| parse_limit(limit));
// Return the unoptimized query plan
Ok(plan)
Ok(Plan::Select(plan))
}
_ => todo!(),
}
}
pub fn prepare_delete_plan(
schema: &Schema,
tbl_name: &QualifiedName,
where_clause: Option<Expr>,
limit: Option<Limit>,
) -> Result<Plan> {
let table = match schema.get_table(tbl_name.name.0.as_str()) {
Some(table) => table,
None => crate::bail_corrupt_error!("Parse error: no such table: {}", tbl_name),
};
let table_ref = BTreeTableReference {
table: table.clone(),
table_identifier: table.name.clone(),
table_index: 0,
};
let referenced_tables = vec![table_ref.clone()];
// Parse the WHERE clause
let resolved_where_clauses = parse_where(where_clause, &[table_ref.clone()])?;
// Parse the LIMIT clause
let resolved_limit = limit.and_then(|limit| parse_limit(limit));
let plan = DeletePlan {
source: SourceOperator::Scan {
id: 0,
table_reference: table_ref.clone(),
predicates: resolved_where_clauses.clone(),
iter_dir: None,
},
result_columns: vec![],
where_clause: resolved_where_clauses,
order_by: None,
limit: resolved_limit,
referenced_tables,
available_indexes: vec![],
contains_constant_false_condition: false,
};
Ok(Plan::Delete(plan))
}
#[allow(clippy::type_complexity)]
fn parse_from(
schema: &Schema,
@@ -555,6 +589,22 @@ fn parse_from(
Ok((operator, tables))
}
fn parse_where(
where_clause: Option<Expr>,
referenced_tables: &[BTreeTableReference],
) -> Result<Option<Vec<Expr>>> {
if let Some(where_expr) = where_clause {
let mut predicates = vec![];
break_predicate_at_and_boundaries(where_expr, &mut predicates);
for expr in predicates.iter_mut() {
bind_column_references(expr, referenced_tables)?;
}
Ok(Some(predicates))
} else {
Ok(None)
}
}
fn parse_join(
schema: &Schema,
join: ast::JoinedSelectTable,
@@ -738,6 +788,14 @@ fn parse_join(
))
}
fn parse_limit(limit: Limit) -> Option<usize> {
if let Expr::Literal(ast::Literal::Numeric(n)) = limit.expr {
n.parse().ok()
} else {
None
}
}
fn break_predicate_at_and_boundaries(predicate: ast::Expr, out_predicates: &mut Vec<ast::Expr>) {
match predicate {
ast::Expr::Binary(left, ast::Operator::And, right) => {

7
core/translate/select.rs
View File

@@ -1,15 +1,14 @@
use std::rc::Weak;
use std::{cell::RefCell, rc::Rc};
use super::emitter::emit_program;
use super::planner::prepare_select_plan;
use crate::storage::sqlite3_ondisk::DatabaseHeader;
use crate::translate::optimizer::optimize_plan;
use crate::Connection;
use crate::{schema::Schema, vdbe::Program, Result};
use sqlite3_parser::ast;
use super::emitter::emit_program;
use super::optimizer::optimize_plan;
use super::planner::prepare_select_plan;
pub fn translate_select(
schema: &Schema,
select: ast::Select,

488
core/types.rs
View File

@@ -18,11 +18,11 @@ pub enum Value<'a> {
impl<'a> Display for Value<'a> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Value::Null => write!(f, "NULL"),
Value::Integer(i) => write!(f, "{}", i),
Value::Float(fl) => write!(f, "{}", fl),
Value::Text(s) => write!(f, "{}", s),
Value::Blob(b) => write!(f, "{:?}", b),
Self::Null => write!(f, "NULL"),
Self::Integer(i) => write!(f, "{}", i),
Self::Float(fl) => write!(f, "{}", fl),
Self::Text(s) => write!(f, "{}", s),
Self::Blob(b) => write!(f, "{:?}", b),
}
}
}
@@ -69,27 +69,27 @@ pub enum OwnedValue {
impl OwnedValue {
// A helper function that makes building a text OwnedValue easier.
pub fn build_text(text: Rc<String>) -> Self {
OwnedValue::Text(LimboText::new(text))
Self::Text(LimboText::new(text))
}
}
impl Display for OwnedValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
OwnedValue::Null => write!(f, "NULL"),
OwnedValue::Integer(i) => write!(f, "{}", i),
OwnedValue::Float(fl) => write!(f, "{:?}", fl),
OwnedValue::Text(s) => write!(f, "{}", s.value),
OwnedValue::Blob(b) => write!(f, "{}", String::from_utf8_lossy(b)),
OwnedValue::Agg(a) => match a.as_ref() {
Self::Null => write!(f, "NULL"),
Self::Integer(i) => write!(f, "{}", i),
Self::Float(fl) => write!(f, "{:?}", fl),
Self::Text(s) => write!(f, "{}", s.value),
Self::Blob(b) => write!(f, "{}", String::from_utf8_lossy(b)),
Self::Agg(a) => match a.as_ref() {
AggContext::Avg(acc, _count) => write!(f, "{}", acc),
AggContext::Sum(acc) => write!(f, "{}", acc),
AggContext::Count(count) => write!(f, "{}", count),
AggContext::Max(max) => write!(f, "{}", max.as_ref().unwrap_or(&OwnedValue::Null)),
AggContext::Min(min) => write!(f, "{}", min.as_ref().unwrap_or(&OwnedValue::Null)),
AggContext::Max(max) => write!(f, "{}", max.as_ref().unwrap_or(&Self::Null)),
AggContext::Min(min) => write!(f, "{}", min.as_ref().unwrap_or(&Self::Null)),
AggContext::GroupConcat(s) => write!(f, "{}", s),
},
OwnedValue::Record(r) => write!(f, "{:?}", r),
Self::Record(r) => write!(f, "{:?}", r),
}
}
}
@@ -109,12 +109,12 @@ const NULL: OwnedValue = OwnedValue::Null;
impl AggContext {
pub fn final_value(&self) -> &OwnedValue {
match self {
AggContext::Avg(acc, _count) => acc,
AggContext::Sum(acc) => acc,
AggContext::Count(count) => count,
AggContext::Max(max) => max.as_ref().unwrap_or(&NULL),
AggContext::Min(min) => min.as_ref().unwrap_or(&NULL),
AggContext::GroupConcat(s) => s,
Self::Avg(acc, _count) => acc,
Self::Sum(acc) => acc,
Self::Count(count) => count,
Self::Max(max) => max.as_ref().unwrap_or(&NULL),
Self::Min(min) => min.as_ref().unwrap_or(&NULL),
Self::GroupConcat(s) => s,
}
}
}
@@ -123,44 +123,38 @@ impl AggContext {
impl PartialOrd<OwnedValue> for OwnedValue {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
match (self, other) {
(OwnedValue::Integer(int_left), OwnedValue::Integer(int_right)) => {
int_left.partial_cmp(int_right)
}
(OwnedValue::Integer(int_left), OwnedValue::Float(float_right)) => {
(Self::Integer(int_left), Self::Integer(int_right)) => int_left.partial_cmp(int_right),
(Self::Integer(int_left), Self::Float(float_right)) => {
(*int_left as f64).partial_cmp(float_right)
}
(OwnedValue::Float(float_left), OwnedValue::Integer(int_right)) => {
(Self::Float(float_left), Self::Integer(int_right)) => {
float_left.partial_cmp(&(*int_right as f64))
}
(OwnedValue::Float(float_left), OwnedValue::Float(float_right)) => {
(Self::Float(float_left), Self::Float(float_right)) => {
float_left.partial_cmp(float_right)
}
// Numeric vs Text/Blob
(
OwnedValue::Integer(_) | OwnedValue::Float(_),
OwnedValue::Text(_) | OwnedValue::Blob(_),
) => Some(std::cmp::Ordering::Less),
(
OwnedValue::Text(_) | OwnedValue::Blob(_),
OwnedValue::Integer(_) | OwnedValue::Float(_),
) => Some(std::cmp::Ordering::Greater),
(Self::Integer(_) | Self::Float(_), Self::Text(_) | Self::Blob(_)) => {
Some(std::cmp::Ordering::Less)
}
(Self::Text(_) | Self::Blob(_), Self::Integer(_) | Self::Float(_)) => {
Some(std::cmp::Ordering::Greater)
}
(OwnedValue::Text(text_left), OwnedValue::Text(text_right)) => {
(Self::Text(text_left), Self::Text(text_right)) => {
text_left.value.partial_cmp(&text_right.value)
}
// Text vs Blob
(OwnedValue::Text(_), OwnedValue::Blob(_)) => Some(std::cmp::Ordering::Less),
(OwnedValue::Blob(_), OwnedValue::Text(_)) => Some(std::cmp::Ordering::Greater),
(Self::Text(_), Self::Blob(_)) => Some(std::cmp::Ordering::Less),
(Self::Blob(_), Self::Text(_)) => Some(std::cmp::Ordering::Greater),
(OwnedValue::Blob(blob_left), OwnedValue::Blob(blob_right)) => {
blob_left.partial_cmp(blob_right)
}
(OwnedValue::Null, OwnedValue::Null) => Some(std::cmp::Ordering::Equal),
(OwnedValue::Null, _) => Some(std::cmp::Ordering::Less),
(_, OwnedValue::Null) => Some(std::cmp::Ordering::Greater),
(OwnedValue::Agg(a), OwnedValue::Agg(b)) => a.partial_cmp(b),
(OwnedValue::Agg(a), other) => a.final_value().partial_cmp(other),
(other, OwnedValue::Agg(b)) => other.partial_cmp(b.final_value()),
(Self::Blob(blob_left), Self::Blob(blob_right)) => blob_left.partial_cmp(blob_right),
(Self::Null, Self::Null) => Some(std::cmp::Ordering::Equal),
(Self::Null, _) => Some(std::cmp::Ordering::Less),
(_, Self::Null) => Some(std::cmp::Ordering::Greater),
(Self::Agg(a), Self::Agg(b)) => a.partial_cmp(b),
(Self::Agg(a), other) => a.final_value().partial_cmp(other),
(other, Self::Agg(b)) => other.partial_cmp(b.final_value()),
other => todo!("{:?}", other),
}
}
@@ -169,12 +163,12 @@ impl PartialOrd<OwnedValue> for OwnedValue {
impl std::cmp::PartialOrd<AggContext> for AggContext {
fn partial_cmp(&self, other: &AggContext) -> Option<std::cmp::Ordering> {
match (self, other) {
(AggContext::Avg(a, _), AggContext::Avg(b, _)) => a.partial_cmp(b),
(AggContext::Sum(a), AggContext::Sum(b)) => a.partial_cmp(b),
(AggContext::Count(a), AggContext::Count(b)) => a.partial_cmp(b),
(AggContext::Max(a), AggContext::Max(b)) => a.partial_cmp(b),
(AggContext::Min(a), AggContext::Min(b)) => a.partial_cmp(b),
(AggContext::GroupConcat(a), AggContext::GroupConcat(b)) => a.partial_cmp(b),
(Self::Avg(a, _), Self::Avg(b, _)) => a.partial_cmp(b),
(Self::Sum(a), Self::Sum(b)) => a.partial_cmp(b),
(Self::Count(a), Self::Count(b)) => a.partial_cmp(b),
(Self::Max(a), Self::Max(b)) => a.partial_cmp(b),
(Self::Min(a), Self::Min(b)) => a.partial_cmp(b),
(Self::GroupConcat(a), Self::GroupConcat(b)) => a.partial_cmp(b),
_ => None,
}
}
@@ -193,44 +187,38 @@ impl std::ops::Add<OwnedValue> for OwnedValue {
fn add(self, rhs: Self) -> Self::Output {
match (self, rhs) {
(OwnedValue::Integer(int_left), OwnedValue::Integer(int_right)) => {
OwnedValue::Integer(int_left + int_right)
(Self::Integer(int_left), Self::Integer(int_right)) => {
Self::Integer(int_left + int_right)
}
(OwnedValue::Integer(int_left), OwnedValue::Float(float_right)) => {
OwnedValue::Float(int_left as f64 + float_right)
(Self::Integer(int_left), Self::Float(float_right)) => {
Self::Float(int_left as f64 + float_right)
}
(OwnedValue::Float(float_left), OwnedValue::Integer(int_right)) => {
OwnedValue::Float(float_left + int_right as f64)
(Self::Float(float_left), Self::Integer(int_right)) => {
Self::Float(float_left + int_right as f64)
}
(OwnedValue::Float(float_left), OwnedValue::Float(float_right)) => {
OwnedValue::Float(float_left + float_right)
(Self::Float(float_left), Self::Float(float_right)) => {
Self::Float(float_left + float_right)
}
(OwnedValue::Text(string_left), OwnedValue::Text(string_right)) => {
OwnedValue::build_text(Rc::new(
string_left.value.to_string() + &string_right.value.to_string(),
))
(Self::Text(string_left), Self::Text(string_right)) => Self::build_text(Rc::new(
string_left.value.to_string() + &string_right.value.to_string(),
)),
(Self::Text(string_left), Self::Integer(int_right)) => Self::build_text(Rc::new(
string_left.value.to_string() + &int_right.to_string(),
)),
(Self::Integer(int_left), Self::Text(string_right)) => Self::build_text(Rc::new(
int_left.to_string() + &string_right.value.to_string(),
)),
(Self::Text(string_left), Self::Float(float_right)) => {
let string_right = Self::Float(float_right).to_string();
Self::build_text(Rc::new(string_left.value.to_string() + &string_right))
}
(OwnedValue::Text(string_left), OwnedValue::Integer(int_right)) => {
OwnedValue::build_text(Rc::new(
string_left.value.to_string() + &int_right.to_string(),
))
(Self::Float(float_left), Self::Text(string_right)) => {
let string_left = Self::Float(float_left).to_string();
Self::build_text(Rc::new(string_left + &string_right.value.to_string()))
}
(OwnedValue::Integer(int_left), OwnedValue::Text(string_right)) => {
OwnedValue::build_text(Rc::new(
int_left.to_string() + &string_right.value.to_string(),
))
}
(OwnedValue::Text(string_left), OwnedValue::Float(float_right)) => {
let string_right = OwnedValue::Float(float_right).to_string();
OwnedValue::build_text(Rc::new(string_left.value.to_string() + &string_right))
}
(OwnedValue::Float(float_left), OwnedValue::Text(string_right)) => {
let string_left = OwnedValue::Float(float_left).to_string();
OwnedValue::build_text(Rc::new(string_left + &string_right.value.to_string()))
}
(lhs, OwnedValue::Null) => lhs,
(OwnedValue::Null, rhs) => rhs,
_ => OwnedValue::Float(0.0),
(lhs, Self::Null) => lhs,
(Self::Null, rhs) => rhs,
_ => Self::Float(0.0),
}
}
}
@@ -240,8 +228,8 @@ impl std::ops::Add<f64> for OwnedValue {
fn add(self, rhs: f64) -> Self::Output {
match self {
OwnedValue::Integer(int_left) => OwnedValue::Float(int_left as f64 + rhs),
OwnedValue::Float(float_left) => OwnedValue::Float(float_left + rhs),
Self::Integer(int_left) => Self::Float(int_left as f64 + rhs),
Self::Float(float_left) => Self::Float(float_left + rhs),
_ => unreachable!(),
}
}
@@ -252,8 +240,8 @@ impl std::ops::Add<i64> for OwnedValue {
fn add(self, rhs: i64) -> Self::Output {
match self {
OwnedValue::Integer(int_left) => OwnedValue::Integer(int_left + rhs),
OwnedValue::Float(float_left) => OwnedValue::Float(float_left + rhs as f64),
Self::Integer(int_left) => Self::Integer(int_left + rhs),
Self::Float(float_left) => Self::Float(float_left + rhs as f64),
_ => unreachable!(),
}
}
@@ -282,19 +270,19 @@ impl std::ops::Div<OwnedValue> for OwnedValue {
fn div(self, rhs: OwnedValue) -> Self::Output {
match (self, rhs) {
(OwnedValue::Integer(int_left), OwnedValue::Integer(int_right)) => {
OwnedValue::Integer(int_left / int_right)
(Self::Integer(int_left), Self::Integer(int_right)) => {
Self::Integer(int_left / int_right)
}
(OwnedValue::Integer(int_left), OwnedValue::Float(float_right)) => {
OwnedValue::Float(int_left as f64 / float_right)
(Self::Integer(int_left), Self::Float(float_right)) => {
Self::Float(int_left as f64 / float_right)
}
(OwnedValue::Float(float_left), OwnedValue::Integer(int_right)) => {
OwnedValue::Float(float_left / int_right as f64)
(Self::Float(float_left), Self::Integer(int_right)) => {
Self::Float(float_left / int_right as f64)
}
(OwnedValue::Float(float_left), OwnedValue::Float(float_right)) => {
OwnedValue::Float(float_left / float_right)
(Self::Float(float_left), Self::Float(float_right)) => {
Self::Float(float_left / float_right)
}
_ => OwnedValue::Float(0.0),
_ => Self::Float(0.0),
}
}
}
@@ -387,6 +375,75 @@ pub struct OwnedRecord {
pub values: Vec<OwnedValue>,
}
const I8_LOW: i64 = -128;
const I8_HIGH: i64 = 127;
const I16_LOW: i64 = -32768;
const I16_HIGH: i64 = 32767;
const I24_LOW: i64 = -8388608;
const I24_HIGH: i64 = 8388607;
const I32_LOW: i64 = -2147483648;
const I32_HIGH: i64 = 2147483647;
const I48_LOW: i64 = -140737488355328;
const I48_HIGH: i64 = 140737488355327;
/// Sqlite Serial Types
/// https://www.sqlite.org/fileformat.html#record_format
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
enum SerialType {
Null,
I8,
I16,
I24,
I32,
I48,
I64,
F64,
Text { content_size: usize },
Blob { content_size: usize },
}
impl From<&OwnedValue> for SerialType {
fn from(value: &OwnedValue) -> Self {
match value {
OwnedValue::Null => SerialType::Null,
OwnedValue::Integer(i) => match i {
i if *i >= I8_LOW && *i <= I8_HIGH => SerialType::I8,
i if *i >= I16_LOW && *i <= I16_HIGH => SerialType::I16,
i if *i >= I24_LOW && *i <= I24_HIGH => SerialType::I24,
i if *i >= I32_LOW && *i <= I32_HIGH => SerialType::I32,
i if *i >= I48_LOW && *i <= I48_HIGH => SerialType::I48,
_ => SerialType::I64,
},
OwnedValue::Float(_) => SerialType::F64,
OwnedValue::Text(t) => SerialType::Text {
content_size: t.value.len(),
},
OwnedValue::Blob(b) => SerialType::Blob {
content_size: b.len(),
},
OwnedValue::Agg(_) => unreachable!(),
OwnedValue::Record(_) => unreachable!(),
}
}
}
impl From<SerialType> for u64 {
fn from(serial_type: SerialType) -> Self {
match serial_type {
SerialType::Null => 0,
SerialType::I8 => 1,
SerialType::I16 => 2,
SerialType::I24 => 3,
SerialType::I32 => 4,
SerialType::I48 => 5,
SerialType::I64 => 6,
SerialType::F64 => 7,
SerialType::Text { content_size } => (content_size * 2 + 13) as u64,
SerialType::Blob { content_size } => (content_size * 2 + 12) as u64,
}
}
}
impl OwnedRecord {
pub fn new(values: Vec<OwnedValue>) -> Self {
Self { values }
@@ -395,31 +452,32 @@ impl OwnedRecord {
pub fn serialize(&self, buf: &mut Vec<u8>) {
let initial_i = buf.len();
// write serial types
for value in &self.values {
let serial_type = match value {
OwnedValue::Null => 0,
OwnedValue::Integer(_) => 6, // for now let's only do i64
OwnedValue::Float(_) => 7,
OwnedValue::Text(t) => (t.value.len() * 2 + 13) as u64,
OwnedValue::Blob(b) => (b.len() * 2 + 12) as u64,
// not serializable values
OwnedValue::Agg(_) => unreachable!(),
OwnedValue::Record(_) => unreachable!(),
};
buf.resize(buf.len() + 9, 0); // Ensure space for varint
let serial_type = SerialType::from(value);
buf.resize(buf.len() + 9, 0); // Ensure space for varint (1-9 bytes in length)
let len = buf.len();
let n = write_varint(&mut buf[len - 9..], serial_type);
let n = write_varint(&mut buf[len - 9..], serial_type.into());
buf.truncate(buf.len() - 9 + n); // Remove unused bytes
}
let mut header_size = buf.len() - initial_i;
// write content
for value in &self.values {
// TODO: make integers and floats with smaller serial types
match value {
OwnedValue::Null => {}
OwnedValue::Integer(i) => buf.extend_from_slice(&i.to_be_bytes()),
OwnedValue::Integer(i) => {
let serial_type = SerialType::from(value);
match serial_type {
SerialType::I8 => buf.extend_from_slice(&(*i as i8).to_be_bytes()),
SerialType::I16 => buf.extend_from_slice(&(*i as i16).to_be_bytes()),
SerialType::I24 => buf.extend_from_slice(&(*i as i32).to_be_bytes()[1..]), // remove most significant byte
SerialType::I32 => buf.extend_from_slice(&(*i as i32).to_be_bytes()),
SerialType::I48 => buf.extend_from_slice(&i.to_be_bytes()[2..]), // remove 2 most significant bytes
SerialType::I64 => buf.extend_from_slice(&i.to_be_bytes()),
_ => unreachable!(),
}
}
OwnedValue::Float(f) => buf.extend_from_slice(&f.to_be_bytes()),
OwnedValue::Text(t) => buf.extend_from_slice(t.value.as_bytes()),
OwnedValue::Blob(b) => buf.extend_from_slice(b),
@@ -484,8 +542,212 @@ pub trait Cursor {
record: &OwnedRecord,
moved_before: bool, /* Tells inserter that it doesn't need to traverse in order to find leaf page */
) -> Result<CursorResult<()>>; //
fn delete(&mut self) -> Result<CursorResult<()>>;
fn exists(&mut self, key: &OwnedValue) -> Result<CursorResult<bool>>;
fn set_null_flag(&mut self, flag: bool);
fn get_null_flag(&self) -> bool;
fn btree_create(&mut self, flags: usize) -> u32;
}
#[cfg(test)]
mod tests {
use super::*;
use std::rc::Rc;
#[test]
fn test_serialize_null() {
let record = OwnedRecord::new(vec![OwnedValue::Null]);
let mut buf = Vec::new();
record.serialize(&mut buf);
let header_length = record.values.len() + 1;
let header = &buf[0..header_length];
// First byte should be header size
assert_eq!(header[0], header_length as u8);
// Second byte should be serial type for NULL
assert_eq!(header[1] as u64, u64::from(SerialType::Null));
// Check that the buffer is empty after the header
assert_eq!(buf.len(), header_length);
}
#[test]
fn test_serialize_integers() {
let record = OwnedRecord::new(vec![
OwnedValue::Integer(42), // Should use SERIAL_TYPE_I8
OwnedValue::Integer(1000), // Should use SERIAL_TYPE_I16
OwnedValue::Integer(1_000_000), // Should use SERIAL_TYPE_I24
OwnedValue::Integer(1_000_000_000), // Should use SERIAL_TYPE_I32
OwnedValue::Integer(1_000_000_000_000), // Should use SERIAL_TYPE_I48
OwnedValue::Integer(i64::MAX), // Should use SERIAL_TYPE_I64
]);
let mut buf = Vec::new();
record.serialize(&mut buf);
let header_length = record.values.len() + 1;
let header = &buf[0..header_length];
// First byte should be header size
assert!(header[0] == header_length as u8); // Header should be larger than number of values
// Check that correct serial types were chosen
assert_eq!(header[1] as u64, u64::from(SerialType::I8));
assert_eq!(header[2] as u64, u64::from(SerialType::I16));
assert_eq!(header[3] as u64, u64::from(SerialType::I24));
assert_eq!(header[4] as u64, u64::from(SerialType::I32));
assert_eq!(header[5] as u64, u64::from(SerialType::I48));
assert_eq!(header[6] as u64, u64::from(SerialType::I64));
// test that the bytes after the header can be interpreted as the correct values
let mut cur_offset = header_length;
let i8_bytes = &buf[cur_offset..cur_offset + size_of::<i8>()];
cur_offset += size_of::<i8>();
let i16_bytes = &buf[cur_offset..cur_offset + size_of::<i16>()];
cur_offset += size_of::<i16>();
let i24_bytes = &buf[cur_offset..cur_offset + size_of::<i32>() - 1];
cur_offset += size_of::<i32>() - 1; // i24
let i32_bytes = &buf[cur_offset..cur_offset + size_of::<i32>()];
cur_offset += size_of::<i32>();
let i48_bytes = &buf[cur_offset..cur_offset + size_of::<i64>() - 2];
cur_offset += size_of::<i64>() - 2; // i48
let i64_bytes = &buf[cur_offset..cur_offset + size_of::<i64>()];
let val_int8 = i8::from_be_bytes(i8_bytes.try_into().unwrap());
let val_int16 = i16::from_be_bytes(i16_bytes.try_into().unwrap());
let mut leading_0 = vec![0];
leading_0.extend(i24_bytes);
let val_int24 = i32::from_be_bytes(leading_0.try_into().unwrap());
let val_int32 = i32::from_be_bytes(i32_bytes.try_into().unwrap());
let mut leading_00 = vec![0, 0];
leading_00.extend(i48_bytes);
let val_int48 = i64::from_be_bytes(leading_00.try_into().unwrap());
let val_int64 = i64::from_be_bytes(i64_bytes.try_into().unwrap());
assert_eq!(val_int8, 42);
assert_eq!(val_int16, 1000);
assert_eq!(val_int24, 1_000_000);
assert_eq!(val_int32, 1_000_000_000);
assert_eq!(val_int48, 1_000_000_000_000);
assert_eq!(val_int64, i64::MAX);
// assert correct size of buffer: header + values (bytes per value depends on serial type)
assert_eq!(
buf.len(),
header_length
+ size_of::<i8>()
+ size_of::<i16>()
+ (size_of::<i32>() - 1) // i24
+ size_of::<i32>()
+ (size_of::<i64>() - 2) // i48
+ size_of::<f64>()
);
}
#[test]
fn test_serialize_float() {
let record = OwnedRecord::new(vec![OwnedValue::Float(3.14159)]);
let mut buf = Vec::new();
record.serialize(&mut buf);
let header_length = record.values.len() + 1;
let header = &buf[0..header_length];
// First byte should be header size
assert_eq!(header[0], header_length as u8);
// Second byte should be serial type for FLOAT
assert_eq!(header[1] as u64, u64::from(SerialType::F64));
// Check that the bytes after the header can be interpreted as the float
let float_bytes = &buf[header_length..header_length + size_of::<f64>()];
let float = f64::from_be_bytes(float_bytes.try_into().unwrap());
assert_eq!(float, 3.14159);
// Check that buffer length is correct
assert_eq!(buf.len(), header_length + size_of::<f64>());
}
#[test]
fn test_serialize_text() {
let text = Rc::new("hello".to_string());
let record = OwnedRecord::new(vec![OwnedValue::Text(LimboText::new(text.clone()))]);
let mut buf = Vec::new();
record.serialize(&mut buf);
let header_length = record.values.len() + 1;
let header = &buf[0..header_length];
// First byte should be header size
assert_eq!(header[0], header_length as u8);
// Second byte should be serial type for TEXT, which is (len * 2 + 13)
assert_eq!(header[1], (5 * 2 + 13) as u8);
// Check the actual text bytes
assert_eq!(&buf[2..7], b"hello");
// Check that buffer length is correct
assert_eq!(buf.len(), header_length + text.len());
}
#[test]
fn test_serialize_blob() {
let blob = Rc::new(vec![1, 2, 3, 4, 5]);
let record = OwnedRecord::new(vec![OwnedValue::Blob(blob.clone())]);
let mut buf = Vec::new();
record.serialize(&mut buf);
let header_length = record.values.len() + 1;
let header = &buf[0..header_length];
// First byte should be header size
assert_eq!(header[0], header_length as u8);
// Second byte should be serial type for BLOB, which is (len * 2 + 12)
assert_eq!(header[1], (5 * 2 + 12) as u8);
// Check the actual blob bytes
assert_eq!(&buf[2..7], &[1, 2, 3, 4, 5]);
// Check that buffer length is correct
assert_eq!(buf.len(), header_length + blob.len());
}
#[test]
fn test_serialize_mixed_types() {
let text = Rc::new("test".to_string());
let record = OwnedRecord::new(vec![
OwnedValue::Null,
OwnedValue::Integer(42),
OwnedValue::Float(3.14),
OwnedValue::Text(LimboText::new(text.clone())),
]);
let mut buf = Vec::new();
record.serialize(&mut buf);
let header_length = record.values.len() + 1;
let header = &buf[0..header_length];
// First byte should be header size
assert_eq!(header[0], header_length as u8);
// Second byte should be serial type for NULL
assert_eq!(header[1] as u64, u64::from(SerialType::Null));
// Third byte should be serial type for I8
assert_eq!(header[2] as u64, u64::from(SerialType::I8));
// Fourth byte should be serial type for F64
assert_eq!(header[3] as u64, u64::from(SerialType::F64));
// Fifth byte should be serial type for TEXT, which is (len * 2 + 13)
assert_eq!(header[4] as u64, (4 * 2 + 13) as u64);
// Check that the bytes after the header can be interpreted as the correct values
let mut cur_offset = header_length;
let i8_bytes = &buf[cur_offset..cur_offset + size_of::<i8>()];
cur_offset += size_of::<i8>();
let f64_bytes = &buf[cur_offset..cur_offset + size_of::<f64>()];
cur_offset += size_of::<f64>();
let text_bytes = &buf[cur_offset..cur_offset + text.len()];
let val_int8 = i8::from_be_bytes(i8_bytes.try_into().unwrap());
let val_float = f64::from_be_bytes(f64_bytes.try_into().unwrap());
let val_text = String::from_utf8(text_bytes.to_vec()).unwrap();
assert_eq!(val_int8, 42);
assert_eq!(val_float, 3.14);
assert_eq!(val_text, "test");
// Check that buffer length is correct
assert_eq!(
buf.len(),
header_length + size_of::<i8>() + size_of::<f64>() + text.len()
);
}
}

13
core/util.rs
View File

@@ -4,7 +4,7 @@ use sqlite3_parser::ast::{Expr, FunctionTail, Literal};
use crate::{
schema::{self, Schema},
Result, RowResult, Rows, IO,
Result, Rows, StepResult, IO,
};
// https://sqlite.org/lang_keywords.html
@@ -15,7 +15,7 @@ pub fn normalize_ident(identifier: &str) -> String {
.iter()
.find(|&(start, end)| identifier.starts_with(*start) && identifier.ends_with(*end));
if let Some(&(start, end)) = quote_pair {
if let Some(&(_, _)) = quote_pair {
&identifier[1..identifier.len() - 1]
} else {
identifier
@@ -27,7 +27,7 @@ pub fn parse_schema_rows(rows: Option<Rows>, schema: &mut Schema, io: Arc<dyn IO
if let Some(mut rows) = rows {
loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
let ty = row.get::<&str>(0)?;
if ty != "table" && ty != "index" {
continue;
@@ -53,13 +53,14 @@ pub fn parse_schema_rows(rows: Option<Rows>, schema: &mut Schema, io: Arc<dyn IO
_ => continue,
}
}
RowResult::IO => {
StepResult::IO => {
// TODO: How do we ensure that the I/O we submitted to
// read the schema is actually complete?
io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => break,
}
}
}

15
core/vdbe/builder.rs
View File

@@ -59,10 +59,6 @@ impl ProgramBuilder {
reg
}
pub fn next_free_register(&self) -> usize {
self.next_free_register
}
pub fn alloc_cursor_id(
&mut self,
table_identifier: Option<String>,
@@ -144,6 +140,17 @@ impl ProgramBuilder {
.push((label, insn_reference));
}
/// Resolve unresolved labels to a specific offset in the instruction list.
///
/// This function updates all instructions that reference the given label
/// to point to the specified offset. It ensures that the label and offset
/// are valid and updates the target program counter (PC) of each instruction
/// that references the label.
///
/// # Arguments
///
/// * `label` - The label to resolve.
/// * `to_offset` - The offset to which the labeled instructions should be resolved to.
pub fn resolve_label(&mut self, label: BranchOffset, to_offset: BranchOffset) {
assert!(label < 0);
assert!(to_offset >= 0);

28
core/vdbe/explain.rs
View File

@@ -1,5 +1,4 @@
use super::{Insn, InsnReference, OwnedValue, Program};
use crate::types::LimboText;
use std::rc::Rc;
pub fn insn_to_str(
@@ -83,6 +82,15 @@ pub fn insn_to_str(
0,
format!("r[{}]=~r[{}]", dest, reg),
),
Insn::Remainder { lhs, rhs, dest } => (
"Remainder",
*lhs as i32,
*rhs as i32,
*dest as i32,
OwnedValue::build_text(Rc::new("".to_string())),
0,
format!("r[{}]=r[{}]%r[{}]", dest, lhs, rhs),
),
Insn::Null { dest, dest_end } => (
"Null",
0,
@@ -834,6 +842,24 @@ pub fn insn_to_str(
0,
"".to_string(),
),
Insn::DeleteAsync { cursor_id } => (
"DeleteAsync",
*cursor_id as i32,
0,
0,
OwnedValue::build_text(Rc::new("".to_string())),
0,
"".to_string(),
),
Insn::DeleteAwait { cursor_id } => (
"DeleteAwait",
*cursor_id as i32,
0,
0,
OwnedValue::build_text(Rc::new("".to_string())),
0,
"".to_string(),
),
Insn::NewRowid {
cursor,
rowid_reg,

490
core/vdbe/insn.rs Normal file
View File

@@ -0,0 +1,490 @@
use super::{AggFunc, BranchOffset, CursorID, FuncCtx, PageIdx};
use crate::types::OwnedRecord;
use limbo_macros::Description;
#[derive(Description, Debug)]
pub enum Insn {
// Initialize the program state and jump to the given PC.
Init {
target_pc: BranchOffset,
},
// Write a NULL into register dest. If dest_end is Some, then also write NULL into register dest_end and every register in between dest and dest_end. If dest_end is not set, then only register dest is set to NULL.
Null {
dest: usize,
dest_end: Option<usize>,
},
// Move the cursor P1 to a null row. Any Column operations that occur while the cursor is on the null row will always write a NULL.
NullRow {
cursor_id: CursorID,
},
// Add two registers and store the result in a third register.
Add {
lhs: usize,
rhs: usize,
dest: usize,
},
// Subtract rhs from lhs and store in dest
Subtract {
lhs: usize,
rhs: usize,
dest: usize,
},
// Multiply two registers and store the result in a third register.
Multiply {
lhs: usize,
rhs: usize,
dest: usize,
},
// Divide lhs by rhs and store the result in a third register.
Divide {
lhs: usize,
rhs: usize,
dest: usize,
},
// Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of the comparison for use by the next Jump instruct.
Compare {
start_reg_a: usize,
start_reg_b: usize,
count: usize,
},
// Place the result of rhs bitwise AND lhs in third register.
BitAnd {
lhs: usize,
rhs: usize,
dest: usize,
},
// Place the result of rhs bitwise OR lhs in third register.
BitOr {
lhs: usize,
rhs: usize,
dest: usize,
},
// Place the result of bitwise NOT register P1 in dest register.
BitNot {
reg: usize,
dest: usize,
},
// Divide lhs by rhs and place the remainder in dest register.
Remainder {
lhs: usize,
rhs: usize,
dest: usize,
},
// Jump to the instruction at address P1, P2, or P3 depending on whether in the most recent Compare instruction the P1 vector was less than, equal to, or greater than the P2 vector, respectively.
Jump {
target_pc_lt: BranchOffset,
target_pc_eq: BranchOffset,
target_pc_gt: BranchOffset,
},
// Move the P3 values in register P1..P1+P3-1 over into registers P2..P2+P3-1. Registers P1..P1+P3-1 are left holding a NULL. It is an error for register ranges P1..P1+P3-1 and P2..P2+P3-1 to overlap. It is an error for P3 to be less than 1.
Move {
source_reg: usize,
dest_reg: usize,
count: usize,
},
// If the given register is a positive integer, decrement it by decrement_by and jump to the given PC.
IfPos {
reg: usize,
target_pc: BranchOffset,
decrement_by: usize,
},
// If the given register is not NULL, jump to the given PC.
NotNull {
reg: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if they are equal.
Eq {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if they are not equal.
Ne {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if the left-hand side is less than the right-hand side.
Lt {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if the left-hand side is less than or equal to the right-hand side.
Le {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if the left-hand side is greater than the right-hand side.
Gt {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if the left-hand side is greater than or equal to the right-hand side.
Ge {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
/// Jump to target_pc if r\[reg\] != 0 or (r\[reg\] == NULL && r\[null_reg\] != 0)
If {
reg: usize, // P1
target_pc: BranchOffset, // P2
/// P3. If r\[reg\] is null, jump iff r\[null_reg\] != 0
null_reg: usize,
},
/// Jump to target_pc if r\[reg\] != 0 or (r\[reg\] == NULL && r\[null_reg\] != 0)
IfNot {
reg: usize, // P1
target_pc: BranchOffset, // P2
/// P3. If r\[reg\] is null, jump iff r\[null_reg\] != 0
null_reg: usize,
},
// Open a cursor for reading.
OpenReadAsync {
cursor_id: CursorID,
root_page: PageIdx,
},
// Await for the completion of open cursor.
OpenReadAwait,
// Open a cursor for a pseudo-table that contains a single row.
OpenPseudo {
cursor_id: CursorID,
content_reg: usize,
num_fields: usize,
},
// Rewind the cursor to the beginning of the B-Tree.
RewindAsync {
cursor_id: CursorID,
},
// Await for the completion of cursor rewind.
RewindAwait {
cursor_id: CursorID,
pc_if_empty: BranchOffset,
},
LastAsync {
cursor_id: CursorID,
},
LastAwait {
cursor_id: CursorID,
pc_if_empty: BranchOffset,
},
// Read a column from the current row of the cursor.
Column {
cursor_id: CursorID,
column: usize,
dest: usize,
},
// Make a record and write it to destination register.
MakeRecord {
start_reg: usize, // P1
count: usize, // P2
dest_reg: usize, // P3
},
// Emit a row of results.
ResultRow {
start_reg: usize, // P1
count: usize, // P2
},
// Advance the cursor to the next row.
NextAsync {
cursor_id: CursorID,
},
// Await for the completion of cursor advance.
NextAwait {
cursor_id: CursorID,
pc_if_next: BranchOffset,
},
PrevAsync {
cursor_id: CursorID,
},
PrevAwait {
cursor_id: CursorID,
pc_if_next: BranchOffset,
},
// Halt the program.
Halt {
err_code: usize,
description: String,
},
// Start a transaction.
Transaction {
write: bool,
},
// Branch to the given PC.
Goto {
target_pc: BranchOffset,
},
// Stores the current program counter into register 'return_reg' then jumps to address target_pc.
Gosub {
target_pc: BranchOffset,
return_reg: usize,
},
// Returns to the program counter stored in register 'return_reg'.
Return {
return_reg: usize,
},
// Write an integer value into a register.
Integer {
value: i64,
dest: usize,
},
// Write a float value into a register
Real {
value: f64,
dest: usize,
},
// If register holds an integer, transform it to a float
RealAffinity {
register: usize,
},
// Write a string value into a register.
String8 {
value: String,
dest: usize,
},
// Write a blob value into a register.
Blob {
value: Vec<u8>,
dest: usize,
},
// Read the rowid of the current row.
RowId {
cursor_id: CursorID,
dest: usize,
},
// Seek to a rowid in the cursor. If not found, jump to the given PC. Otherwise, continue to the next instruction.
SeekRowid {
cursor_id: CursorID,
src_reg: usize,
target_pc: BranchOffset,
},
// P1 is an open index cursor and P3 is a cursor on the corresponding table. This opcode does a deferred seek of the P3 table cursor to the row that corresponds to the current row of P1.
// This is a deferred seek. Nothing actually happens until the cursor is used to read a record. That way, if no reads occur, no unnecessary I/O happens.
DeferredSeek {
index_cursor_id: CursorID,
table_cursor_id: CursorID,
},
// If cursor_id refers to an SQL table (B-Tree that uses integer keys), use the value in start_reg as the key.
// If cursor_id refers to an SQL index, then start_reg is the first in an array of num_regs registers that are used as an unpacked index key.
// Seek to the first index entry that is greater than or equal to the given key. If not found, jump to the given PC. Otherwise, continue to the next instruction.
SeekGE {
is_index: bool,
cursor_id: CursorID,
start_reg: usize,
num_regs: usize,
target_pc: BranchOffset,
},
// If cursor_id refers to an SQL table (B-Tree that uses integer keys), use the value in start_reg as the key.
// If cursor_id refers to an SQL index, then start_reg is the first in an array of num_regs registers that are used as an unpacked index key.
// Seek to the first index entry that is greater than the given key. If not found, jump to the given PC. Otherwise, continue to the next instruction.
SeekGT {
is_index: bool,
cursor_id: CursorID,
start_reg: usize,
num_regs: usize,
target_pc: BranchOffset,
},
// The P4 register values beginning with P3 form an unpacked index key that omits the PRIMARY KEY. Compare this key value against the index that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID fields at the end.
// If the P1 index entry is greater or equal than the key value then jump to P2. Otherwise fall through to the next instruction.
IdxGE {
cursor_id: CursorID,
start_reg: usize,
num_regs: usize,
target_pc: BranchOffset,
},
// The P4 register values beginning with P3 form an unpacked index key that omits the PRIMARY KEY. Compare this key value against the index that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID fields at the end.
// If the P1 index entry is greater than the key value then jump to P2. Otherwise fall through to the next instruction.
IdxGT {
cursor_id: CursorID,
start_reg: usize,
num_regs: usize,
target_pc: BranchOffset,
},
// Decrement the given register and jump to the given PC if the result is zero.
DecrJumpZero {
reg: usize,
target_pc: BranchOffset,
},
AggStep {
acc_reg: usize,
col: usize,
delimiter: usize,
func: AggFunc,
},
AggFinal {
register: usize,
func: AggFunc,
},
// Open a sorter.
SorterOpen {
cursor_id: CursorID, // P1
columns: usize, // P2
order: OwnedRecord, // P4. 0 if ASC and 1 if DESC
},
// Insert a row into the sorter.
SorterInsert {
cursor_id: CursorID,
record_reg: usize,
},
// Sort the rows in the sorter.
SorterSort {
cursor_id: CursorID,
pc_if_empty: BranchOffset,
},
// Retrieve the next row from the sorter.
SorterData {
cursor_id: CursorID, // P1
dest_reg: usize, // P2
pseudo_cursor: usize, // P3
},
// Advance to the next row in the sorter.
SorterNext {
cursor_id: CursorID,
pc_if_next: BranchOffset,
},
// Function
Function {
constant_mask: i32, // P1
start_reg: usize, // P2, start of argument registers
dest: usize, // P3
func: FuncCtx, // P4
},
InitCoroutine {
yield_reg: usize,
jump_on_definition: BranchOffset,
start_offset: BranchOffset,
},
EndCoroutine {
yield_reg: usize,
},
Yield {
yield_reg: usize,
end_offset: BranchOffset,
},
InsertAsync {
cursor: CursorID,
key_reg: usize, // Must be int.
record_reg: usize, // Blob of record data.
flag: usize, // Flags used by insert, for now not used.
},
InsertAwait {
cursor_id: usize,
},
DeleteAsync {
cursor_id: CursorID,
},
DeleteAwait {
cursor_id: CursorID,
},
NewRowid {
cursor: CursorID, // P1
rowid_reg: usize, // P2 Destination register to store the new rowid
prev_largest_reg: usize, // P3 Previous largest rowid in the table (Not used for now)
},
MustBeInt {
reg: usize,
},
SoftNull {
reg: usize,
},
NotExists {
cursor: CursorID,
rowid_reg: usize,
target_pc: BranchOffset,
},
OpenWriteAsync {
cursor_id: CursorID,
root_page: PageIdx,
},
OpenWriteAwait {},
Copy {
src_reg: usize,
dst_reg: usize,
amount: usize, // 0 amount means we include src_reg, dst_reg..=dst_reg+amount = src_reg..=src_reg+amount
},
/// Allocate a new b-tree.
CreateBtree {
/// Allocate b-tree in main database if zero or in temp database if non-zero (P1).
db: usize,
/// The root page of the new b-tree (P2).
root: usize,
/// Flags (P3).
flags: usize,
},
/// Close a cursor.
Close {
cursor_id: CursorID,
},
/// Check if the register is null.
IsNull {
/// Source register (P1).
src: usize,
/// Jump to this PC if the register is null (P2).
target_pc: BranchOffset,
},
ParseSchema {
db: usize,
where_clause: String,
},
}

87
core/vdbe/likeop.rs Normal file
View File

@@ -0,0 +1,87 @@
use regex::{Regex, RegexBuilder};
use crate::{types::OwnedValue, LimboError};
pub fn construct_like_escape_arg(escape_value: &OwnedValue) -> Result<char, LimboError> {
match escape_value {
OwnedValue::Text(text) => {
let mut escape_chars = text.value.chars();
match (escape_chars.next(), escape_chars.next()) {
(Some(escape), None) => Ok(escape),
_ => {
return Result::Err(LimboError::Constraint(
"ESCAPE expression must be a single character".to_string(),
))
}
}
}
_ => {
unreachable!("Like on non-text registers");
}
}
}
// Implements LIKE pattern matching with escape
pub fn exec_like_with_escape(pattern: &str, text: &str, escape: char) -> bool {
construct_like_regex_with_escape(pattern, escape).is_match(text)
}
fn construct_like_regex_with_escape(pattern: &str, escape: char) -> Regex {
let mut regex_pattern = String::with_capacity(pattern.len() * 2);
regex_pattern.push('^');
let mut chars = pattern.chars();
while let Some(ch) = chars.next() {
match ch {
esc_ch if esc_ch == escape => {
if let Some(escaped_char) = chars.next() {
if regex_syntax::is_meta_character(escaped_char) {
regex_pattern.push('\\');
}
regex_pattern.push(escaped_char);
}
}
'%' => regex_pattern.push_str(".*"),
'_' => regex_pattern.push('.'),
c => {
if regex_syntax::is_meta_character(c) {
regex_pattern.push('\\');
}
regex_pattern.push(c);
}
}
}
regex_pattern.push('$');
RegexBuilder::new(&regex_pattern)
.case_insensitive(true)
.dot_matches_new_line(true)
.build()
.unwrap()
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_exec_like_with_escape() {
assert!(exec_like_with_escape("abcX%", "abc%", 'X'));
assert!(!exec_like_with_escape("abcX%", "abc5", 'X'));
assert!(!exec_like_with_escape("abcX%", "abc", 'X'));
assert!(!exec_like_with_escape("abcX%", "abcX%", 'X'));
assert!(!exec_like_with_escape("abcX%", "abc%%", 'X'));
assert!(exec_like_with_escape("abcX_", "abc_", 'X'));
assert!(!exec_like_with_escape("abcX_", "abc5", 'X'));
assert!(!exec_like_with_escape("abcX_", "abc", 'X'));
assert!(!exec_like_with_escape("abcX_", "abcX_", 'X'));
assert!(!exec_like_with_escape("abcX_", "abc__", 'X'));
assert!(exec_like_with_escape("abcXX", "abcX", 'X'));
assert!(!exec_like_with_escape("abcXX", "abc5", 'X'));
assert!(!exec_like_with_escape("abcXX", "abc", 'X'));
assert!(!exec_like_with_escape("abcXX", "abcXX", 'X'));
}
}

788
core/vdbe/mod.rs
View File

@@ -18,14 +18,18 @@
//! https://www.sqlite.org/opcode.html
pub mod builder;
mod datetime;
pub mod explain;
pub mod insn;
pub mod likeop;
pub mod sorter;
mod datetime;
use crate::error::{LimboError, SQLITE_CONSTRAINT_PRIMARYKEY};
#[cfg(feature = "uuid")]
use crate::ext::{exec_ts_from_uuid7, exec_uuid, exec_uuidblob, exec_uuidstr, ExtFunc, UuidFunc};
use crate::function::{AggFunc, FuncCtx, MathFunc, MathFuncArity, ScalarFunc};
use crate::pseudo::PseudoCursor;
use crate::result::LimboResult;
use crate::schema::Table;
use crate::storage::sqlite3_ondisk::DatabaseHeader;
use crate::storage::{btree::BTreeCursor, pager::Pager};
@@ -33,521 +37,27 @@ use crate::types::{
AggContext, Cursor, CursorResult, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp,
};
use crate::util::parse_schema_rows;
use crate::vdbe::insn::Insn;
#[cfg(feature = "json")]
use crate::{function::JsonFunc, json::get_json, json::json_array};
use crate::{Connection, Result, TransactionState};
use crate::{Rows, DATABASE_VERSION};
use crate::{function::JsonFunc, json::get_json, json::json_array, json::json_array_length};
use crate::{Connection, Result, Rows, TransactionState, DATABASE_VERSION};
use datetime::{exec_date, exec_time, exec_unixepoch};
use likeop::{construct_like_escape_arg, exec_like_with_escape};
use crate::json::json_extract;
use rand::distributions::{Distribution, Uniform};
use rand::{thread_rng, Rng};
use regex::Regex;
use std::borrow::BorrowMut;
use regex::{Regex, RegexBuilder};
use std::borrow::{Borrow, BorrowMut};
use std::cell::RefCell;
use std::collections::{BTreeMap, HashMap};
use std::fmt::Display;
use std::rc::{Rc, Weak};
pub type BranchOffset = i64;
pub type CursorID = usize;
pub type PageIdx = usize;
#[allow(dead_code)]
#[derive(Debug)]
pub enum Func {
Scalar(ScalarFunc),
#[cfg(feature = "json")]
Json(JsonFunc),
}
impl Display for Func {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let str = match self {
Func::Scalar(scalar_func) => scalar_func.to_string(),
#[cfg(feature = "json")]
Func::Json(json_func) => json_func.to_string(),
};
write!(f, "{}", str)
}
}
#[derive(Debug)]
pub enum Insn {
// Initialize the program state and jump to the given PC.
Init {
target_pc: BranchOffset,
},
// Write a NULL into register dest. If dest_end is Some, then also write NULL into register dest_end and every register in between dest and dest_end. If dest_end is not set, then only register dest is set to NULL.
Null {
dest: usize,
dest_end: Option<usize>,
},
// Move the cursor P1 to a null row. Any Column operations that occur while the cursor is on the null row will always write a NULL.
NullRow {
cursor_id: CursorID,
},
// Add two registers and store the result in a third register.
Add {
lhs: usize,
rhs: usize,
dest: usize,
},
// Subtract rhs from lhs and store in dest
Subtract {
lhs: usize,
rhs: usize,
dest: usize,
},
// Multiply two registers and store the result in a third register.
Multiply {
lhs: usize,
rhs: usize,
dest: usize,
},
// Divide lhs by rhs and store the result in a third register.
Divide {
lhs: usize,
rhs: usize,
dest: usize,
},
// Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of the comparison for use by the next Jump instruct.
Compare {
start_reg_a: usize,
start_reg_b: usize,
count: usize,
},
// Place the result of rhs bitwise AND lhs in third register.
BitAnd {
lhs: usize,
rhs: usize,
dest: usize,
},
// Place the result of rhs bitwise OR lhs in third register.
BitOr {
lhs: usize,
rhs: usize,
dest: usize,
},
// Place the result of bitwise NOT register P1 in dest register.
BitNot {
reg: usize,
dest: usize,
},
// Jump to the instruction at address P1, P2, or P3 depending on whether in the most recent Compare instruction the P1 vector was less than, equal to, or greater than the P2 vector, respectively.
Jump {
target_pc_lt: BranchOffset,
target_pc_eq: BranchOffset,
target_pc_gt: BranchOffset,
},
// Move the P3 values in register P1..P1+P3-1 over into registers P2..P2+P3-1. Registers P1..P1+P3-1 are left holding a NULL. It is an error for register ranges P1..P1+P3-1 and P2..P2+P3-1 to overlap. It is an error for P3 to be less than 1.
Move {
source_reg: usize,
dest_reg: usize,
count: usize,
},
// If the given register is a positive integer, decrement it by decrement_by and jump to the given PC.
IfPos {
reg: usize,
target_pc: BranchOffset,
decrement_by: usize,
},
// If the given register is not NULL, jump to the given PC.
NotNull {
reg: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if they are equal.
Eq {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if they are not equal.
Ne {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if the left-hand side is less than the right-hand side.
Lt {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if the left-hand side is less than or equal to the right-hand side.
Le {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if the left-hand side is greater than the right-hand side.
Gt {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
// Compare two registers and jump to the given PC if the left-hand side is greater than or equal to the right-hand side.
Ge {
lhs: usize,
rhs: usize,
target_pc: BranchOffset,
},
/// Jump to target_pc if r\[reg\] != 0 or (r\[reg\] == NULL && r\[null_reg\] != 0)
If {
reg: usize, // P1
target_pc: BranchOffset, // P2
/// P3. If r\[reg\] is null, jump iff r\[null_reg\] != 0
null_reg: usize,
},
/// Jump to target_pc if r\[reg\] != 0 or (r\[reg\] == NULL && r\[null_reg\] != 0)
IfNot {
reg: usize, // P1
target_pc: BranchOffset, // P2
/// P3. If r\[reg\] is null, jump iff r\[null_reg\] != 0
null_reg: usize,
},
// Open a cursor for reading.
OpenReadAsync {
cursor_id: CursorID,
root_page: PageIdx,
},
// Await for the completion of open cursor.
OpenReadAwait,
// Open a cursor for a pseudo-table that contains a single row.
OpenPseudo {
cursor_id: CursorID,
content_reg: usize,
num_fields: usize,
},
// Rewind the cursor to the beginning of the B-Tree.
RewindAsync {
cursor_id: CursorID,
},
// Await for the completion of cursor rewind.
RewindAwait {
cursor_id: CursorID,
pc_if_empty: BranchOffset,
},
LastAsync {
cursor_id: CursorID,
},
LastAwait {
cursor_id: CursorID,
pc_if_empty: BranchOffset,
},
// Read a column from the current row of the cursor.
Column {
cursor_id: CursorID,
column: usize,
dest: usize,
},
// Make a record and write it to destination register.
MakeRecord {
start_reg: usize, // P1
count: usize, // P2
dest_reg: usize, // P3
},
// Emit a row of results.
ResultRow {
start_reg: usize, // P1
count: usize, // P2
},
// Advance the cursor to the next row.
NextAsync {
cursor_id: CursorID,
},
// Await for the completion of cursor advance.
NextAwait {
cursor_id: CursorID,
pc_if_next: BranchOffset,
},
PrevAsync {
cursor_id: CursorID,
},
PrevAwait {
cursor_id: CursorID,
pc_if_next: BranchOffset,
},
// Halt the program.
Halt {
err_code: usize,
description: String,
},
// Start a transaction.
Transaction {
write: bool,
},
// Branch to the given PC.
Goto {
target_pc: BranchOffset,
},
// Stores the current program counter into register 'return_reg' then jumps to address target_pc.
Gosub {
target_pc: BranchOffset,
return_reg: usize,
},
// Returns to the program counter stored in register 'return_reg'.
Return {
return_reg: usize,
},
// Write an integer value into a register.
Integer {
value: i64,
dest: usize,
},
// Write a float value into a register
Real {
value: f64,
dest: usize,
},
// If register holds an integer, transform it to a float
RealAffinity {
register: usize,
},
// Write a string value into a register.
String8 {
value: String,
dest: usize,
},
// Write a blob value into a register.
Blob {
value: Vec<u8>,
dest: usize,
},
// Read the rowid of the current row.
RowId {
cursor_id: CursorID,
dest: usize,
},
// Seek to a rowid in the cursor. If not found, jump to the given PC. Otherwise, continue to the next instruction.
SeekRowid {
cursor_id: CursorID,
src_reg: usize,
target_pc: BranchOffset,
},
// P1 is an open index cursor and P3 is a cursor on the corresponding table. This opcode does a deferred seek of the P3 table cursor to the row that corresponds to the current row of P1.
// This is a deferred seek. Nothing actually happens until the cursor is used to read a record. That way, if no reads occur, no unnecessary I/O happens.
DeferredSeek {
index_cursor_id: CursorID,
table_cursor_id: CursorID,
},
// If cursor_id refers to an SQL table (B-Tree that uses integer keys), use the value in start_reg as the key.
// If cursor_id refers to an SQL index, then start_reg is the first in an array of num_regs registers that are used as an unpacked index key.
// Seek to the first index entry that is greater than or equal to the given key. If not found, jump to the given PC. Otherwise, continue to the next instruction.
SeekGE {
is_index: bool,
cursor_id: CursorID,
start_reg: usize,
num_regs: usize,
target_pc: BranchOffset,
},
// If cursor_id refers to an SQL table (B-Tree that uses integer keys), use the value in start_reg as the key.
// If cursor_id refers to an SQL index, then start_reg is the first in an array of num_regs registers that are used as an unpacked index key.
// Seek to the first index entry that is greater than the given key. If not found, jump to the given PC. Otherwise, continue to the next instruction.
SeekGT {
is_index: bool,
cursor_id: CursorID,
start_reg: usize,
num_regs: usize,
target_pc: BranchOffset,
},
// The P4 register values beginning with P3 form an unpacked index key that omits the PRIMARY KEY. Compare this key value against the index that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID fields at the end.
// If the P1 index entry is greater or equal than the key value then jump to P2. Otherwise fall through to the next instruction.
IdxGE {
cursor_id: CursorID,
start_reg: usize,
num_regs: usize,
target_pc: BranchOffset,
},
// The P4 register values beginning with P3 form an unpacked index key that omits the PRIMARY KEY. Compare this key value against the index that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID fields at the end.
// If the P1 index entry is greater than the key value then jump to P2. Otherwise fall through to the next instruction.
IdxGT {
cursor_id: CursorID,
start_reg: usize,
num_regs: usize,
target_pc: BranchOffset,
},
// Decrement the given register and jump to the given PC if the result is zero.
DecrJumpZero {
reg: usize,
target_pc: BranchOffset,
},
AggStep {
acc_reg: usize,
col: usize,
delimiter: usize,
func: AggFunc,
},
AggFinal {
register: usize,
func: AggFunc,
},
// Open a sorter.
SorterOpen {
cursor_id: CursorID, // P1
columns: usize, // P2
order: OwnedRecord, // P4. 0 if ASC and 1 if DESC
},
// Insert a row into the sorter.
SorterInsert {
cursor_id: CursorID,
record_reg: usize,
},
// Sort the rows in the sorter.
SorterSort {
cursor_id: CursorID,
pc_if_empty: BranchOffset,
},
// Retrieve the next row from the sorter.
SorterData {
cursor_id: CursorID, // P1
dest_reg: usize, // P2
pseudo_cursor: usize, // P3
},
// Advance to the next row in the sorter.
SorterNext {
cursor_id: CursorID,
pc_if_next: BranchOffset,
},
// Function
Function {
constant_mask: i32, // P1
start_reg: usize, // P2, start of argument registers
dest: usize, // P3
func: FuncCtx, // P4
},
InitCoroutine {
yield_reg: usize,
jump_on_definition: BranchOffset,
start_offset: BranchOffset,
},
EndCoroutine {
yield_reg: usize,
},
Yield {
yield_reg: usize,
end_offset: BranchOffset,
},
InsertAsync {
cursor: CursorID,
key_reg: usize, // Must be int.
record_reg: usize, // Blob of record data.
flag: usize, // Flags used by insert, for now not used.
},
InsertAwait {
cursor_id: usize,
},
NewRowid {
cursor: CursorID, // P1
rowid_reg: usize, // P2 Destination register to store the new rowid
prev_largest_reg: usize, // P3 Previous largest rowid in the table (Not used for now)
},
MustBeInt {
reg: usize,
},
SoftNull {
reg: usize,
},
NotExists {
cursor: CursorID,
rowid_reg: usize,
target_pc: BranchOffset,
},
OpenWriteAsync {
cursor_id: CursorID,
root_page: PageIdx,
},
OpenWriteAwait {},
Copy {
src_reg: usize,
dst_reg: usize,
amount: usize, // 0 amount means we include src_reg, dst_reg..=dst_reg+amount = src_reg..=src_reg+amount
},
/// Allocate a new b-tree.
CreateBtree {
/// Allocate b-tree in main database if zero or in temp database if non-zero (P1).
db: usize,
/// The root page of the new b-tree (P2).
root: usize,
/// Flags (P3).
flags: usize,
},
/// Close a cursor.
Close {
cursor_id: CursorID,
},
/// Check if the register is null.
IsNull {
/// Source register (P1).
src: usize,
/// Jump to this PC if the register is null (P2).
target_pc: BranchOffset,
},
ParseSchema {
db: usize,
where_clause: String,
},
}
// Index of insn in list of insns
type InsnReference = usize;
@@ -556,6 +66,7 @@ pub enum StepResult<'a> {
IO,
Row(Record<'a>),
Interrupt,
Busy,
}
/// If there is I/O, the instruction is restarted.
@@ -573,9 +84,10 @@ struct RegexCache {
like: HashMap<String, Regex>,
glob: HashMap<String, Regex>,
}
impl RegexCache {
fn new() -> Self {
RegexCache {
Self {
like: HashMap::new(),
glob: HashMap::new(),
}
@@ -1233,6 +745,103 @@ impl Program {
}
state.pc += 1;
}
Insn::Remainder { lhs, rhs, dest } => {
let lhs = *lhs;
let rhs = *rhs;
let dest = *dest;
state.registers[dest] = match (&state.registers[lhs], &state.registers[rhs]) {
(OwnedValue::Null, _)
| (_, OwnedValue::Null)
| (_, OwnedValue::Integer(0))
| (_, OwnedValue::Float(0.0)) => OwnedValue::Null,
(OwnedValue::Integer(lhs), OwnedValue::Integer(rhs)) => {
OwnedValue::Integer(lhs % rhs)
}
(OwnedValue::Float(lhs), OwnedValue::Float(rhs)) => {
OwnedValue::Float(((*lhs as i64) % (*rhs as i64)) as f64)
}
(OwnedValue::Float(lhs), OwnedValue::Integer(rhs)) => {
OwnedValue::Float(((*lhs as i64) % rhs) as f64)
}
(OwnedValue::Integer(lhs), OwnedValue::Float(rhs)) => {
OwnedValue::Float((lhs % *rhs as i64) as f64)
}
(lhs, OwnedValue::Agg(agg_rhs)) => match lhs {
OwnedValue::Agg(agg_lhs) => {
let acc = agg_lhs.final_value();
let acc2 = agg_rhs.final_value();
match (acc, acc2) {
(_, OwnedValue::Integer(0))
| (_, OwnedValue::Float(0.0))
| (_, OwnedValue::Null)
| (OwnedValue::Null, _) => OwnedValue::Null,
(OwnedValue::Integer(l), OwnedValue::Integer(r)) => {
OwnedValue::Integer(l % r)
}
(OwnedValue::Float(lh_f), OwnedValue::Float(rh_f)) => {
OwnedValue::Float(((*lh_f as i64) % (*rh_f as i64)) as f64)
}
(OwnedValue::Integer(lh_i), OwnedValue::Float(rh_f)) => {
OwnedValue::Float((lh_i % (*rh_f as i64)) as f64)
}
_ => {
todo!("{:?} {:?}", acc, acc2);
}
}
}
OwnedValue::Integer(lh_i) => match agg_rhs.final_value() {
OwnedValue::Null => OwnedValue::Null,
OwnedValue::Float(rh_f) => {
OwnedValue::Float((lh_i % (*rh_f as i64)) as f64)
}
OwnedValue::Integer(rh_i) => OwnedValue::Integer(lh_i % rh_i),
_ => {
todo!("{:?}", agg_rhs);
}
},
OwnedValue::Float(lh_f) => match agg_rhs.final_value() {
OwnedValue::Null => OwnedValue::Null,
OwnedValue::Float(rh_f) => {
OwnedValue::Float(((*lh_f as i64) % (*rh_f as i64)) as f64)
}
OwnedValue::Integer(rh_i) => {
OwnedValue::Float(((*lh_f as i64) % rh_i) as f64)
}
_ => {
todo!("{:?}", agg_rhs);
}
},
_ => todo!("{:?}", rhs),
},
(OwnedValue::Agg(aggctx), rhs) => match rhs {
OwnedValue::Integer(rh_i) => match aggctx.final_value() {
OwnedValue::Null => OwnedValue::Null,
OwnedValue::Float(lh_f) => {
OwnedValue::Float(((*lh_f as i64) % rh_i) as f64)
}
OwnedValue::Integer(lh_i) => OwnedValue::Integer(lh_i % rh_i),
_ => {
todo!("{:?}", aggctx);
}
},
OwnedValue::Float(rh_f) => match aggctx.final_value() {
OwnedValue::Null => OwnedValue::Null,
OwnedValue::Float(lh_f) => {
OwnedValue::Float(((*lh_f as i64) % (*rh_f as i64)) as f64)
}
OwnedValue::Integer(lh_i) => {
OwnedValue::Float((lh_i % (*rh_f as i64)) as f64)
}
_ => {
todo!("{:?}", aggctx);
}
},
_ => todo!("{:?}", rhs),
},
_ => todo!("{:?} {:?}", state.registers[lhs], state.registers[rhs]),
};
state.pc += 1;
}
Insn::Null { dest, dest_end } => {
if let Some(dest_end) = dest_end {
for i in *dest..=*dest_end {
@@ -1676,29 +1285,34 @@ impl Program {
}
Insn::Transaction { write } => {
let connection = self.connection.upgrade().unwrap();
if let Some(db) = connection.db.upgrade() {
// TODO(pere): are backpointers good ?? this looks ugly af
// upgrade transaction if needed
let new_transaction_state =
match (db.transaction_state.borrow().clone(), write) {
(crate::TransactionState::Write, true) => TransactionState::Write,
(crate::TransactionState::Write, false) => TransactionState::Write,
(crate::TransactionState::Read, true) => TransactionState::Write,
(crate::TransactionState::Read, false) => TransactionState::Read,
(crate::TransactionState::None, true) => TransactionState::Read,
(crate::TransactionState::None, false) => TransactionState::Read,
};
// TODO(Pere):
// 1. lock wal
// 2. lock shared
// 3. lock write db if write
db.transaction_state.replace(new_transaction_state.clone());
if matches!(new_transaction_state, TransactionState::Write) {
pager.begin_read_tx()?;
} else {
pager.begin_write_tx()?;
let current_state = connection.transaction_state.borrow().clone();
let (new_transaction_state, updated) = match (&current_state, write) {
(crate::TransactionState::Write, true) => (TransactionState::Write, false),
(crate::TransactionState::Write, false) => (TransactionState::Write, false),
(crate::TransactionState::Read, true) => (TransactionState::Write, true),
(crate::TransactionState::Read, false) => (TransactionState::Read, false),
(crate::TransactionState::None, true) => (TransactionState::Write, true),
(crate::TransactionState::None, false) => (TransactionState::Read, true),
};
if updated && matches!(current_state, TransactionState::None) {
if let LimboResult::Busy = pager.begin_read_tx()? {
log::trace!("begin_read_tx busy");
return Ok(StepResult::Busy);
}
}
if updated && matches!(new_transaction_state, TransactionState::Write) {
if let LimboResult::Busy = pager.begin_write_tx()? {
log::trace!("begin_write_tx busy");
return Ok(StepResult::Busy);
}
}
if updated {
connection
.transaction_state
.replace(new_transaction_state.clone());
}
state.pc += 1;
}
Insn::Goto { target_pc } => {
@@ -2309,6 +1923,21 @@ impl Program {
Err(e) => return Err(e),
}
}
#[cfg(feature = "json")]
crate::function::Func::Json(JsonFunc::JsonArrayLength) => {
let json_value = &state.registers[*start_reg];
let path_value = if arg_count > 1 {
Some(&state.registers[*start_reg + 1])
} else {
None
};
let json_array_length = json_array_length(json_value, path_value);
match json_array_length {
Ok(length) => state.registers[*dest] = length,
Err(e) => return Err(e),
}
}
crate::function::Func::Scalar(scalar_func) => match scalar_func {
ScalarFunc::Cast => {
assert!(arg_count == 2);
@@ -2382,7 +2011,25 @@ impl Program {
ScalarFunc::Like => {
let pattern = &state.registers[*start_reg];
let text = &state.registers[*start_reg + 1];
let result = match (pattern, text) {
(OwnedValue::Text(pattern), OwnedValue::Text(text))
if arg_count == 3 =>
{
let escape = match construct_like_escape_arg(
&state.registers[*start_reg + 2],
) {
Ok(x) => x,
Err(e) => return Result::Err(e),
};
OwnedValue::Integer(exec_like_with_escape(
&pattern.value,
&text.value,
escape,
)
as i64)
}
(OwnedValue::Text(pattern), OwnedValue::Text(text)) => {
let cache = if *constant_mask > 0 {
Some(&mut state.regex_cache.like)
@@ -2400,6 +2047,7 @@ impl Program {
unreachable!("Like on non-text registers");
}
};
state.registers[*dest] = result;
}
ScalarFunc::Abs
@@ -2549,6 +2197,39 @@ impl Program {
state.registers[*dest] = exec_replace(source, pattern, replacement);
}
},
#[allow(unreachable_patterns)]
crate::function::Func::Extension(extfn) => match extfn {
#[cfg(feature = "uuid")]
ExtFunc::Uuid(uuidfn) => match uuidfn {
UuidFunc::Uuid4 | UuidFunc::Uuid4Str => {
state.registers[*dest] = exec_uuid(uuidfn, None)?
}
UuidFunc::Uuid7 => match arg_count {
0 => {
state.registers[*dest] =
exec_uuid(uuidfn, None).unwrap_or(OwnedValue::Null);
}
1 => {
let reg_value = state.registers[*start_reg].borrow();
state.registers[*dest] = exec_uuid(uuidfn, Some(reg_value))
.unwrap_or(OwnedValue::Null);
}
_ => unreachable!(),
},
_ => {
// remaining accept 1 arg
let reg_value = state.registers[*start_reg].borrow();
state.registers[*dest] = match uuidfn {
UuidFunc::Uuid7TS => Some(exec_ts_from_uuid7(reg_value)),
UuidFunc::UuidStr => exec_uuidstr(reg_value).ok(),
UuidFunc::UuidBlob => exec_uuidblob(reg_value).ok(),
_ => unreachable!(),
}
.unwrap_or(OwnedValue::Null);
}
},
_ => unreachable!(), // when more extension types are added
},
crate::function::Func::Math(math_func) => match math_func.arity() {
MathFuncArity::Nullary => match math_func {
MathFunc::Pi => {
@@ -2667,6 +2348,16 @@ impl Program {
}
state.pc += 1;
}
Insn::DeleteAsync { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
return_if_io!(cursor.delete());
state.pc += 1;
}
Insn::DeleteAwait { cursor_id } => {
let cursor = cursors.get_mut(cursor_id).unwrap();
cursor.wait_for_completion()?;
state.pc += 1;
}
Insn::NewRowid {
cursor, rowid_reg, ..
} => {
@@ -3162,10 +2853,31 @@ fn exec_char(values: Vec<OwnedValue>) -> OwnedValue {
}
fn construct_like_regex(pattern: &str) -> Regex {
let mut regex_pattern = String::from("(?i)^");
regex_pattern.push_str(&pattern.replace('%', ".*").replace('_', "."));
let mut regex_pattern = String::with_capacity(pattern.len() * 2);
regex_pattern.push('^');
for c in pattern.chars() {
match c {
'\\' => regex_pattern.push_str("\\\\"),
'%' => regex_pattern.push_str(".*"),
'_' => regex_pattern.push('.'),
ch => {
if regex_syntax::is_meta_character(c) {
regex_pattern.push('\\');
}
regex_pattern.push(ch);
}
}
}
regex_pattern.push('$');
Regex::new(&regex_pattern).unwrap()
RegexBuilder::new(&regex_pattern)
.case_insensitive(true)
.dot_matches_new_line(true)
.build()
.unwrap()
}
// Implements LIKE pattern matching. Caches the constructed regex if a cache is provided
@@ -3896,6 +3608,10 @@ mod tests {
unimplemented!()
}
fn delete(&mut self) -> Result<CursorResult<()>> {
unimplemented!()
}
fn wait_for_completion(&mut self) -> Result<()> {
unimplemented!()
}
@@ -4311,12 +4027,18 @@ mod tests {
);
}
#[test]
fn test_like_with_escape_or_regexmeta_chars() {
assert!(exec_like(None, r#"\%A"#, r#"\A"#));
assert!(exec_like(None, "%a%a", "aaaa"));
}
#[test]
fn test_like_no_cache() {
assert!(exec_like(None, "a%", "aaaa"));
assert!(exec_like(None, "%a%a", "aaaa"));
assert!(exec_like(None, "%a.a", "aaaa"));
assert!(exec_like(None, "a.a%", "aaaa"));
assert!(!exec_like(None, "%a.a", "aaaa"));
assert!(!exec_like(None, "a.a%", "aaaa"));
assert!(!exec_like(None, "%a.ab", "aaaa"));
}
@@ -4325,15 +4047,15 @@ mod tests {
let mut cache = HashMap::new();
assert!(exec_like(Some(&mut cache), "a%", "aaaa"));
assert!(exec_like(Some(&mut cache), "%a%a", "aaaa"));
assert!(exec_like(Some(&mut cache), "%a.a", "aaaa"));
assert!(exec_like(Some(&mut cache), "a.a%", "aaaa"));
assert!(!exec_like(Some(&mut cache), "%a.a", "aaaa"));
assert!(!exec_like(Some(&mut cache), "a.a%", "aaaa"));
assert!(!exec_like(Some(&mut cache), "%a.ab", "aaaa"));
// again after values have been cached
assert!(exec_like(Some(&mut cache), "a%", "aaaa"));
assert!(exec_like(Some(&mut cache), "%a%a", "aaaa"));
assert!(exec_like(Some(&mut cache), "%a.a", "aaaa"));
assert!(exec_like(Some(&mut cache), "a.a%", "aaaa"));
assert!(!exec_like(Some(&mut cache), "%a.a", "aaaa"));
assert!(!exec_like(Some(&mut cache), "a.a%", "aaaa"));
assert!(!exec_like(Some(&mut cache), "%a.ab", "aaaa"));
}

4
core/vdbe/sorter.rs
View File

@@ -96,6 +96,10 @@ impl Cursor for Sorter {
Ok(CursorResult::Ok(()))
}
fn delete(&mut self) -> Result<CursorResult<()>> {
unimplemented!()
}
fn set_null_flag(&mut self, _flag: bool) {
todo!();
}

13
macros/Cargo.toml Normal file
View File

@@ -0,0 +1,13 @@
# Copyright 2024 the Limbo authors. All rights reserved. MIT license.
[package]
name = "limbo_macros"
version.workspace = true
authors.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
description = "The Limbo database library"
[lib]
proc-macro = true

135
macros/src/lib.rs Normal file
View File

@@ -0,0 +1,135 @@
extern crate proc_macro;
use proc_macro::{token_stream::IntoIter, Group, TokenStream, TokenTree};
use std::collections::HashMap;
/// A procedural macro that derives a `Description` trait for enums.
/// This macro extracts documentation comments (specified with `/// Description...`) for enum variants
/// and generates an implementation for `get_description`, which returns the associated description.
#[proc_macro_derive(Description, attributes(desc))]
pub fn derive_description_from_doc(item: TokenStream) -> TokenStream {
// Convert the TokenStream into an iterator of TokenTree
let mut tokens = item.into_iter();
let mut enum_name = String::new();
// Vector to store enum variants and their associated payloads (if any)
let mut enum_variants: Vec<(String, Option<String>)> = Vec::<(String, Option<String>)>::new();
// HashMap to store descriptions associated with each enum variant
let mut variant_description_map: HashMap<String, String> = HashMap::new();
// Parses the token stream to extract the enum name and its variants
while let Some(token) = tokens.next() {
match token {
TokenTree::Ident(ident) if ident.to_string() == "enum" => {
// Get the enum name
if let Some(TokenTree::Ident(name)) = tokens.next() {
enum_name = name.to_string();
}
}
TokenTree::Group(group) => {
let mut group_tokens_iter: IntoIter = group.stream().into_iter();
let mut last_seen_desc: Option<String> = None;
while let Some(token) = group_tokens_iter.next() {
match token {
TokenTree::Punct(punct) => {
if punct.to_string() == "#" {
last_seen_desc = process_description(&mut group_tokens_iter);
}
}
TokenTree::Ident(ident) => {
// Capture the enum variant name and associate it with its description
let ident_str = ident.to_string();
if let Some(desc) = &last_seen_desc {
variant_description_map.insert(ident_str.clone(), desc.clone());
}
enum_variants.push((ident_str, None));
last_seen_desc = None;
}
TokenTree::Group(group) => {
// Capture payload information for the current enum variant
if let Some(last_variant) = enum_variants.last_mut() {
last_variant.1 = Some(process_payload(group));
}
}
_ => {}
}
}
}
_ => {}
}
}
generate_get_description(enum_name, &variant_description_map, enum_variants)
}
/// Processes a Rust docs to extract the description string.
fn process_description(token_iter: &mut IntoIter) -> Option<String> {
if let Some(TokenTree::Group(doc_group)) = token_iter.next() {
let mut doc_group_iter = doc_group.stream().into_iter();
// Skip the `desc` and `(` tokens to reach the actual description
doc_group_iter.next();
doc_group_iter.next();
if let Some(TokenTree::Literal(description)) = doc_group_iter.next() {
return Some(description.to_string());
}
}
None
}
/// Processes the payload of an enum variant to extract variable names (ignoring types).
fn process_payload(payload_group: Group) -> String {
let payload_group_iter = payload_group.stream().into_iter();
let mut variable_name_list = String::from("");
let mut is_variable_name = true;
for token in payload_group_iter {
match token {
TokenTree::Ident(ident) => {
if is_variable_name {
variable_name_list.push_str(&format!("{},", ident));
}
is_variable_name = false;
}
TokenTree::Punct(punct) => {
if punct.to_string() == "," {
is_variable_name = true;
}
}
_ => {}
}
}
format!("{{ {} }}", variable_name_list).to_string()
}
/// Generates the `get_description` implementation for the processed enum.
fn generate_get_description(
enum_name: String,
variant_description_map: &HashMap<String, String>,
enum_variants: Vec<(String, Option<String>)>,
) -> TokenStream {
let mut all_enum_arms = String::from("");
for (variant, payload) in enum_variants {
let payload = payload.unwrap_or("".to_string());
let desc;
if let Some(description) = variant_description_map.get(&variant) {
desc = format!("Some({})", description);
} else {
desc = "None".to_string();
}
all_enum_arms.push_str(&format!(
"{}::{} {} => {},\n",
enum_name, variant, payload, desc
));
}
let enum_impl = format!(
"impl {} {{
pub fn get_description(&self) -> Option<&str> {{
match self {{
{}
}}
}}
}}",
enum_name, all_enum_arms
);
enum_impl.parse().unwrap()
}

6
perf/latency/limbo/src/main.rs
View File

@@ -38,11 +38,11 @@ fn main() {
loop {
let row = rows.next_row().unwrap();
match row {
limbo_core::RowResult::Row(_) => {
limbo_core::StepResult::Row(_) => {
count += 1;
}
limbo_core::RowResult::IO => yield,
limbo_core::RowResult::Done => break,
limbo_core::StepResult::IO => yield,
limbo_core::StepResult::Done => break,
}
}
assert!(count == 100);

1
simulator/Cargo.toml
View File

@@ -22,3 +22,4 @@ log = "0.4.20"
tempfile = "3.0.7"
env_logger = "0.10.1"
anarchist-readable-name-generator-lib = "0.1.2"
clap = { version = "4.5", features = ["derive"] }

74
simulator/README.md Normal file
View File

@@ -0,0 +1,74 @@
# Limbo Simulator
Limbo simulator uses randomized deterministic simulations to test the Limbo database behaviors.
Each simulations begins with a random configurations;
- the database workload distribution(percentages of reads, writes, deletes...),
- database parameters(page size),
- number of reader or writers, etc.
Based on these parameters, we randomly generate **interaction plans**. Interaction plans consist of statements/queries, and assertions that will be executed in order. The building blocks of interaction plans are;
- Randomly generated SQL queries satisfying the workload distribution,
- Properties, which contain multiple matching queries with assertions indicating the expected result.
An example of a property is the following:
```json
{
"name": "Read your own writes",
"queries": [
"INSERT INTO t1 (id) VALUES (1)",
"SELECT * FROM t1 WHERE id = 1",
],
"assertions": [
"result.rows.length == 1",
"result.rows[0].id == 1"
]
}
```
The simulator executes the interaction plans in a loop, and checks the assertions. It can add random queries unrelated to the properties without
breaking the property invariants to reach more diverse states and respect the configured workload distribution.
The simulator code is broken into 4 main parts:
- **Simulator(main.rs)**: The main entry point of the simulator. It generates random configurations and interaction plans, and executes them.
- **Model(model.rs, model/table.rs, model/query.rs)**: A simpler model of the database, it contains atomic actions for insertion and selection, we use this model while deciding the next actions.
- **Generation(generation.rs, generation/table.rs, generation/query.rs, generation/plan.rs)**: Random generation functions for the database model and interaction plans.
- **Properties(properties.rs)**: Contains the properties that we want to test.
## Running the simulator
To run the simulator, you can use the following command:
```bash
cargo run
```
This prompt (in the future) will invoke a clap command line interface to configure the simulator. For now, the simulator runs with the default configurations changing the `main.rs` file. If you want to see the logs, you can change the `RUST_LOG` environment variable.
```bash
RUST_LOG=info cargo run --bin limbo_sim
```
## Adding new properties
Todo
## Adding new generation functions
Todo
## Adding new models
Todo
## Coverage with Limbo
Todo
## Automatic Compatibility Testing with SQLite
Todo

65
simulator/generation/mod.rs Normal file
View File

@@ -0,0 +1,65 @@
use anarchist_readable_name_generator_lib::readable_name_custom;
use rand::Rng;
pub mod plan;
pub mod query;
pub mod table;
pub trait Arbitrary {
fn arbitrary<R: Rng>(rng: &mut R) -> Self;
}
pub trait ArbitraryFrom<T> {
fn arbitrary_from<R: Rng>(rng: &mut R, t: &T) -> Self;
}
pub(crate) fn frequency<'a, T, R: rand::Rng>(
choices: Vec<(usize, Box<dyn FnOnce(&mut R) -> T + 'a>)>,
rng: &mut R,
) -> T {
let total = choices.iter().map(|(weight, _)| weight).sum::<usize>();
let mut choice = rng.gen_range(0..total);
for (weight, f) in choices {
if choice < weight {
return f(rng);
}
choice -= weight;
}
unreachable!()
}
pub(crate) fn one_of<'a, T, R: rand::Rng>(
choices: Vec<Box<dyn Fn(&mut R) -> T + 'a>>,
rng: &mut R,
) -> T {
let index = rng.gen_range(0..choices.len());
choices[index](rng)
}
pub(crate) fn pick<'a, T, R: rand::Rng>(choices: &'a Vec<T>, rng: &mut R) -> &'a T {
let index = rng.gen_range(0..choices.len());
&choices[index]
}
pub(crate) fn pick_index<R: rand::Rng>(choices: usize, rng: &mut R) -> usize {
rng.gen_range(0..choices)
}
fn gen_random_text<T: Rng>(rng: &mut T) -> String {
let big_text = rng.gen_ratio(1, 1000);
if big_text {
// let max_size: u64 = 2 * 1024 * 1024 * 1024;
let max_size: u64 = 2 * 1024; // todo: change this back to 2 * 1024 * 1024 * 1024
let size = rng.gen_range(1024..max_size);
let mut name = String::new();
for i in 0..size {
name.push(((i % 26) as u8 + b'A') as char);
}
name
} else {
let name = readable_name_custom("_", rng);
name.replace("-", "_")
}
}

408
simulator/generation/plan.rs Normal file
View File

@@ -0,0 +1,408 @@
use std::{fmt::Display, rc::Rc};
use limbo_core::{Connection, Result, StepResult};
use rand::SeedableRng;
use rand_chacha::ChaCha8Rng;
use crate::{
model::{
query::{Create, Insert, Predicate, Query, Select},
table::Value,
},
SimConnection, SimulatorEnv,
};
use crate::generation::{frequency, Arbitrary, ArbitraryFrom};
use super::{pick, pick_index};
pub(crate) type ResultSet = Vec<Vec<Value>>;
pub(crate) struct InteractionPlan {
pub(crate) plan: Vec<Interaction>,
pub(crate) stack: Vec<ResultSet>,
pub(crate) interaction_pointer: usize,
}
impl Display for InteractionPlan {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
for interaction in &self.plan {
match interaction {
Interaction::Query(query) => writeln!(f, "{};", query)?,
Interaction::Assertion(assertion) => {
writeln!(f, "-- ASSERT: {};", assertion.message)?
}
Interaction::Fault(fault) => writeln!(f, "-- FAULT: {};", fault)?,
}
}
Ok(())
}
}
#[derive(Debug)]
pub(crate) struct InteractionStats {
pub(crate) read_count: usize,
pub(crate) write_count: usize,
pub(crate) delete_count: usize,
}
impl Display for InteractionStats {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"Read: {}, Write: {}, Delete: {}",
self.read_count, self.write_count, self.delete_count
)
}
}
pub(crate) enum Interaction {
Query(Query),
Assertion(Assertion),
Fault(Fault),
}
impl Display for Interaction {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Query(query) => write!(f, "{}", query),
Self::Assertion(assertion) => write!(f, "ASSERT: {}", assertion.message),
Self::Fault(fault) => write!(f, "FAULT: {}", fault),
}
}
}
type AssertionFunc = dyn Fn(&Vec<ResultSet>) -> bool;
pub(crate) struct Assertion {
pub(crate) func: Box<AssertionFunc>,
pub(crate) message: String,
}
pub(crate) enum Fault {
Disconnect,
}
impl Display for Fault {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Fault::Disconnect => write!(f, "DISCONNECT"),
}
}
}
pub(crate) struct Interactions(Vec<Interaction>);
impl Interactions {
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) {
for interaction in &self.0 {
match interaction {
Interaction::Query(query) => match query {
Query::Create(create) => {
env.tables.push(create.table.clone());
}
Query::Insert(insert) => {
let table = env
.tables
.iter_mut()
.find(|t| t.name == insert.table)
.unwrap();
table.rows.extend(insert.values.clone());
}
Query::Delete(_) => todo!(),
Query::Select(_) => {}
},
Interaction::Assertion(_) => {}
Interaction::Fault(_) => {}
}
}
}
}
impl InteractionPlan {
pub(crate) fn new() -> Self {
Self {
plan: Vec::new(),
stack: Vec::new(),
interaction_pointer: 0,
}
}
pub(crate) fn push(&mut self, interaction: Interaction) {
self.plan.push(interaction);
}
pub(crate) fn stats(&self) -> InteractionStats {
let mut read = 0;
let mut write = 0;
let mut delete = 0;
for interaction in &self.plan {
match interaction {
Interaction::Query(query) => match query {
Query::Select(_) => read += 1,
Query::Insert(_) => write += 1,
Query::Delete(_) => delete += 1,
Query::Create(_) => {}
},
Interaction::Assertion(_) => {}
Interaction::Fault(_) => {}
}
}
InteractionStats {
read_count: read,
write_count: write,
delete_count: delete,
}
}
}
impl ArbitraryFrom<SimulatorEnv> for InteractionPlan {
fn arbitrary_from<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Self {
let mut plan = InteractionPlan::new();
let mut env = SimulatorEnv {
opts: env.opts.clone(),
tables: vec![],
connections: vec![],
io: env.io.clone(),
db: env.db.clone(),
rng: ChaCha8Rng::seed_from_u64(rng.next_u64()),
};
let num_interactions = rng.gen_range(0..env.opts.max_interactions);
// First create at least one table
let create_query = Create::arbitrary(rng);
env.tables.push(create_query.table.clone());
plan.push(Interaction::Query(Query::Create(create_query)));
while plan.plan.len() < num_interactions {
log::debug!(
"Generating interaction {}/{}",
plan.plan.len(),
num_interactions
);
let interactions = Interactions::arbitrary_from(rng, &(&env, plan.stats()));
interactions.shadow(&mut env);
plan.plan.extend(interactions.0.into_iter());
}
log::info!("Generated plan with {} interactions", plan.plan.len());
plan
}
}
impl Interaction {
pub(crate) fn execute_query(&self, conn: &mut Rc<Connection>) -> Result<ResultSet> {
match self {
Self::Query(query) => {
let query_str = query.to_string();
let rows = conn.query(&query_str);
if rows.is_err() {
let err = rows.err();
log::error!(
"Error running query '{}': {:?}",
&query_str[0..query_str.len().min(4096)],
err
);
return Err(err.unwrap());
}
let rows = rows.unwrap();
assert!(rows.is_some());
let mut rows = rows.unwrap();
let mut out = Vec::new();
while let Ok(row) = rows.next_row() {
match row {
StepResult::Row(row) => {
let mut r = Vec::new();
for el in &row.values {
let v = match el {
limbo_core::Value::Null => Value::Null,
limbo_core::Value::Integer(i) => Value::Integer(*i),
limbo_core::Value::Float(f) => Value::Float(*f),
limbo_core::Value::Text(t) => Value::Text(t.to_string()),
limbo_core::Value::Blob(b) => Value::Blob(b.to_vec()),
};
r.push(v);
}
out.push(r);
}
StepResult::IO => {}
StepResult::Interrupt => {}
StepResult::Done => {
break;
}
StepResult::Busy => {}
}
}
Ok(out)
}
Self::Assertion(_) => {
unreachable!("unexpected: this function should only be called on queries")
}
Interaction::Fault(_) => {
unreachable!("unexpected: this function should only be called on queries")
}
}
}
pub(crate) fn execute_assertion(&self, stack: &Vec<ResultSet>) -> Result<()> {
match self {
Self::Query(_) => {
unreachable!("unexpected: this function should only be called on assertions")
}
Self::Assertion(assertion) => {
if !assertion.func.as_ref()(stack) {
return Err(limbo_core::LimboError::InternalError(
assertion.message.clone(),
));
}
Ok(())
}
Self::Fault(_) => {
unreachable!("unexpected: this function should only be called on assertions")
}
}
}
pub(crate) fn execute_fault(&self, env: &mut SimulatorEnv, conn_index: usize) -> Result<()> {
match self {
Self::Query(_) => {
unreachable!("unexpected: this function should only be called on faults")
}
Self::Assertion(_) => {
unreachable!("unexpected: this function should only be called on faults")
}
Self::Fault(fault) => {
match fault {
Fault::Disconnect => {
match env.connections[conn_index] {
SimConnection::Connected(ref mut conn) => {
conn.close()?;
}
SimConnection::Disconnected => {
return Err(limbo_core::LimboError::InternalError(
"Tried to disconnect a disconnected connection".to_string(),
));
}
}
env.connections[conn_index] = SimConnection::Disconnected;
}
}
Ok(())
}
}
}
}
fn property_insert_select<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Interactions {
// Get a random table
let table = pick(&env.tables, rng);
// Pick a random column
let column_index = pick_index(table.columns.len(), rng);
let column = &table.columns[column_index].clone();
// Generate a random value of the column type
let value = Value::arbitrary_from(rng, &column.column_type);
// Create a whole new row
let mut row = Vec::new();
for (i, column) in table.columns.iter().enumerate() {
if i == column_index {
row.push(value.clone());
} else {
let value = Value::arbitrary_from(rng, &column.column_type);
row.push(value);
}
}
// Insert the row
let insert_query = Interaction::Query(Query::Insert(Insert {
table: table.name.clone(),
values: vec![row.clone()],
}));
// Select the row
let select_query = Interaction::Query(Query::Select(Select {
table: table.name.clone(),
predicate: Predicate::Eq(column.name.clone(), value.clone()),
}));
// Check that the row is there
let assertion = Interaction::Assertion(Assertion {
message: format!(
"row [{:?}] not found in table {} after inserting ({} = {})",
row.iter().map(|v| v.to_string()).collect::<Vec<String>>(),
table.name,
column.name,
value,
),
func: Box::new(move |stack: &Vec<ResultSet>| {
let rows = stack.last().unwrap();
rows.iter().any(|r| r == &row)
}),
});
Interactions(vec![insert_query, select_query, assertion])
}
fn create_table<R: rand::Rng>(rng: &mut R, _env: &SimulatorEnv) -> Interactions {
let create_query = Interaction::Query(Query::Create(Create::arbitrary(rng)));
Interactions(vec![create_query])
}
fn random_read<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Interactions {
let select_query = Interaction::Query(Query::Select(Select::arbitrary_from(rng, &env.tables)));
Interactions(vec![select_query])
}
fn random_write<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Interactions {
let table = pick(&env.tables, rng);
let insert_query = Interaction::Query(Query::Insert(Insert::arbitrary_from(rng, table)));
Interactions(vec![insert_query])
}
fn random_fault<R: rand::Rng>(_rng: &mut R, _env: &SimulatorEnv) -> Interactions {
let fault = Interaction::Fault(Fault::Disconnect);
Interactions(vec![fault])
}
impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions {
fn arbitrary_from<R: rand::Rng>(
rng: &mut R,
(env, stats): &(&SimulatorEnv, InteractionStats),
) -> Self {
let remaining_read =
((((env.opts.max_interactions * env.opts.read_percent) as f64) / 100.0) as usize)
.saturating_sub(stats.read_count);
let remaining_write = ((((env.opts.max_interactions * env.opts.write_percent) as f64)
/ 100.0) as usize)
.saturating_sub(stats.write_count);
frequency(
vec![
(
usize::min(remaining_read, remaining_write),
Box::new(|rng: &mut R| property_insert_select(rng, env)),
),
(
remaining_read,
Box::new(|rng: &mut R| random_read(rng, env)),
),
(
remaining_write,
Box::new(|rng: &mut R| random_write(rng, env)),
),
(
remaining_write / 10,
Box::new(|rng: &mut R| create_table(rng, env)),
),
(1, Box::new(|rng: &mut R| random_fault(rng, env))),
],
rng,
)
}
}

247
simulator/generation/query.rs Normal file
View File

@@ -0,0 +1,247 @@
use crate::generation::table::{GTValue, LTValue};
use crate::generation::{one_of, Arbitrary, ArbitraryFrom};
use crate::model::query::{Create, Delete, Insert, Predicate, Query, Select};
use crate::model::table::{Table, Value};
use rand::Rng;
use super::{frequency, pick};
impl Arbitrary for Create {
fn arbitrary<R: Rng>(rng: &mut R) -> Self {
Create {
table: Table::arbitrary(rng),
}
}
}
impl ArbitraryFrom<Vec<Table>> for Select {
fn arbitrary_from<R: Rng>(rng: &mut R, tables: &Vec<Table>) -> Self {
let table = pick(tables, rng);
Self {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, table),
}
}
}
impl ArbitraryFrom<Vec<&Table>> for Select {
fn arbitrary_from<R: Rng>(rng: &mut R, tables: &Vec<&Table>) -> Self {
let table = pick(tables, rng);
Self {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, *table),
}
}
}
impl ArbitraryFrom<Table> for Insert {
fn arbitrary_from<R: Rng>(rng: &mut R, table: &Table) -> Self {
let num_rows = rng.gen_range(1..10);
let values: Vec<Vec<Value>> = (0..num_rows)
.map(|_| {
table
.columns
.iter()
.map(|c| Value::arbitrary_from(rng, &c.column_type))
.collect()
})
.collect();
Self {
table: table.name.clone(),
values,
}
}
}
impl ArbitraryFrom<Table> for Delete {
fn arbitrary_from<R: Rng>(rng: &mut R, table: &Table) -> Self {
Self {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, table),
}
}
}
impl ArbitraryFrom<Table> for Query {
fn arbitrary_from<R: Rng>(rng: &mut R, table: &Table) -> Self {
frequency(
vec![
(1, Box::new(|rng| Self::Create(Create::arbitrary(rng)))),
(
100,
Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![table]))),
),
(
100,
Box::new(|rng| Self::Insert(Insert::arbitrary_from(rng, table))),
),
(
0,
Box::new(|rng| Self::Delete(Delete::arbitrary_from(rng, table))),
),
],
rng,
)
}
}
struct CompoundPredicate(Predicate);
struct SimplePredicate(Predicate);
impl ArbitraryFrom<(&Table, bool)> for SimplePredicate {
fn arbitrary_from<R: Rng>(rng: &mut R, (table, predicate_value): &(&Table, bool)) -> Self {
// Pick a random column
let column_index = rng.gen_range(0..table.columns.len());
let column = &table.columns[column_index];
let column_values = table
.rows
.iter()
.map(|r| &r[column_index])
.collect::<Vec<_>>();
// Pick an operator
let operator = match predicate_value {
true => one_of(
vec![
Box::new(|rng| {
Predicate::Eq(
column.name.clone(),
Value::arbitrary_from(rng, &column_values),
)
}),
Box::new(|rng| {
Predicate::Gt(
column.name.clone(),
GTValue::arbitrary_from(rng, &column_values).0,
)
}),
Box::new(|rng| {
Predicate::Lt(
column.name.clone(),
LTValue::arbitrary_from(rng, &column_values).0,
)
}),
],
rng,
),
false => one_of(
vec![
Box::new(|rng| {
Predicate::Neq(
column.name.clone(),
Value::arbitrary_from(rng, &column.column_type),
)
}),
Box::new(|rng| {
Predicate::Gt(
column.name.clone(),
LTValue::arbitrary_from(rng, &column_values).0,
)
}),
Box::new(|rng| {
Predicate::Lt(
column.name.clone(),
GTValue::arbitrary_from(rng, &column_values).0,
)
}),
],
rng,
),
};
Self(operator)
}
}
impl ArbitraryFrom<(&Table, bool)> for CompoundPredicate {
fn arbitrary_from<R: Rng>(rng: &mut R, (table, predicate_value): &(&Table, bool)) -> Self {
// Decide if you want to create an AND or an OR
Self(if rng.gen_bool(0.7) {
// An AND for true requires each of its children to be true
// An AND for false requires at least one of its children to be false
if *predicate_value {
Predicate::And(
(0..rng.gen_range(1..=3))
.map(|_| SimplePredicate::arbitrary_from(rng, &(*table, true)).0)
.collect(),
)
} else {
// Create a vector of random booleans
let mut booleans = (0..rng.gen_range(1..=3))
.map(|_| rng.gen_bool(0.5))
.collect::<Vec<_>>();
let len = booleans.len();
// Make sure at least one of them is false
if booleans.iter().all(|b| *b) {
booleans[rng.gen_range(0..len)] = false;
}
Predicate::And(
booleans
.iter()
.map(|b| SimplePredicate::arbitrary_from(rng, &(*table, *b)).0)
.collect(),
)
}
} else {
// An OR for true requires at least one of its children to be true
// An OR for false requires each of its children to be false
if *predicate_value {
// Create a vector of random booleans
let mut booleans = (0..rng.gen_range(1..=3))
.map(|_| rng.gen_bool(0.5))
.collect::<Vec<_>>();
let len = booleans.len();
// Make sure at least one of them is true
if booleans.iter().all(|b| !*b) {
booleans[rng.gen_range(0..len)] = true;
}
Predicate::Or(
booleans
.iter()
.map(|b| SimplePredicate::arbitrary_from(rng, &(*table, *b)).0)
.collect(),
)
} else {
Predicate::Or(
(0..rng.gen_range(1..=3))
.map(|_| SimplePredicate::arbitrary_from(rng, &(*table, false)).0)
.collect(),
)
}
})
}
}
impl ArbitraryFrom<Table> for Predicate {
fn arbitrary_from<R: Rng>(rng: &mut R, table: &Table) -> Self {
let predicate_value = rng.gen_bool(0.5);
CompoundPredicate::arbitrary_from(rng, &(table, predicate_value)).0
}
}
impl ArbitraryFrom<(&str, &Value)> for Predicate {
fn arbitrary_from<R: Rng>(rng: &mut R, (column_name, value): &(&str, &Value)) -> Self {
one_of(
vec![
Box::new(|_| Predicate::Eq(column_name.to_string(), (*value).clone())),
Box::new(|rng| {
Self::Gt(
column_name.to_string(),
GTValue::arbitrary_from(rng, *value).0,
)
}),
Box::new(|rng| {
Self::Lt(
column_name.to_string(),
LTValue::arbitrary_from(rng, *value).0,
)
}),
],
rng,
)
}
}

191
simulator/generation/table.rs Normal file
View File

@@ -0,0 +1,191 @@
use rand::Rng;
use crate::generation::{
gen_random_text, pick, pick_index, readable_name_custom, Arbitrary, ArbitraryFrom,
};
use crate::model::table::{Column, ColumnType, Name, Table, Value};
impl Arbitrary for Name {
fn arbitrary<R: Rng>(rng: &mut R) -> Self {
let name = readable_name_custom("_", rng);
Name(name.replace("-", "_"))
}
}
impl Arbitrary for Table {
fn arbitrary<R: Rng>(rng: &mut R) -> Self {
let name = Name::arbitrary(rng).0;
let columns = (1..=rng.gen_range(1..5))
.map(|_| Column::arbitrary(rng))
.collect();
Table {
rows: Vec::new(),
name,
columns,
}
}
}
impl Arbitrary for Column {
fn arbitrary<R: Rng>(rng: &mut R) -> Self {
let name = Name::arbitrary(rng).0;
let column_type = ColumnType::arbitrary(rng);
Self {
name,
column_type,
primary: false,
unique: false,
}
}
}
impl Arbitrary for ColumnType {
fn arbitrary<R: Rng>(rng: &mut R) -> Self {
pick(
&vec![Self::Integer, Self::Float, Self::Text, Self::Blob],
rng,
)
.to_owned()
}
}
impl ArbitraryFrom<Vec<&Value>> for Value {
fn arbitrary_from<R: Rng>(rng: &mut R, values: &Vec<&Self>) -> Self {
if values.is_empty() {
return Self::Null;
}
pick(values, rng).to_owned().clone()
}
}
impl ArbitraryFrom<ColumnType> for Value {
fn arbitrary_from<R: Rng>(rng: &mut R, column_type: &ColumnType) -> Self {
match column_type {
ColumnType::Integer => Self::Integer(rng.gen_range(i64::MIN..i64::MAX)),
ColumnType::Float => Self::Float(rng.gen_range(-1e10..1e10)),
ColumnType::Text => Self::Text(gen_random_text(rng)),
ColumnType::Blob => Self::Blob(gen_random_text(rng).as_bytes().to_vec()),
}
}
}
pub(crate) struct LTValue(pub(crate) Value);
impl ArbitraryFrom<Vec<&Value>> for LTValue {
fn arbitrary_from<R: Rng>(rng: &mut R, values: &Vec<&Value>) -> Self {
if values.is_empty() {
return Self(Value::Null);
}
let index = pick_index(values.len(), rng);
Self::arbitrary_from(rng, values[index])
}
}
impl ArbitraryFrom<Value> for LTValue {
fn arbitrary_from<R: Rng>(rng: &mut R, value: &Value) -> Self {
match value {
Value::Integer(i) => Self(Value::Integer(rng.gen_range(i64::MIN..*i - 1))),
Value::Float(f) => Self(Value::Float(rng.gen_range(-1e10..*f - 1.0))),
Value::Text(t) => {
// Either shorten the string, or make at least one character smaller and mutate the rest
let mut t = t.clone();
if rng.gen_bool(0.01) {
t.pop();
Self(Value::Text(t))
} else {
let mut t = t.chars().map(|c| c as u32).collect::<Vec<_>>();
let index = rng.gen_range(0..t.len());
t[index] -= 1;
// Mutate the rest of the string
for i in (index + 1)..t.len() {
t[i] = rng.gen_range('a' as u32..='z' as u32);
}
let t = t
.into_iter()
.map(|c| char::from_u32(c).unwrap_or('z'))
.collect::<String>();
Self(Value::Text(t))
}
}
Value::Blob(b) => {
// Either shorten the blob, or make at least one byte smaller and mutate the rest
let mut b = b.clone();
if rng.gen_bool(0.01) {
b.pop();
Self(Value::Blob(b))
} else {
let index = rng.gen_range(0..b.len());
b[index] -= 1;
// Mutate the rest of the blob
for i in (index + 1)..b.len() {
b[i] = rng.gen_range(0..=255);
}
Self(Value::Blob(b))
}
}
_ => unreachable!(),
}
}
}
pub(crate) struct GTValue(pub(crate) Value);
impl ArbitraryFrom<Vec<&Value>> for GTValue {
fn arbitrary_from<R: Rng>(rng: &mut R, values: &Vec<&Value>) -> Self {
if values.is_empty() {
return Self(Value::Null);
}
let index = pick_index(values.len(), rng);
Self::arbitrary_from(rng, values[index])
}
}
impl ArbitraryFrom<Value> for GTValue {
fn arbitrary_from<R: Rng>(rng: &mut R, value: &Value) -> Self {
match value {
Value::Integer(i) => Self(Value::Integer(rng.gen_range(*i..i64::MAX))),
Value::Float(f) => Self(Value::Float(rng.gen_range(*f..1e10))),
Value::Text(t) => {
// Either lengthen the string, or make at least one character smaller and mutate the rest
let mut t = t.clone();
if rng.gen_bool(0.01) {
t.push(rng.gen_range(0..=255) as u8 as char);
Self(Value::Text(t))
} else {
let mut t = t.chars().map(|c| c as u32).collect::<Vec<_>>();
let index = rng.gen_range(0..t.len());
t[index] += 1;
// Mutate the rest of the string
for i in (index + 1)..t.len() {
t[i] = rng.gen_range('a' as u32..='z' as u32);
}
let t = t
.into_iter()
.map(|c| char::from_u32(c).unwrap_or('a'))
.collect::<String>();
Self(Value::Text(t))
}
}
Value::Blob(b) => {
// Either lengthen the blob, or make at least one byte smaller and mutate the rest
let mut b = b.clone();
if rng.gen_bool(0.01) {
b.push(rng.gen_range(0..=255));
Self(Value::Blob(b))
} else {
let index = rng.gen_range(0..b.len());
b[index] += 1;
// Mutate the rest of the blob
for i in (index + 1)..b.len() {
b[i] = rng.gen_range(0..=255);
}
Self(Value::Blob(b))
}
}
_ => unreachable!(),
}
}
}

730
simulator/main.rs
View File

@@ -1,80 +1,139 @@
use limbo_core::{Connection, Database, File, OpenFlags, PlatformIO, Result, RowResult, IO};
#![allow(clippy::arc_with_non_send_sync, dead_code)]
use clap::Parser;
use generation::plan::{Interaction, InteractionPlan, ResultSet};
use generation::{pick_index, ArbitraryFrom};
use limbo_core::{Database, Result};
use model::table::Value;
use rand::prelude::*;
use rand_chacha::ChaCha8Rng;
use std::cell::RefCell;
use std::rc::Rc;
use runner::cli::SimulatorCLI;
use runner::env::{SimConnection, SimulatorEnv, SimulatorOpts};
use runner::io::SimulatorIO;
use std::backtrace::Backtrace;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use tempfile::TempDir;
use anarchist_readable_name_generator_lib::readable_name_custom;
mod generation;
mod model;
mod runner;
struct SimulatorEnv {
opts: SimulatorOpts,
tables: Vec<Table>,
connections: Vec<SimConnection>,
io: Arc<SimulatorIO>,
db: Arc<Database>,
rng: ChaCha8Rng,
}
#[derive(Clone)]
enum SimConnection {
Connected(Rc<Connection>),
Disconnected,
}
#[derive(Debug)]
struct SimulatorOpts {
ticks: usize,
max_connections: usize,
max_tables: usize,
// this next options are the distribution of workload where read_percent + write_percent +
// delete_percent == 100%
read_percent: usize,
write_percent: usize,
delete_percent: usize,
page_size: usize,
}
struct Table {
rows: Vec<Vec<Value>>,
name: String,
columns: Vec<Column>,
}
#[derive(Clone)]
struct Column {
name: String,
column_type: ColumnType,
primary: bool,
unique: bool,
}
#[derive(Clone)]
enum ColumnType {
Integer,
Float,
Text,
Blob,
}
#[derive(Debug, PartialEq)]
enum Value {
Null,
Integer(i64),
Float(f64),
Text(String),
Blob(Vec<u8>),
}
#[allow(clippy::arc_with_non_send_sync)]
fn main() {
let _ = env_logger::try_init();
let seed = match std::env::var("SEED") {
Ok(seed) => seed.parse::<u64>().unwrap(),
Err(_) => rand::thread_rng().next_u64(),
let cli_opts = SimulatorCLI::parse();
let seed = match cli_opts.seed {
Some(seed) => seed,
None => rand::thread_rng().next_u64(),
};
println!("Seed: {}", seed);
let output_dir = match &cli_opts.output_dir {
Some(dir) => Path::new(dir).to_path_buf(),
None => TempDir::new().unwrap().into_path(),
};
let db_path = output_dir.join("simulator.db");
let plan_path = output_dir.join("simulator.plan");
// Print the seed, the locations of the database and the plan file
log::info!("database path: {:?}", db_path);
log::info!("simulator plan path: {:?}", plan_path);
log::info!("seed: {}", seed);
std::panic::set_hook(Box::new(move |info| {
log::error!("panic occurred");
let payload = info.payload();
if let Some(s) = payload.downcast_ref::<&str>() {
log::error!("{}", s);
} else if let Some(s) = payload.downcast_ref::<String>() {
log::error!("{}", s);
} else {
log::error!("unknown panic payload");
}
let bt = Backtrace::force_capture();
log::error!("captured backtrace:\n{}", bt);
}));
let result = std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path));
if cli_opts.doublecheck {
// Move the old database and plan file to a new location
let old_db_path = db_path.with_extension("_old.db");
let old_plan_path = plan_path.with_extension("_old.plan");
std::fs::rename(&db_path, &old_db_path).unwrap();
std::fs::rename(&plan_path, &old_plan_path).unwrap();
// Run the simulation again
let result2 =
std::panic::catch_unwind(|| run_simulation(seed, &cli_opts, &db_path, &plan_path));
match (result, result2) {
(Ok(Ok(_)), Err(_)) => {
log::error!("doublecheck failed! first run succeeded, but second run panicked.");
}
(Ok(Err(_)), Err(_)) => {
log::error!(
"doublecheck failed! first run failed assertion, but second run panicked."
);
}
(Err(_), Ok(Ok(_))) => {
log::error!("doublecheck failed! first run panicked, but second run succeeded.");
}
(Err(_), Ok(Err(_))) => {
log::error!(
"doublecheck failed! first run panicked, but second run failed assertion."
);
}
(Ok(Ok(_)), Ok(Err(_))) => {
log::error!(
"doublecheck failed! first run succeeded, but second run failed assertion."
);
}
(Ok(Err(_)), Ok(Ok(_))) => {
log::error!(
"doublecheck failed! first run failed assertion, but second run succeeded."
);
}
(Err(_), Err(_)) | (Ok(_), Ok(_)) => {
// Compare the two database files byte by byte
let old_db = std::fs::read(&old_db_path).unwrap();
let new_db = std::fs::read(&db_path).unwrap();
if old_db != new_db {
log::error!("doublecheck failed! database files are different.");
} else {
log::info!("doublecheck succeeded! database files are the same.");
}
}
}
// Move the new database and plan file to a new location
let new_db_path = db_path.with_extension("_double.db");
let new_plan_path = plan_path.with_extension("_double.plan");
std::fs::rename(&db_path, &new_db_path).unwrap();
std::fs::rename(&plan_path, &new_plan_path).unwrap();
// Move the old database and plan file back
std::fs::rename(&old_db_path, &db_path).unwrap();
std::fs::rename(&old_plan_path, &plan_path).unwrap();
}
// Print the seed, the locations of the database and the plan file at the end again for easily accessing them.
println!("database path: {:?}", db_path);
println!("simulator plan path: {:?}", plan_path);
println!("seed: {}", seed);
}
fn run_simulation(
seed: u64,
cli_opts: &SimulatorCLI,
db_path: &Path,
plan_path: &Path,
) -> Result<()> {
let mut rng = ChaCha8Rng::seed_from_u64(seed);
let (read_percent, write_percent, delete_percent) = {
@@ -87,8 +146,12 @@ fn main() {
(read_percent, write_percent, delete_percent)
};
if cli_opts.maximum_size < 1 {
panic!("maximum size must be at least 1");
}
let opts = SimulatorOpts {
ticks: rng.gen_range(0..4096),
ticks: rng.gen_range(1..=cli_opts.maximum_size),
max_connections: 1, // TODO: for now let's use one connection as we didn't implement
// correct transactions procesing
max_tables: rng.gen_range(0..128),
@@ -96,20 +159,19 @@ fn main() {
write_percent,
delete_percent,
page_size: 4096, // TODO: randomize this too
max_interactions: rng.gen_range(1..=cli_opts.maximum_size),
};
let io = Arc::new(SimulatorIO::new(seed, opts.page_size).unwrap());
let mut path = TempDir::new().unwrap().into_path();
path.push("simulator.db");
println!("path to db '{:?}'", path);
let db = match Database::open_file(io.clone(), path.as_path().to_str().unwrap()) {
let db = match Database::open_file(io.clone(), db_path.to_str().unwrap()) {
Ok(db) => db,
Err(e) => {
panic!("error opening simulator test file {:?}: {:?}", path, e);
panic!("error opening simulator test file {:?}: {:?}", db_path, e);
}
};
let connections = vec![SimConnection::Disconnected; opts.max_connections];
let mut env = SimulatorEnv {
opts,
tables: Vec::new(),
@@ -119,108 +181,104 @@ fn main() {
db,
};
println!("Initial opts {:?}", env.opts);
log::info!("Generating database interaction plan...");
let mut plans = (1..=env.opts.max_connections)
.map(|_| InteractionPlan::arbitrary_from(&mut env.rng.clone(), &env))
.collect::<Vec<_>>();
for _ in 0..env.opts.ticks {
let connection_index = env.rng.gen_range(0..env.opts.max_connections);
let mut connection = env.connections[connection_index].clone();
let mut f = std::fs::File::create(plan_path).unwrap();
// todo: create a detailed plan file with all the plans. for now, we only use 1 connection, so it's safe to use the first plan.
f.write_all(plans[0].to_string().as_bytes()).unwrap();
match &mut connection {
SimConnection::Connected(conn) => {
let disconnect = env.rng.gen_ratio(1, 100);
if disconnect {
log::info!("disconnecting {}", connection_index);
let _ = conn.close();
env.connections[connection_index] = SimConnection::Disconnected;
} else {
match process_connection(&mut env, conn) {
Ok(_) => {}
Err(err) => {
log::error!("error {}", err);
break;
}
}
}
}
SimConnection::Disconnected => {
log::info!("disconnecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect());
}
}
log::info!("{}", plans[0].stats());
log::info!("Executing database interaction plan...");
let result = execute_plans(&mut env, &mut plans);
if result.is_err() {
log::error!("error executing plans: {:?}", result.as_ref().err());
}
env.io.print_stats();
log::info!("Simulation completed");
result
}
fn process_connection(env: &mut SimulatorEnv, conn: &mut Rc<Connection>) -> Result<()> {
let management = env.rng.gen_ratio(1, 100);
if management {
// for now create table only
maybe_add_table(env, conn)?;
} else if env.tables.is_empty() {
maybe_add_table(env, conn)?;
fn execute_plans(env: &mut SimulatorEnv, plans: &mut [InteractionPlan]) -> Result<()> {
// todo: add history here by recording which interaction was executed at which tick
for _tick in 0..env.opts.ticks {
// Pick the connection to interact with
let connection_index = pick_index(env.connections.len(), &mut env.rng);
// Execute the interaction for the selected connection
execute_plan(env, connection_index, plans)?;
}
Ok(())
}
fn execute_plan(
env: &mut SimulatorEnv,
connection_index: usize,
plans: &mut [InteractionPlan],
) -> Result<()> {
let connection = &env.connections[connection_index];
let plan = &mut plans[connection_index];
if plan.interaction_pointer >= plan.plan.len() {
return Ok(());
}
let interaction = &plan.plan[plan.interaction_pointer];
if let SimConnection::Disconnected = connection {
log::info!("connecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect());
} else {
let roll = env.rng.gen_range(0..100);
if roll < env.opts.read_percent {
// read
do_select(env, conn)?;
} else if roll < env.opts.read_percent + env.opts.write_percent {
// write
do_write(env, conn)?;
} else {
// delete
// TODO
match execute_interaction(env, connection_index, interaction, &mut plan.stack) {
Ok(_) => {
log::debug!("connection {} processed", connection_index);
plan.interaction_pointer += 1;
}
Err(err) => {
log::error!("error {}", err);
return Err(err);
}
}
}
Ok(())
}
fn do_select(env: &mut SimulatorEnv, conn: &mut Rc<Connection>) -> Result<()> {
let table = env.rng.gen_range(0..env.tables.len());
let table_name = {
let table = &env.tables[table];
table.name.clone()
};
let rows = get_all_rows(env, conn, format!("SELECT * FROM {}", table_name).as_str())?;
fn execute_interaction(
env: &mut SimulatorEnv,
connection_index: usize,
interaction: &Interaction,
stack: &mut Vec<ResultSet>,
) -> Result<()> {
log::info!("executing: {}", interaction);
match interaction {
generation::plan::Interaction::Query(_) => {
let conn = match &mut env.connections[connection_index] {
SimConnection::Connected(conn) => conn,
SimConnection::Disconnected => unreachable!(),
};
let table = &env.tables[table];
compare_equal_rows(&table.rows, &rows);
Ok(())
}
fn do_write(env: &mut SimulatorEnv, conn: &mut Rc<Connection>) -> Result<()> {
let mut query = String::new();
let table = env.rng.gen_range(0..env.tables.len());
{
let table = &env.tables[table];
query.push_str(format!("INSERT INTO {} VALUES (", table.name).as_str());
log::debug!("{}", interaction);
let results = interaction.execute_query(conn)?;
log::debug!("{:?}", results);
stack.push(results);
}
generation::plan::Interaction::Assertion(_) => {
interaction.execute_assertion(stack)?;
stack.clear();
}
Interaction::Fault(_) => {
interaction.execute_fault(env, connection_index)?;
}
}
let columns = env.tables[table].columns.clone();
let mut row = Vec::new();
// gen insert query
for column in &columns {
let value = match column.column_type {
ColumnType::Integer => Value::Integer(env.rng.gen_range(i64::MIN..i64::MAX)),
ColumnType::Float => Value::Float(env.rng.gen_range(-1e10..1e10)),
ColumnType::Text => Value::Text(gen_random_text(env)),
ColumnType::Blob => Value::Blob(gen_random_text(env).as_bytes().to_vec()),
};
query.push_str(value.to_string().as_str());
query.push(',');
row.push(value);
}
let table = &mut env.tables[table];
table.rows.push(row);
query.pop();
query.push_str(");");
let _ = get_all_rows(env, conn, query.as_str())?;
Ok(())
}
@@ -232,355 +290,3 @@ fn compare_equal_rows(a: &[Vec<Value>], b: &[Vec<Value>]) {
}
}
}
fn maybe_add_table(env: &mut SimulatorEnv, conn: &mut Rc<Connection>) -> Result<()> {
if env.tables.len() < env.opts.max_tables {
let table = Table {
rows: Vec::new(),
name: gen_random_name(env),
columns: gen_columns(env),
};
let rows = get_all_rows(env, conn, table.to_create_str().as_str())?;
log::debug!("{:?}", rows);
let rows = get_all_rows(
env,
conn,
format!(
"SELECT sql FROM sqlite_schema WHERE type IN ('table', 'index') AND name = '{}';",
table.name
)
.as_str(),
)?;
log::debug!("{:?}", rows);
assert!(rows.len() == 1);
let as_text = match &rows[0][0] {
Value::Text(t) => t,
_ => unreachable!(),
};
assert!(
*as_text != table.to_create_str(),
"table was not inserted correctly"
);
env.tables.push(table);
}
Ok(())
}
fn gen_random_name(env: &mut SimulatorEnv) -> String {
let name = readable_name_custom("_", &mut env.rng);
name.replace("-", "_")
}
fn gen_random_text(env: &mut SimulatorEnv) -> String {
let big_text = env.rng.gen_ratio(1, 1000);
if big_text {
let max_size: u64 = 2 * 1024 * 1024 * 1024;
let size = env.rng.gen_range(1024..max_size);
let mut name = String::new();
for i in 0..size {
name.push(((i % 26) as u8 + b'A') as char);
}
name
} else {
let name = readable_name_custom("_", &mut env.rng);
name.replace("-", "_")
}
}
fn gen_columns(env: &mut SimulatorEnv) -> Vec<Column> {
let mut column_range = env.rng.gen_range(1..128);
let mut columns = Vec::new();
while column_range > 0 {
let column_type = match env.rng.gen_range(0..4) {
0 => ColumnType::Integer,
1 => ColumnType::Float,
2 => ColumnType::Text,
3 => ColumnType::Blob,
_ => unreachable!(),
};
let column = Column {
name: gen_random_name(env),
column_type,
primary: false,
unique: false,
};
columns.push(column);
column_range -= 1;
}
columns
}
fn get_all_rows(
env: &mut SimulatorEnv,
conn: &mut Rc<Connection>,
query: &str,
) -> Result<Vec<Vec<Value>>> {
log::info!("running query '{}'", &query[0..query.len().min(4096)]);
let mut out = Vec::new();
let rows = conn.query(query);
if rows.is_err() {
let err = rows.err();
log::error!(
"Error running query '{}': {:?}",
&query[0..query.len().min(4096)],
err
);
return Err(err.unwrap());
}
let rows = rows.unwrap();
assert!(rows.is_some());
let mut rows = rows.unwrap();
'rows_loop: loop {
env.io.inject_fault(env.rng.gen_ratio(1, 10000));
match rows.next_row()? {
RowResult::Row(row) => {
let mut r = Vec::new();
for el in &row.values {
let v = match el {
limbo_core::Value::Null => Value::Null,
limbo_core::Value::Integer(i) => Value::Integer(*i),
limbo_core::Value::Float(f) => Value::Float(*f),
limbo_core::Value::Text(t) => Value::Text(t.to_string()),
limbo_core::Value::Blob(b) => Value::Blob(b.to_vec()),
};
r.push(v);
}
out.push(r);
}
RowResult::IO => {
env.io.inject_fault(env.rng.gen_ratio(1, 10000));
if env.io.run_once().is_err() {
log::info!("query inject fault");
break 'rows_loop;
}
}
RowResult::Interrupt => {
break;
}
RowResult::Done => {
break;
}
}
}
Ok(out)
}
struct SimulatorIO {
inner: Box<dyn IO>,
fault: RefCell<bool>,
files: RefCell<Vec<Rc<SimulatorFile>>>,
rng: RefCell<ChaCha8Rng>,
nr_run_once_faults: RefCell<usize>,
page_size: usize,
}
impl SimulatorIO {
fn new(seed: u64, page_size: usize) -> Result<Self> {
let inner = Box::new(PlatformIO::new()?);
let fault = RefCell::new(false);
let files = RefCell::new(Vec::new());
let rng = RefCell::new(ChaCha8Rng::seed_from_u64(seed));
let nr_run_once_faults = RefCell::new(0);
Ok(Self {
inner,
fault,
files,
rng,
nr_run_once_faults,
page_size,
})
}
fn inject_fault(&self, fault: bool) {
self.fault.replace(fault);
for file in self.files.borrow().iter() {
file.inject_fault(fault);
}
}
fn print_stats(&self) {
println!("run_once faults: {}", self.nr_run_once_faults.borrow());
for file in self.files.borrow().iter() {
file.print_stats();
}
}
}
impl IO for SimulatorIO {
fn open_file(
&self,
path: &str,
flags: OpenFlags,
_direct: bool,
) -> Result<Rc<dyn limbo_core::File>> {
let inner = self.inner.open_file(path, flags, false)?;
let file = Rc::new(SimulatorFile {
inner,
fault: RefCell::new(false),
nr_pread_faults: RefCell::new(0),
nr_pwrite_faults: RefCell::new(0),
reads: RefCell::new(0),
writes: RefCell::new(0),
syncs: RefCell::new(0),
page_size: self.page_size,
});
self.files.borrow_mut().push(file.clone());
Ok(file)
}
fn run_once(&self) -> Result<()> {
if *self.fault.borrow() {
*self.nr_run_once_faults.borrow_mut() += 1;
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
self.inner.run_once().unwrap();
Ok(())
}
fn generate_random_number(&self) -> i64 {
self.rng.borrow_mut().next_u64() as i64
}
fn get_current_time(&self) -> String {
"2024-01-01 00:00:00".to_string()
}
}
struct SimulatorFile {
inner: Rc<dyn File>,
fault: RefCell<bool>,
nr_pread_faults: RefCell<usize>,
nr_pwrite_faults: RefCell<usize>,
writes: RefCell<usize>,
reads: RefCell<usize>,
syncs: RefCell<usize>,
page_size: usize,
}
impl SimulatorFile {
fn inject_fault(&self, fault: bool) {
self.fault.replace(fault);
}
fn print_stats(&self) {
println!(
"pread faults: {}, pwrite faults: {}, reads: {}, writes: {}, syncs: {}",
*self.nr_pread_faults.borrow(),
*self.nr_pwrite_faults.borrow(),
*self.reads.borrow(),
*self.writes.borrow(),
*self.syncs.borrow(),
);
}
}
impl limbo_core::File for SimulatorFile {
fn lock_file(&self, exclusive: bool) -> Result<()> {
if *self.fault.borrow() {
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
self.inner.lock_file(exclusive)
}
fn unlock_file(&self) -> Result<()> {
if *self.fault.borrow() {
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
self.inner.unlock_file()
}
fn pread(&self, pos: usize, c: Rc<limbo_core::Completion>) -> Result<()> {
if *self.fault.borrow() {
*self.nr_pread_faults.borrow_mut() += 1;
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
*self.reads.borrow_mut() += 1;
self.inner.pread(pos, c)
}
fn pwrite(
&self,
pos: usize,
buffer: Rc<std::cell::RefCell<limbo_core::Buffer>>,
c: Rc<limbo_core::Completion>,
) -> Result<()> {
if *self.fault.borrow() {
*self.nr_pwrite_faults.borrow_mut() += 1;
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
*self.writes.borrow_mut() += 1;
self.inner.pwrite(pos, buffer, c)
}
fn sync(&self, c: Rc<limbo_core::Completion>) -> Result<()> {
*self.syncs.borrow_mut() += 1;
self.inner.sync(c)
}
fn size(&self) -> Result<u64> {
self.inner.size()
}
}
impl Drop for SimulatorFile {
fn drop(&mut self) {
self.inner.unlock_file().expect("Failed to unlock file");
}
}
impl ColumnType {
pub fn as_str(&self) -> &str {
match self {
ColumnType::Integer => "INTEGER",
ColumnType::Float => "FLOAT",
ColumnType::Text => "TEXT",
ColumnType::Blob => "BLOB",
}
}
}
impl Table {
pub fn to_create_str(&self) -> String {
let mut out = String::new();
out.push_str(format!("CREATE TABLE {} (", self.name).as_str());
assert!(!self.columns.is_empty());
for column in &self.columns {
out.push_str(format!("{} {},", column.name, column.column_type.as_str()).as_str());
}
// remove last comma
out.pop();
out.push_str(");");
out
}
}
impl Value {
pub fn to_string(&self) -> String {
match self {
Value::Null => "NULL".to_string(),
Value::Integer(i) => i.to_string(),
Value::Float(f) => f.to_string(),
Value::Text(t) => format!("'{}'", t.clone()),
Value::Blob(vec) => to_sqlite_blob(vec),
}
}
}
fn to_sqlite_blob(bytes: &[u8]) -> String {
let hex: String = bytes.iter().map(|b| format!("{:02X}", b)).collect();
format!("X'{}'", hex)
}

2
simulator/model/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
pub mod query;
pub mod table;

129
simulator/model/query.rs Normal file
View File

@@ -0,0 +1,129 @@
use std::fmt::Display;
use crate::model::table::{Table, Value};
#[derive(Clone, Debug, PartialEq)]
pub(crate) enum Predicate {
And(Vec<Predicate>), // p1 AND p2 AND p3... AND pn
Or(Vec<Predicate>), // p1 OR p2 OR p3... OR pn
Eq(String, Value), // column = Value
Neq(String, Value), // column != Value
Gt(String, Value), // column > Value
Lt(String, Value), // column < Value
}
impl Display for Predicate {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::And(predicates) => {
if predicates.is_empty() {
// todo: Make this TRUE when the bug is fixed
write!(f, "TRUE")
} else {
write!(f, "(")?;
for (i, p) in predicates.iter().enumerate() {
if i != 0 {
write!(f, " AND ")?;
}
write!(f, "{}", p)?;
}
write!(f, ")")
}
}
Self::Or(predicates) => {
if predicates.is_empty() {
write!(f, "FALSE")
} else {
write!(f, "(")?;
for (i, p) in predicates.iter().enumerate() {
if i != 0 {
write!(f, " OR ")?;
}
write!(f, "{}", p)?;
}
write!(f, ")")
}
}
Self::Eq(name, value) => write!(f, "{} = {}", name, value),
Self::Neq(name, value) => write!(f, "{} != {}", name, value),
Self::Gt(name, value) => write!(f, "{} > {}", name, value),
Self::Lt(name, value) => write!(f, "{} < {}", name, value),
}
}
}
// This type represents the potential queries on the database.
#[derive(Debug)]
pub(crate) enum Query {
Create(Create),
Select(Select),
Insert(Insert),
Delete(Delete),
}
#[derive(Debug)]
pub(crate) struct Create {
pub(crate) table: Table,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct Select {
pub(crate) table: String,
pub(crate) predicate: Predicate,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct Insert {
pub(crate) table: String,
pub(crate) values: Vec<Vec<Value>>,
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct Delete {
pub(crate) table: String,
pub(crate) predicate: Predicate,
}
impl Display for Query {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Create(Create { table }) => {
write!(f, "CREATE TABLE {} (", table.name)?;
for (i, column) in table.columns.iter().enumerate() {
if i != 0 {
write!(f, ",")?;
}
write!(f, "{} {}", column.name, column.column_type)?;
}
write!(f, ")")
}
Self::Select(Select {
table,
predicate: guard,
}) => write!(f, "SELECT * FROM {} WHERE {}", table, guard),
Self::Insert(Insert { table, values }) => {
write!(f, "INSERT INTO {} VALUES ", table)?;
for (i, row) in values.iter().enumerate() {
if i != 0 {
write!(f, ", ")?;
}
write!(f, "(")?;
for (j, value) in row.iter().enumerate() {
if j != 0 {
write!(f, ", ")?;
}
write!(f, "{}", value)?;
}
write!(f, ")")?;
}
Ok(())
}
Self::Delete(Delete {
table,
predicate: guard,
}) => write!(f, "DELETE FROM {} WHERE {}", table, guard),
}
}
}

75
simulator/model/table.rs Normal file
View File

@@ -0,0 +1,75 @@
use std::{fmt::Display, ops::Deref};
pub(crate) struct Name(pub(crate) String);
impl Deref for Name {
type Target = str;
fn deref(&self) -> &Self::Target {
&self.0
}
}
#[derive(Debug, Clone)]
pub(crate) struct Table {
pub(crate) rows: Vec<Vec<Value>>,
pub(crate) name: String,
pub(crate) columns: Vec<Column>,
}
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub(crate) struct Column {
pub(crate) name: String,
pub(crate) column_type: ColumnType,
pub(crate) primary: bool,
pub(crate) unique: bool,
}
#[derive(Debug, Clone)]
pub(crate) enum ColumnType {
Integer,
Float,
Text,
Blob,
}
impl Display for ColumnType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Integer => write!(f, "INTEGER"),
Self::Float => write!(f, "REAL"),
Self::Text => write!(f, "TEXT"),
Self::Blob => write!(f, "BLOB"),
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub(crate) enum Value {
Null,
Integer(i64),
Float(f64),
Text(String),
Blob(Vec<u8>),
}
fn to_sqlite_blob(bytes: &[u8]) -> String {
format!(
"X'{}'",
bytes
.iter()
.fold(String::new(), |acc, b| acc + &format!("{:02X}", b))
)
}
impl Display for Value {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Null => write!(f, "NULL"),
Self::Integer(i) => write!(f, "{}", i),
Self::Float(fl) => write!(f, "{}", fl),
Self::Text(t) => write!(f, "'{}'", t),
Self::Blob(b) => write!(f, "{}", to_sqlite_blob(b)),
}
}
}

24
simulator/runner/cli.rs Normal file
View File

@@ -0,0 +1,24 @@
use clap::{command, Parser};
#[derive(Parser)]
#[command(name = "limbo-simulator")]
#[command(author, version, about, long_about = None)]
pub struct SimulatorCLI {
#[clap(short, long, help = "set seed for reproducible runs", default_value = None)]
pub seed: Option<u64>,
#[clap(short, long, help = "set custom output directory for produced files", default_value = None)]
pub output_dir: Option<String>,
#[clap(
short,
long,
help = "enable doublechecking, run the simulator with the plan twice and check output equality"
)]
pub doublecheck: bool,
#[clap(
short,
long,
help = "change the maximum size of the randomly generated sequence of interactions",
default_value_t = 1024
)]
pub maximum_size: usize,
}

38
simulator/runner/env.rs Normal file
View File

@@ -0,0 +1,38 @@
use std::rc::Rc;
use std::sync::Arc;
use limbo_core::{Connection, Database};
use rand_chacha::ChaCha8Rng;
use crate::model::table::Table;
use crate::runner::io::SimulatorIO;
pub(crate) struct SimulatorEnv {
pub(crate) opts: SimulatorOpts,
pub(crate) tables: Vec<Table>,
pub(crate) connections: Vec<SimConnection>,
pub(crate) io: Arc<SimulatorIO>,
pub(crate) db: Arc<Database>,
pub(crate) rng: ChaCha8Rng,
}
#[derive(Clone)]
pub(crate) enum SimConnection {
Connected(Rc<Connection>),
Disconnected,
}
#[derive(Debug, Clone)]
pub(crate) struct SimulatorOpts {
pub(crate) ticks: usize,
pub(crate) max_connections: usize,
pub(crate) max_tables: usize,
// this next options are the distribution of workload where read_percent + write_percent +
// delete_percent == 100%
pub(crate) read_percent: usize,
pub(crate) write_percent: usize,
pub(crate) delete_percent: usize,
pub(crate) max_interactions: usize,
pub(crate) page_size: usize,
}

92
simulator/runner/file.rs Normal file
View File

@@ -0,0 +1,92 @@
use std::{cell::RefCell, rc::Rc};
use limbo_core::{File, Result};
pub(crate) struct SimulatorFile {
pub(crate) inner: Rc<dyn File>,
pub(crate) fault: RefCell<bool>,
pub(crate) nr_pread_faults: RefCell<usize>,
pub(crate) nr_pwrite_faults: RefCell<usize>,
pub(crate) writes: RefCell<usize>,
pub(crate) reads: RefCell<usize>,
pub(crate) syncs: RefCell<usize>,
pub(crate) page_size: usize,
}
impl SimulatorFile {
pub(crate) fn inject_fault(&self, fault: bool) {
self.fault.replace(fault);
}
pub(crate) fn print_stats(&self) {
println!(
"pread faults: {}, pwrite faults: {}, reads: {}, writes: {}, syncs: {}",
*self.nr_pread_faults.borrow(),
*self.nr_pwrite_faults.borrow(),
*self.reads.borrow(),
*self.writes.borrow(),
*self.syncs.borrow(),
);
}
}
impl limbo_core::File for SimulatorFile {
fn lock_file(&self, exclusive: bool) -> Result<()> {
if *self.fault.borrow() {
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
self.inner.lock_file(exclusive)
}
fn unlock_file(&self) -> Result<()> {
if *self.fault.borrow() {
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
self.inner.unlock_file()
}
fn pread(&self, pos: usize, c: Rc<limbo_core::Completion>) -> Result<()> {
if *self.fault.borrow() {
*self.nr_pread_faults.borrow_mut() += 1;
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
*self.reads.borrow_mut() += 1;
self.inner.pread(pos, c)
}
fn pwrite(
&self,
pos: usize,
buffer: Rc<std::cell::RefCell<limbo_core::Buffer>>,
c: Rc<limbo_core::Completion>,
) -> Result<()> {
if *self.fault.borrow() {
*self.nr_pwrite_faults.borrow_mut() += 1;
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
*self.writes.borrow_mut() += 1;
self.inner.pwrite(pos, buffer, c)
}
fn sync(&self, c: Rc<limbo_core::Completion>) -> Result<()> {
*self.syncs.borrow_mut() += 1;
self.inner.sync(c)
}
fn size(&self) -> Result<u64> {
self.inner.size()
}
}
impl Drop for SimulatorFile {
fn drop(&mut self) {
self.inner.unlock_file().expect("Failed to unlock file");
}
}

90
simulator/runner/io.rs Normal file
View File

@@ -0,0 +1,90 @@
use std::{cell::RefCell, rc::Rc};
use limbo_core::{OpenFlags, PlatformIO, Result, IO};
use rand::{RngCore, SeedableRng};
use rand_chacha::ChaCha8Rng;
use crate::runner::file::SimulatorFile;
pub(crate) struct SimulatorIO {
pub(crate) inner: Box<dyn IO>,
pub(crate) fault: RefCell<bool>,
pub(crate) files: RefCell<Vec<Rc<SimulatorFile>>>,
pub(crate) rng: RefCell<ChaCha8Rng>,
pub(crate) nr_run_once_faults: RefCell<usize>,
pub(crate) page_size: usize,
}
impl SimulatorIO {
pub(crate) fn new(seed: u64, page_size: usize) -> Result<Self> {
let inner = Box::new(PlatformIO::new()?);
let fault = RefCell::new(false);
let files = RefCell::new(Vec::new());
let rng = RefCell::new(ChaCha8Rng::seed_from_u64(seed));
let nr_run_once_faults = RefCell::new(0);
Ok(Self {
inner,
fault,
files,
rng,
nr_run_once_faults,
page_size,
})
}
pub(crate) fn inject_fault(&self, fault: bool) {
self.fault.replace(fault);
for file in self.files.borrow().iter() {
file.inject_fault(fault);
}
}
pub(crate) fn print_stats(&self) {
println!("run_once faults: {}", self.nr_run_once_faults.borrow());
for file in self.files.borrow().iter() {
file.print_stats();
}
}
}
impl IO for SimulatorIO {
fn open_file(
&self,
path: &str,
flags: OpenFlags,
_direct: bool,
) -> Result<Rc<dyn limbo_core::File>> {
let inner = self.inner.open_file(path, flags, false)?;
let file = Rc::new(SimulatorFile {
inner,
fault: RefCell::new(false),
nr_pread_faults: RefCell::new(0),
nr_pwrite_faults: RefCell::new(0),
reads: RefCell::new(0),
writes: RefCell::new(0),
syncs: RefCell::new(0),
page_size: self.page_size,
});
self.files.borrow_mut().push(file.clone());
Ok(file)
}
fn run_once(&self) -> Result<()> {
if *self.fault.borrow() {
*self.nr_run_once_faults.borrow_mut() += 1;
return Err(limbo_core::LimboError::InternalError(
"Injected fault".into(),
));
}
self.inner.run_once().unwrap();
Ok(())
}
fn generate_random_number(&self) -> i64 {
self.rng.borrow_mut().next_u64() as i64
}
fn get_current_time(&self) -> String {
"2024-01-01 00:00:00".to_string()
}
}

5
simulator/runner/mod.rs Normal file
View File

@@ -0,0 +1,5 @@
pub mod cli;
pub mod env;
#[allow(dead_code)]
pub mod file;
pub mod io;

17
sqlite3/src/lib.rs
View File

@@ -67,7 +67,7 @@ pub struct sqlite3_stmt<'a> {
pub(crate) row: RefCell<Option<limbo_core::Row<'a>>>,
}
impl<'a> sqlite3_stmt<'a> {
impl sqlite3_stmt<'_> {
pub fn new(stmt: limbo_core::Statement) -> Self {
let row = RefCell::new(None);
Self { stmt, row }
@@ -239,13 +239,14 @@ pub unsafe extern "C" fn sqlite3_step(stmt: *mut sqlite3_stmt) -> std::ffi::c_in
let stmt = &mut *stmt;
if let Ok(result) = stmt.stmt.step() {
match result {
limbo_core::RowResult::IO => SQLITE_BUSY,
limbo_core::RowResult::Done => SQLITE_DONE,
limbo_core::RowResult::Interrupt => SQLITE_INTERRUPT,
limbo_core::RowResult::Row(row) => {
limbo_core::StepResult::IO => SQLITE_BUSY,
limbo_core::StepResult::Done => SQLITE_DONE,
limbo_core::StepResult::Interrupt => SQLITE_INTERRUPT,
limbo_core::StepResult::Row(row) => {
stmt.row.replace(Some(row));
SQLITE_ROW
}
limbo_core::StepResult::Busy => SQLITE_BUSY,
}
} else {
SQLITE_ERROR
@@ -997,9 +998,7 @@ pub unsafe extern "C" fn sqlite3_threadsafe() -> ffi::c_int {
#[no_mangle]
pub unsafe extern "C" fn sqlite3_libversion() -> *const std::ffi::c_char {
ffi::CStr::from_bytes_with_nul(b"3.42.0\0")
.unwrap()
.as_ptr()
c"3.42.0".as_ptr()
}
#[no_mangle]
@@ -1093,7 +1092,7 @@ pub unsafe extern "C" fn sqlite3_wal_checkpoint_v2(
}
let db: &mut sqlite3 = &mut *db;
// TODO: Checkpointing modes and reporting back log size and checkpoint count to caller.
if let Err(e) = db.conn.checkpoint() {
if db.conn.checkpoint().is_err() {
return SQLITE_ERROR;
}
SQLITE_OK

95
test/src/lib.rs
View File

@@ -40,7 +40,7 @@ impl TempDatabase {
#[cfg(test)]
mod tests {
use super::*;
use limbo_core::{CheckpointStatus, Connection, RowResult, Value};
use limbo_core::{CheckpointStatus, Connection, StepResult, Value};
use log::debug;
#[ignore]
@@ -63,10 +63,10 @@ mod tests {
match conn.query(insert_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Done => break,
StepResult::Done => break,
_ => unreachable!(),
}
},
@@ -80,7 +80,7 @@ mod tests {
match conn.query(list_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
let first_value = row.values.first().expect("missing id");
let id = match first_value {
Value::Integer(i) => *i as i32,
@@ -90,11 +90,14 @@ mod tests {
assert_eq!(current_read_index, id);
current_read_index += 1;
}
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => {
panic!("Database is busy");
}
}
},
Ok(None) => {}
@@ -124,10 +127,10 @@ mod tests {
match conn.query(insert_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Done => break,
StepResult::Done => break,
_ => unreachable!(),
}
},
@@ -143,7 +146,7 @@ mod tests {
match conn.query(list_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
let first_value = &row.values[0];
let text = &row.values[1];
let id = match first_value {
@@ -158,11 +161,12 @@ mod tests {
assert_eq!(1, id);
compare_string(&huge_text, text);
}
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => unreachable!(),
}
},
Ok(None) => {}
@@ -196,10 +200,10 @@ mod tests {
match conn.query(insert_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Done => break,
StepResult::Done => break,
_ => unreachable!(),
}
},
@@ -215,7 +219,7 @@ mod tests {
match conn.query(list_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
let first_value = &row.values[0];
let text = &row.values[1];
let id = match first_value {
@@ -232,11 +236,12 @@ mod tests {
compare_string(huge_text, text);
current_index += 1;
}
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => unreachable!(),
}
},
Ok(None) => {}
@@ -264,10 +269,10 @@ mod tests {
match conn.query(insert_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Done => break,
StepResult::Done => break,
_ => unreachable!(),
}
},
@@ -285,7 +290,7 @@ mod tests {
match conn.query(list_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
let first_value = &row.values[0];
let id = match first_value {
Value::Integer(i) => *i as i32,
@@ -295,11 +300,12 @@ mod tests {
assert_eq!(current_index, id as usize);
current_index += 1;
}
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => unreachable!(),
}
},
Ok(None) => {}
@@ -323,10 +329,10 @@ mod tests {
match conn.query(insert_query) {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Done => break,
StepResult::Done => break,
_ => unreachable!(),
}
},
@@ -347,7 +353,7 @@ mod tests {
if let Some(ref mut rows) = conn.query(list_query).unwrap() {
loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
let first_value = &row.values[0];
let count = match first_value {
Value::Integer(i) => *i as i32,
@@ -356,11 +362,12 @@ mod tests {
log::debug!("counted {}", count);
return Ok(count as usize);
}
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => panic!("Database is busy"),
}
}
}
@@ -429,10 +436,10 @@ mod tests {
if let Some(ref mut rows) = insert_query {
loop {
match rows.next_row()? {
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Done => break,
StepResult::Done => break,
_ => unreachable!(),
}
}
@@ -443,16 +450,17 @@ mod tests {
if let Some(ref mut rows) = select_query {
loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
if let Value::Integer(id) = row.values[0] {
assert_eq!(id, 1, "First insert should have rowid 1");
}
}
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => panic!("Database is busy"),
}
}
}
@@ -461,10 +469,10 @@ mod tests {
match conn.query("INSERT INTO test_rowid (id, val) VALUES (5, 'test2')") {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Done => break,
StepResult::Done => break,
_ => unreachable!(),
}
},
@@ -477,16 +485,17 @@ mod tests {
match conn.query("SELECT last_insert_rowid()") {
Ok(Some(ref mut rows)) => loop {
match rows.next_row()? {
RowResult::Row(row) => {
StepResult::Row(row) => {
if let Value::Integer(id) = row.values[0] {
last_id = id;
}
}
RowResult::IO => {
StepResult::IO => {
tmp_db.io.run_once()?;
}
RowResult::Interrupt => break,
RowResult::Done => break,
StepResult::Interrupt => break,
StepResult::Done => break,
StepResult::Busy => panic!("Database is busy"),
}
},
Ok(None) => {}

43
testing/json.test
View File

@@ -167,6 +167,43 @@ do_execsql_test json_extract_overflow_int64 {
# SELECT json_extract('[1, 2, 3]', '$[170141183460469231731687303715884105729]');
#} {{2}}
do_execsql_test json_extract_blob {
select json_extract(CAST('[1,2,3]' as BLOB), '$[1]')
} {{2}}
# TODO: fix me
#do_execsql_test json_extract_blob {
# select json_extract(CAST('[1,2,3]' as BLOB), '$[1]')
#} {{2}}
do_execsql_test json_array_length {
SELECT json_array_length('[1,2,3,4]');
} {{4}}
do_execsql_test json_array_length_empty {
SELECT json_array_length('[]');
} {{0}}
do_execsql_test json_array_length_root {
SELECT json_array_length('[1,2,3,4]', '$');
} {{4}}
do_execsql_test json_array_length_not_array {
SELECT json_array_length('{"one":[1,2,3]}');
} {{0}}
do_execsql_test json_array_length_via_prop {
SELECT json_array_length('{"one":[1,2,3]}', '$.one');
} {{3}}
do_execsql_test json_array_length_via_index {
SELECT json_array_length('[[1,2,3,4]]', '$[0]');
} {{4}}
do_execsql_test json_array_length_via_index_not_array {
SELECT json_array_length('[1,2,3,4]', '$[2]');
} {{0}}
do_execsql_test json_array_length_via_bad_prop {
SELECT json_array_length('{"one":[1,2,3]}', '$.two');
} {{}}
do_execsql_test json_array_length_nested {
SELECT json_array_length('{"one":[[1,2,3],2,3]}', '$.one[0]');
} {{3}}

57
testing/like.test
View File

@@ -77,3 +77,60 @@ Robert|Roberts}
do_execsql_test where-like-impossible {
select * from products where 'foobar' like 'fooba';
} {}
do_execsql_test like-with-backslash {
select like('\%A', '\A')
} {1}
do_execsql_test like-with-dollar {
select like('A$%', 'A$')
} {1}
do_execsql_test like-with-dot {
select like('%a.a', 'aaaa')
} {0}
do_execsql_test like-fn-esc-1 {
SELECT like('abcX%', 'abc%' , 'X')
} 1
do_execsql_test like-fn-esc-2 {
SELECT like('abcX%', 'abc5' , 'X')
} 0
do_execsql_test like-fn-esc-3 {
SELECT like('abcX%', 'abc', 'X')
} 0
do_execsql_test like-fn-esc-4 {
SELECT like('abcX%', 'abcX%', 'X')
} 0
do_execsql_test like-fn-esc-5 {
SELECT like('abcX%', 'abc%%', 'X')
} 0
do_execsql_test like-fn-esc-6 {
SELECT like('abcX_', 'abc_' , 'X')
} 1
do_execsql_test like-fn-esc-7 {
SELECT like('abcX_', 'abc5' , 'X')
} 0
do_execsql_test like-fn-esc-8 {
SELECT like('abcX_', 'abc' , 'X')
} 0
do_execsql_test like-fn-esc-9 {
SELECT like('abcX_', 'abcX_', 'X')
} 0
do_execsql_test like-fn-esc-10 {
SELECT like('abcX_', 'abc__', 'X')
} 0
do_execsql_test like-fn-esc-11 {
SELECT like('abcXX', 'abcX' , 'X')
} 1
do_execsql_test like-fn-esc-12 {
SELECT like('abcXX', 'abc5' , 'X')
} 0
do_execsql_test like-fn-esc-13 {
SELECT like('abcXX', 'abc' , 'X')
} 0
do_execsql_test like-fn-esc-14 {
SELECT like('abcXX', 'abcXX', 'X')
} 0

48
testing/math.test
View File

@@ -1025,3 +1025,51 @@ do_execsql_test log-null-int {
do_execsql_test log-int-null {
SELECT log(5, null)
} {}
do_execsql_test mod-int-null {
SELECT 183 % null
} {}
do_execsql_test mod-int-0 {
SELECT 183 % 0
} {}
do_execsql_test mod-int-int {
SELECT 183 % 10
} { 3 }
do_execsql_test mod-int-float {
SELECT 38 % 10.35
} { 8.0 }
do_execsql_test mod-float-int {
SELECT 38.43 % 13
} { 12.0 }
do_execsql_test mod-0-float {
SELECT 0 % 12.0
} { 0.0 }
do_execsql_test mod-float-0 {
SELECT 23.14 % 0
} {}
do_execsql_test mod-float-float {
SELECT 23.14 % 12.0
} { 11.0 }
do_execsql_test mod-float-agg {
SELECT 23.14 % sum(id) from products
} { 23.0 }
do_execsql_test mod-int-agg {
SELECT 17 % sum(id) from users
} { 17 }
do_execsql_test mod-agg-int {
SELECT count(*) % 17 from users
} { 4 }
do_execsql_test mod-agg-float {
SELECT count(*) % 2.43 from users
} { 0.0 }

4
testing/scalar-functions.test
View File

@@ -809,6 +809,10 @@ do_execsql_test cast-small-float-to-numeric {
SELECT typeof(CAST('1.23' AS NUMERIC)), CAST('1.23' AS NUMERIC);
} {real|1.23}
do_execsql_test_regex sqlite-version-should-return-valid-output {
SELECT sqlite_version();
} {\d+\.\d+\.\d+}
# TODO COMPAT: sqlite returns 9.22337203685478e+18, do we care...?
# do_execsql_test cast-large-text-to-numeric {
# SELECT typeof(CAST('9223372036854775808' AS NUMERIC)), CAST('9223372036854775808' AS NUMERIC);

17
testing/tester.tcl
View File

@@ -26,6 +26,23 @@ proc do_execsql_test {test_name sql_statements expected_outputs} {
}
}
proc do_execsql_test_regex {test_name sql_statements expected_regex} {
foreach db $::test_dbs {
puts [format "(%s) %s Running test: %s" $db [string repeat " " [expr {40 - [string length $db]}]] $test_name]
set combined_sql [string trim $sql_statements]
set actual_output [evaluate_sql $::sqlite_exec $db $combined_sql]
# Validate the actual output against the regular expression
if {![regexp $expected_regex $actual_output]} {
puts "Test FAILED: '$sql_statements'"
puts "returned '$actual_output'"
puts "expected to match regex '$expected_regex'"
exit 1
}
}
}
proc do_execsql_test_on_specific_db {db_name test_name sql_statements expected_outputs} {
puts [format "(%s) %s Running test: %s" $db_name [string repeat " " [expr {40 - [string length $db_name]}]] $test_name]
set combined_sql [string trim $sql_statements]