diff --git a/.github/workflows/antithesis.yml b/.github/workflows/antithesis.yml
index 60d5cff52..bcd767447 100644
--- a/.github/workflows/antithesis.yml
+++ b/.github/workflows/antithesis.yml
@@ -13,7 +13,7 @@ env:
ANTITHESIS_PASSWD: ${{ secrets.ANTITHESIS_PASSWD }}
ANTITHESIS_DOCKER_HOST: us-central1-docker.pkg.dev
ANTITHESIS_DOCKER_REPO: ${{ secrets.ANTITHESIS_DOCKER_REPO }}
- ANTITHESIS_EMAIL: "penberg@turso.tech;pmuniz@turso.tech;pere@turso.tech"
+ ANTITHESIS_EMAIL: ${{ secrets.ANTITHESIS_EMAIL }}
ANTITHESIS_REGISTRY_KEY: ${{ secrets.ANTITHESIS_REGISTRY_KEY }}
jobs:
diff --git a/.github/workflows/long_fuzz_tests_btree.yml b/.github/workflows/long_fuzz_tests_btree.yml
index 49e5a252e..0f38f67bf 100644
--- a/.github/workflows/long_fuzz_tests_btree.yml
+++ b/.github/workflows/long_fuzz_tests_btree.yml
@@ -28,10 +28,6 @@ jobs:
run: cargo test -- --ignored fuzz_long
env:
RUST_BACKTRACE: 1
- - name: Run ignored long tests with index
- run: cargo test -- --ignored fuzz_long
- env:
- RUST_BACKTRACE: 1
simple-stress-test:
runs-on: blacksmith-4vcpu-ubuntu-2404
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 65e19f332..fa7cd6b0d 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -73,19 +73,12 @@ jobs:
with:
prefix-key: "v1-rust" # can be updated if we need to reset caches due to non-trivial change in the dependencies (for example, custom env var were set for single workspace project)
- name: Install the project
- run: ./scripts/run-sim --iterations 50
+ run: ./scripts/run-sim --maximum-tests 2000 loop -n 50 -s
test-limbo:
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 20
steps:
- - name: Install cargo-c
- env:
- LINK: https://github.com/lu-zero/cargo-c/releases/download/v0.10.7
- CARGO_C_FILE: cargo-c-x86_64-unknown-linux-musl.tar.gz
- run: |
- curl -L $LINK/$CARGO_C_FILE | tar xz -C ~/.cargo/bin
-
- uses: actions/checkout@v3
- name: Install uv
@@ -96,9 +89,6 @@ jobs:
- name: Set up Python
run: uv python install
- - name: Install the project
- run: uv sync --all-extras --dev --all-packages
-
- uses: "./.github/shared/install_sqlite"
- name: Test
run: make test
diff --git a/Cargo.lock b/Cargo.lock
index 2e8b0bdfc..d8024a08f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -571,7 +571,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "core_tester"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"anyhow",
"assert_cmd",
@@ -1870,14 +1870,14 @@ dependencies = [
[[package]]
name = "limbo-go"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"turso_core",
]
[[package]]
name = "limbo-wasm"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"console_error_panic_hook",
"getrandom 0.2.15",
@@ -1890,7 +1890,7 @@ dependencies = [
[[package]]
name = "limbo_completion"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"mimalloc",
"turso_ext",
@@ -1898,7 +1898,7 @@ dependencies = [
[[package]]
name = "limbo_crypto"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"blake3",
"data-encoding",
@@ -1911,7 +1911,7 @@ dependencies = [
[[package]]
name = "limbo_csv"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"csv",
"mimalloc",
@@ -1921,7 +1921,7 @@ dependencies = [
[[package]]
name = "limbo_ipaddr"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"ipnetwork",
"mimalloc",
@@ -1930,7 +1930,7 @@ dependencies = [
[[package]]
name = "limbo_percentile"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"mimalloc",
"turso_ext",
@@ -1938,7 +1938,7 @@ dependencies = [
[[package]]
name = "limbo_regexp"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"mimalloc",
"regex",
@@ -1947,7 +1947,7 @@ dependencies = [
[[package]]
name = "limbo_sim"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"anarchist-readable-name-generator-lib",
"anyhow",
@@ -1973,7 +1973,7 @@ dependencies = [
[[package]]
name = "limbo_sqlite3"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"env_logger 0.11.7",
"libc",
@@ -1986,7 +1986,7 @@ dependencies = [
[[package]]
name = "limbo_sqlite_test_ext"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"cc",
]
@@ -2222,6 +2222,8 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
+ "regex",
+ "semver",
"syn 2.0.100",
]
@@ -2464,45 +2466,6 @@ version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
-[[package]]
-name = "phf"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
-dependencies = [
- "phf_shared",
-]
-
-[[package]]
-name = "phf_codegen"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
-dependencies = [
- "phf_generator",
- "phf_shared",
-]
-
-[[package]]
-name = "phf_generator"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
-dependencies = [
- "phf_shared",
- "rand 0.8.5",
-]
-
-[[package]]
-name = "phf_shared"
-version = "0.11.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
-dependencies = [
- "siphasher",
- "uncased",
-]
-
[[package]]
name = "pin-project-lite"
version = "0.2.16"
@@ -2690,7 +2653,7 @@ dependencies = [
[[package]]
name = "py-turso"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"anyhow",
"pyo3",
@@ -3285,12 +3248,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "siphasher"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
-
[[package]]
name = "slab"
version = "0.4.9"
@@ -3302,9 +3259,9 @@ dependencies = [
[[package]]
name = "smallvec"
-version = "1.14.0"
+version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
+checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "socket2"
@@ -3801,7 +3758,7 @@ dependencies = [
[[package]]
name = "turso"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"tempfile",
"thiserror 2.0.12",
@@ -3811,7 +3768,7 @@ dependencies = [
[[package]]
name = "turso-java"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"jni",
"thiserror 2.0.12",
@@ -3820,7 +3777,7 @@ dependencies = [
[[package]]
name = "turso_cli"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"anyhow",
"cfg-if",
@@ -3851,7 +3808,7 @@ dependencies = [
[[package]]
name = "turso_core"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"antithesis_sdk",
"bitflags 2.9.0",
@@ -3904,7 +3861,7 @@ dependencies = [
[[package]]
name = "turso_dart"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"flutter_rust_bridge",
"turso_core",
@@ -3912,7 +3869,7 @@ dependencies = [
[[package]]
name = "turso_ext"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"chrono",
"getrandom 0.3.2",
@@ -3921,7 +3878,7 @@ dependencies = [
[[package]]
name = "turso_ext_tests"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"env_logger 0.11.7",
"lazy_static",
@@ -3932,7 +3889,7 @@ dependencies = [
[[package]]
name = "turso_macros"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"proc-macro2",
"quote",
@@ -3941,7 +3898,7 @@ dependencies = [
[[package]]
name = "turso_node"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"napi",
"napi-build",
@@ -3951,7 +3908,7 @@ dependencies = [
[[package]]
name = "turso_sqlite3_parser"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"bitflags 2.9.0",
"cc",
@@ -3961,18 +3918,15 @@ dependencies = [
"log",
"memchr",
"miette",
- "phf",
- "phf_codegen",
- "phf_shared",
"serde",
+ "smallvec",
"strum",
"strum_macros",
- "uncased",
]
[[package]]
name = "turso_stress"
-version = "0.1.1"
+version = "0.1.2-pre.2"
dependencies = [
"anarchist-readable-name-generator-lib",
"antithesis_sdk",
diff --git a/Cargo.toml b/Cargo.toml
index 00fb19f0c..cca1e3091 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -31,25 +31,25 @@ members = [
exclude = ["perf/latency/limbo"]
[workspace.package]
-version = "0.1.1"
+version = "0.1.2-pre.2"
authors = ["the Limbo authors"]
edition = "2021"
license = "MIT"
repository = "https://github.com/tursodatabase/turso"
[workspace.dependencies]
-limbo_completion = { path = "extensions/completion", version = "0.1.1" }
-turso_core = { path = "core", version = "0.1.1" }
-limbo_crypto = { path = "extensions/crypto", version = "0.1.1" }
-limbo_csv = { path = "extensions/csv", version = "0.1.1" }
-turso_ext = { path = "extensions/core", version = "0.1.1" }
-turso_ext_tests = { path = "extensions/tests", version = "0.1.1" }
-limbo_ipaddr = { path = "extensions/ipaddr", version = "0.1.1" }
-turso_macros = { path = "macros", version = "0.1.1" }
-limbo_percentile = { path = "extensions/percentile", version = "0.1.1" }
-limbo_regexp = { path = "extensions/regexp", version = "0.1.1" }
-turso_sqlite3_parser = { path = "vendored/sqlite3-parser", version = "0.1.1" }
-limbo_uuid = { path = "extensions/uuid", version = "0.1.1" }
+limbo_completion = { path = "extensions/completion", version = "0.1.2-pre.2" }
+turso_core = { path = "core", version = "0.1.2-pre.2" }
+limbo_crypto = { path = "extensions/crypto", version = "0.1.2-pre.2" }
+limbo_csv = { path = "extensions/csv", version = "0.1.2-pre.2" }
+turso_ext = { path = "extensions/core", version = "0.1.2-pre.2" }
+turso_ext_tests = { path = "extensions/tests", version = "0.1.2-pre.2" }
+limbo_ipaddr = { path = "extensions/ipaddr", version = "0.1.2-pre.2" }
+turso_macros = { path = "macros", version = "0.1.2-pre.2" }
+limbo_percentile = { path = "extensions/percentile", version = "0.1.2-pre.2" }
+limbo_regexp = { path = "extensions/regexp", version = "0.1.2-pre.2" }
+turso_sqlite3_parser = { path = "vendored/sqlite3-parser", version = "0.1.2-pre.2" }
+limbo_uuid = { path = "extensions/uuid", version = "0.1.2-pre.2" }
strum = { version = "0.26", features = ["derive"] }
strum_macros = "0.26"
serde = "1.0"
diff --git a/Makefile b/Makefile
index 51dbdc052..499007884 100644
--- a/Makefile
+++ b/Makefile
@@ -3,9 +3,11 @@ CURRENT_RUST_VERSION := $(shell rustc -V | sed -E 's/rustc ([0-9]+\.[0-9]+\.[0-9
CURRENT_RUST_TARGET := $(shell rustc -vV | grep host | cut -d ' ' -f 2)
RUSTUP := $(shell command -v rustup 2> /dev/null)
UNAME_S := $(shell uname -s)
+MINIMUM_TCL_VERSION := 8.6
# Executable used to execute the compatibility tests.
SQLITE_EXEC ?= scripts/limbo-sqlite3
+RUST_LOG := off
all: check-rust-version check-wasm-target limbo limbo-wasm
.PHONY: all
@@ -26,6 +28,17 @@ check-rust-version:
fi
.PHONY: check-rust-version
+check-tcl-version:
+ @printf '%s\n' \
+ 'set need "$(MINIMUM_TCL_VERSION)"' \
+ 'set have [info patchlevel]' \
+ 'if {[package vcompare $$have $$need] < 0} {' \
+ ' puts stderr "tclsh $$have found — need $$need+"' \
+ ' exit 1' \
+ '}' \
+ | tclsh
+.PHONY: check-tcl-version
+
check-wasm-target:
@echo "Checking wasm32-wasi target..."
@if ! rustup target list | grep -q "wasm32-wasi (installed)"; then \
@@ -51,27 +64,31 @@ uv-sync:
uv sync --all-packages
.PHONE: uv-sync
-test: limbo uv-sync test-compat test-vector test-sqlite3 test-shell test-extensions test-memory test-write test-update test-constraint test-collate
+uv-sync-test:
+ uv sync --all-extras --dev --package turso_test
+.PHONE: uv-sync
+
+test: limbo uv-sync-test test-compat test-vector test-sqlite3 test-shell test-memory test-write test-update test-constraint test-collate test-extensions
.PHONY: test
-test-extensions: limbo uv-sync
- uv run --project limbo_test test-extensions
+test-extensions: limbo uv-sync-test
+ RUST_LOG=$(RUST_LOG) uv run --project limbo_test test-extensions
.PHONY: test-extensions
-test-shell: limbo uv-sync
- SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-shell
+test-shell: limbo uv-sync-test
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-shell
.PHONY: test-shell
-test-compat:
- SQLITE_EXEC=$(SQLITE_EXEC) ./testing/all.test
+test-compat: check-tcl-version
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) ./testing/all.test
.PHONY: test-compat
test-vector:
- SQLITE_EXEC=$(SQLITE_EXEC) ./testing/vector.test
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) ./testing/vector.test
.PHONY: test-vector
test-time:
- SQLITE_EXEC=$(SQLITE_EXEC) ./testing/time.test
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) ./testing/time.test
.PHONY: test-time
reset-db:
@@ -85,48 +102,48 @@ test-sqlite3: reset-db
.PHONY: test-sqlite3
test-json:
- SQLITE_EXEC=$(SQLITE_EXEC) ./testing/json.test
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) ./testing/json.test
.PHONY: test-json
-test-memory: limbo uv-sync
- SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-memory
+test-memory: limbo uv-sync-test
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-memory
.PHONY: test-memory
-test-write: limbo uv-sync
+test-write: limbo uv-sync-test
@if [ "$(SQLITE_EXEC)" != "scripts/limbo-sqlite3" ]; then \
- SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-write; \
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-write; \
else \
echo "Skipping test-write: SQLITE_EXEC does not have indexes scripts/limbo-sqlite3"; \
fi
.PHONY: test-write
-test-update: limbo uv-sync
+test-update: limbo uv-sync-test
@if [ "$(SQLITE_EXEC)" != "scripts/limbo-sqlite3" ]; then \
- SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-update; \
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-update; \
else \
echo "Skipping test-update: SQLITE_EXEC does not have indexes scripts/limbo-sqlite3"; \
fi
.PHONY: test-update
-test-collate: limbo uv-sync
+test-collate: limbo uv-sync-test
@if [ "$(SQLITE_EXEC)" != "scripts/limbo-sqlite3" ]; then \
- SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-collate; \
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-collate; \
else \
echo "Skipping test-collate: SQLITE_EXEC does not have indexes scripts/limbo-sqlite3"; \
fi
.PHONY: test-collate
-test-constraint: limbo uv-sync
+test-constraint: limbo uv-sync-test
@if [ "$(SQLITE_EXEC)" != "scripts/limbo-sqlite3" ]; then \
- SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-constraint; \
+ RUST_LOG=$(RUST_LOG) SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-constraint; \
else \
echo "Skipping test-constraint: SQLITE_EXEC does not have indexes scripts/limbo-sqlite3"; \
fi
.PHONY: test-constraint
-bench-vfs: uv-sync
+bench-vfs: uv-sync-test
cargo build --release
- uv run --project limbo_test bench-vfs "$(SQL)" "$(N)"
+ RUST_LOG=$(RUST_LOG) uv run --project limbo_test bench-vfs "$(SQL)" "$(N)"
clickbench:
./perf/clickbench/benchmark.sh
diff --git a/README.md b/README.md
index eba882bc1..98a13f674 100644
--- a/README.md
+++ b/README.md
@@ -224,9 +224,11 @@ terms or conditions.
Thanks to all the partners of Turso!
+
+
-
+
## Contributors
diff --git a/antithesis-tests/bank-test/first_setup.py b/antithesis-tests/bank-test/first_setup.py
index fcbc33530..64ad06457 100755
--- a/antithesis-tests/bank-test/first_setup.py
+++ b/antithesis-tests/bank-test/first_setup.py
@@ -50,3 +50,5 @@ cur.execute(f"""
INSERT INTO initial_state (num_accts, total)
VALUES ({num_accts}, {total})
""")
+
+con.commit()
diff --git a/antithesis-tests/stress-composer/first_setup.py b/antithesis-tests/stress-composer/first_setup.py
index 9d755a071..45b37466f 100755
--- a/antithesis-tests/stress-composer/first_setup.py
+++ b/antithesis-tests/stress-composer/first_setup.py
@@ -83,4 +83,6 @@ for i in range(tbl_count):
CREATE TABLE tbl_{i} ({cols_str})
""")
+con.commit()
+
print(f"DB Schemas\n------------\n{json.dumps(schemas, indent=2)}")
diff --git a/antithesis-tests/stress-composer/parallel_driver_delete.py b/antithesis-tests/stress-composer/parallel_driver_delete.py
index 4ec62079b..d2e719fec 100755
--- a/antithesis-tests/stress-composer/parallel_driver_delete.py
+++ b/antithesis-tests/stress-composer/parallel_driver_delete.py
@@ -37,6 +37,13 @@ print(f"Attempt to delete {deletions} rows in tbl_{selected_tbl}...")
for i in range(deletions):
where_clause = f"col_{pk} = {generate_random_value(tbl_schema[f'col_{pk}']['data_type'])}"
- cur.execute(f"""
- DELETE FROM tbl_{selected_tbl} WHERE {where_clause}
- """)
+ try:
+ cur.execute(f"""
+ DELETE FROM tbl_{selected_tbl} WHERE {where_clause}
+ """)
+ except turso.OperationalError:
+ con.rollback()
+ # Re-raise other operational errors
+ raise
+
+con.commit()
diff --git a/antithesis-tests/stress-composer/parallel_driver_insert.py b/antithesis-tests/stress-composer/parallel_driver_insert.py
index 8e4f73e1f..bb5a02170 100755
--- a/antithesis-tests/stress-composer/parallel_driver_insert.py
+++ b/antithesis-tests/stress-composer/parallel_driver_insert.py
@@ -44,5 +44,8 @@ for i in range(insertions):
# Ignore UNIQUE constraint violations
pass
else:
+ con.rollback()
# Re-raise other operational errors
raise
+
+con.commit()
diff --git a/antithesis-tests/stress-composer/parallel_driver_schema_rollback.py b/antithesis-tests/stress-composer/parallel_driver_schema_rollback.py
index d101fcfc5..594925797 100755
--- a/antithesis-tests/stress-composer/parallel_driver_schema_rollback.py
+++ b/antithesis-tests/stress-composer/parallel_driver_schema_rollback.py
@@ -17,8 +17,7 @@ cur_init = con_init.cursor()
tbl_len = cur_init.execute("SELECT count FROM tables").fetchone()[0]
selected_tbl = get_random() % tbl_len
-tbl_schema = json.loads(cur_init.execute(
- f"SELECT schema FROM schemas WHERE tbl = {selected_tbl}").fetchone()[0])
+tbl_schema = json.loads(cur_init.execute(f"SELECT schema FROM schemas WHERE tbl = {selected_tbl}").fetchone()[0])
tbl_name = f"tbl_{selected_tbl}"
@@ -29,8 +28,7 @@ except Exception as e:
exit(0)
cur = con.cursor()
-cur.execute(
- "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = '" + tbl_name + "'")
+cur.execute("SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = '" + tbl_name + "'")
result = cur.fetchone()
@@ -47,10 +45,8 @@ cur.execute("ALTER TABLE " + tbl_name + " RENAME TO " + tbl_name + "_old")
con.rollback()
cur = con.cursor()
-cur.execute(
- "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = '" + tbl_name + "'")
+cur.execute("SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = '" + tbl_name + "'")
schema_after = cur.fetchone()[0]
-always(schema_before == schema_after,
- "schema should be the same after rollback", {})
+always(schema_before == schema_after, "schema should be the same after rollback", {})
diff --git a/antithesis-tests/stress-composer/parallel_driver_update.py b/antithesis-tests/stress-composer/parallel_driver_update.py
index e30d53acd..101508cc2 100755
--- a/antithesis-tests/stress-composer/parallel_driver_update.py
+++ b/antithesis-tests/stress-composer/parallel_driver_update.py
@@ -58,5 +58,8 @@ for i in range(updates):
# Ignore UNIQUE constraint violations
pass
else:
+ con.rollback()
# Re-raise other operational errors
raise
+
+con.commit()
diff --git a/assets/turso-nyrkio.png b/assets/turso-nyrkio.png
new file mode 100644
index 000000000..e65fcd7f5
Binary files /dev/null and b/assets/turso-nyrkio.png differ
diff --git a/bindings/go/rs_src/rows.rs b/bindings/go/rs_src/rows.rs
index 0e7e1bfbc..98739e83a 100644
--- a/bindings/go/rs_src/rows.rs
+++ b/bindings/go/rs_src/rows.rs
@@ -7,7 +7,7 @@ use turso_core::{LimboError, Statement, StepResult, Value};
pub struct LimboRows<'conn> {
stmt: Box,
- conn: &'conn mut LimboConn,
+ _conn: &'conn mut LimboConn,
err: Option,
}
@@ -15,7 +15,7 @@ impl<'conn> LimboRows<'conn> {
pub fn new(stmt: Statement, conn: &'conn mut LimboConn) -> Self {
LimboRows {
stmt: Box::new(stmt),
- conn,
+ _conn: conn,
err: None,
}
}
@@ -55,8 +55,12 @@ pub extern "C" fn rows_next(ctx: *mut c_void) -> ResultCode {
Ok(StepResult::Row) => ResultCode::Row,
Ok(StepResult::Done) => ResultCode::Done,
Ok(StepResult::IO) => {
- let _ = ctx.conn.io.run_once();
- ResultCode::Io
+ let res = ctx.stmt.run_once();
+ if res.is_err() {
+ ResultCode::Error
+ } else {
+ ResultCode::Io
+ }
}
Ok(StepResult::Busy) => ResultCode::Busy,
Ok(StepResult::Interrupt) => ResultCode::Interrupt,
diff --git a/bindings/go/rs_src/statement.rs b/bindings/go/rs_src/statement.rs
index 970ecd7cf..e1b5ae26b 100644
--- a/bindings/go/rs_src/statement.rs
+++ b/bindings/go/rs_src/statement.rs
@@ -64,7 +64,10 @@ pub extern "C" fn stmt_execute(
return ResultCode::Done;
}
Ok(StepResult::IO) => {
- let _ = stmt.conn.io.run_once();
+ let res = statement.run_once();
+ if res.is_err() {
+ return ResultCode::Error;
+ }
}
Ok(StepResult::Busy) => {
return ResultCode::Busy;
diff --git a/bindings/java/rs_src/turso_connection.rs b/bindings/java/rs_src/turso_connection.rs
index 1d2ae9f10..8a55bf169 100644
--- a/bindings/java/rs_src/turso_connection.rs
+++ b/bindings/java/rs_src/turso_connection.rs
@@ -13,12 +13,12 @@ use turso_core::Connection;
#[derive(Clone)]
pub struct TursoConnection {
pub(crate) conn: Arc,
- pub(crate) io: Arc,
+ pub(crate) _io: Arc,
}
impl TursoConnection {
pub fn new(conn: Arc, io: Arc) -> Self {
- TursoConnection { conn, io }
+ TursoConnection { conn, _io: io }
}
#[allow(clippy::wrong_self_convention)]
diff --git a/bindings/java/rs_src/turso_statement.rs b/bindings/java/rs_src/turso_statement.rs
index 17eaa5a5b..444d34707 100644
--- a/bindings/java/rs_src/turso_statement.rs
+++ b/bindings/java/rs_src/turso_statement.rs
@@ -76,7 +76,7 @@ pub extern "system" fn Java_tech_turso_core_TursoStatement_step<'local>(
};
}
StepResult::IO => {
- if let Err(e) = stmt.connection.io.run_once() {
+ if let Err(e) = stmt.stmt.run_once() {
set_err_msg_and_throw_exception(&mut env, obj, TURSO_ETC, e.to_string());
return to_turso_step_result(&mut env, STEP_RESULT_ID_ERROR, None);
}
diff --git a/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4Connection.java
index 88c76dd85..6841a5cbc 100644
--- a/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4Connection.java
+++ b/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4Connection.java
@@ -24,7 +24,9 @@ public final class JDBC4Connection implements Connection {
}
public TursoStatement prepare(String sql) throws SQLException {
- return connection.prepare(sql);
+ final TursoStatement statement = connection.prepare(sql);
+ statement.initializeColumnMetadata();
+ return statement;
}
@Override
diff --git a/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4PreparedStatement.java b/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4PreparedStatement.java
index e947aa272..a3f8b3d4d 100644
--- a/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4PreparedStatement.java
+++ b/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4PreparedStatement.java
@@ -34,7 +34,6 @@ public final class JDBC4PreparedStatement extends JDBC4Statement implements Prep
super(connection);
this.sql = sql;
this.statement = connection.prepare(sql);
- this.statement.initializeColumnMetadata();
this.resultSet = new JDBC4ResultSet(this.statement.getResultSet());
}
diff --git a/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4ResultSet.java b/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4ResultSet.java
index 23421bc51..85dee794d 100644
--- a/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4ResultSet.java
+++ b/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4ResultSet.java
@@ -319,10 +319,8 @@ public final class JDBC4ResultSet implements ResultSet, ResultSetMetaData {
}
@Override
- @SkipNullableCheck
public Object getObject(int columnIndex) throws SQLException {
- // TODO
- return null;
+ return resultSet.get(columnIndex);
}
@Override
@@ -1226,20 +1224,22 @@ public final class JDBC4ResultSet implements ResultSet, ResultSetMetaData {
@Override
public int getColumnDisplaySize(int column) throws SQLException {
- // TODO
- return 0;
+ return Integer.MAX_VALUE;
}
@Override
public String getColumnLabel(int column) throws SQLException {
- // TODO
- return "";
+ // TODO: should consider "AS" keyword
+ return getColumnName(column);
}
@Override
public String getColumnName(int column) throws SQLException {
- // TODO
- return "";
+ if (column > 0 && column <= resultSet.getColumnNames().length) {
+ return resultSet.getColumnNames()[column - 1];
+ }
+
+ throw new SQLException("Index out of bound: " + column);
}
@Override
diff --git a/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4Statement.java b/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4Statement.java
index 02831dbdd..b86b838f5 100644
--- a/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4Statement.java
+++ b/bindings/java/src/main/java/tech/turso/jdbc4/JDBC4Statement.java
@@ -18,6 +18,7 @@ public class JDBC4Statement implements Statement {
private final JDBC4Connection connection;
@Nullable protected TursoStatement statement = null;
+ protected long updateCount;
// Because JDBC4Statement has different life cycle in compared to tursoStatement, let's use this
// field to manage JDBC4Statement lifecycle
@@ -173,8 +174,10 @@ public class JDBC4Statement implements Statement {
// TODO: if sql is a readOnly query, do we still need the locks?
connectionLock.lock();
statement = connection.prepare(sql);
+ final long previousChanges = statement.totalChanges();
final boolean result = statement.execute();
updateGeneratedKeys();
+ updateCount = statement.totalChanges() - previousChanges;
return result;
} finally {
@@ -186,19 +189,13 @@ public class JDBC4Statement implements Statement {
@Override
public ResultSet getResultSet() throws SQLException {
requireNonNull(statement, "statement is null");
+ ensureOpen();
return new JDBC4ResultSet(statement.getResultSet());
}
@Override
public int getUpdateCount() throws SQLException {
- // TODO
- return 0;
- }
-
- @Override
- public boolean getMoreResults() throws SQLException {
- // TODO
- return false;
+ return (int) updateCount;
}
@Override
@@ -254,9 +251,22 @@ public class JDBC4Statement implements Statement {
return connection;
}
+ @Override
+ public boolean getMoreResults() throws SQLException {
+ return getMoreResults(Statement.CLOSE_CURRENT_RESULT);
+ }
+
@Override
public boolean getMoreResults(int current) throws SQLException {
- // TODO
+ requireNonNull(statement, "statement should not be null");
+
+ if (current != Statement.CLOSE_CURRENT_RESULT) {
+ throw new SQLException("Invalid argument");
+ }
+
+ statement.getResultSet().close();
+ updateCount = -1;
+
return false;
}
diff --git a/bindings/javascript/Cargo.toml b/bindings/javascript/Cargo.toml
index f02a8abe9..7d62ede4c 100644
--- a/bindings/javascript/Cargo.toml
+++ b/bindings/javascript/Cargo.toml
@@ -13,7 +13,7 @@ crate-type = ["cdylib"]
[dependencies]
turso_core = { workspace = true }
napi = { version = "2.16.17", default-features = false, features = ["napi4"] }
-napi-derive = { version = "2.16.13", default-features = false }
+napi-derive = { version = "2.16.13", default-features = true }
[build-dependencies]
napi-build = "2.2.0"
diff --git a/bindings/javascript/__test__/better-sqlite3.spec.mjs b/bindings/javascript/__test__/better-sqlite3.spec.mjs
index a5fdf93fe..992cda585 100644
--- a/bindings/javascript/__test__/better-sqlite3.spec.mjs
+++ b/bindings/javascript/__test__/better-sqlite3.spec.mjs
@@ -32,7 +32,7 @@ const genDatabaseFilename = () => {
return `test-${crypto.randomBytes(8).toString('hex')}.db`;
};
-new DualTest().onlySqlitePasses("opening a read-only database fails if the file doesn't exist", async (t) => {
+new DualTest().both("opening a read-only database fails if the file doesn't exist", async (t) => {
t.throws(() => t.context.connect(genDatabaseFilename(), { readonly: true }),
{
any: true,
@@ -104,7 +104,21 @@ inMemoryTest.both("Empty prepared statement should throw", async (t) => {
() => {
db.prepare("");
},
- { instanceOf: Error },
+ { any: true }
+ );
+});
+
+inMemoryTest.onlySqlitePasses("Empty prepared statement should throw the correct error", async (t) => {
+ // the previous test can be removed once this one passes in Turso
+ const db = t.context.db;
+ t.throws(
+ () => {
+ db.prepare("");
+ },
+ {
+ instanceOf: RangeError,
+ message: "The supplied SQL string contains no statements",
+ },
);
});
@@ -156,9 +170,12 @@ inMemoryTest.both("Statement shouldn't bind twice with bind()", async (t) => {
t.throws(
() => {
- db.bind("Bob");
+ stmt.bind("Bob");
+ },
+ {
+ instanceOf: TypeError,
+ message: 'The bind() method can only be invoked once per statement object',
},
- { instanceOf: Error },
);
});
@@ -372,3 +389,4 @@ inMemoryTest.both("Test Statement.source", async t => {
t.is(stmt.source, sql);
});
+
diff --git a/bindings/javascript/__test__/sync.spec.mjs b/bindings/javascript/__test__/sync.spec.mjs
index f8c016021..87fe1d7d4 100644
--- a/bindings/javascript/__test__/sync.spec.mjs
+++ b/bindings/javascript/__test__/sync.spec.mjs
@@ -377,7 +377,7 @@ dualTest.both("Database.pragma()", async (t) => {
t.deepEqual(db.pragma("cache_size"), [{ "cache_size": 2000 }]);
});
-dualTest.onlySqlitePasses("errors", async (t) => {
+dualTest.both("errors", async (t) => {
const db = t.context.db;
const syntaxError = await t.throws(() => {
@@ -385,7 +385,7 @@ dualTest.onlySqlitePasses("errors", async (t) => {
}, {
any: true,
instanceOf: t.context.errorType,
- message: 'near "SYNTAX": syntax error',
+ message: /near "SYNTAX": syntax error/,
code: 'SQLITE_ERROR'
});
const noTableError = await t.throws(() => {
@@ -393,7 +393,7 @@ dualTest.onlySqlitePasses("errors", async (t) => {
}, {
any: true,
instanceOf: t.context.errorType,
- message: "no such table: missing_table",
+ message: /(Parse error: Table missing_table not found|no such table: missing_table)/,
code: 'SQLITE_ERROR'
});
diff --git a/bindings/javascript/index.d.ts b/bindings/javascript/index.d.ts
index 99433b962..37041f67a 100644
--- a/bindings/javascript/index.d.ts
+++ b/bindings/javascript/index.d.ts
@@ -3,41 +3,41 @@
/* auto-generated by NAPI-RS */
-export interface Options {
- readonly: boolean
- fileMustExist: boolean
- timeout: number
+export interface OpenDatabaseOptions {
+ readonly?: boolean
+ fileMustExist?: boolean
+ timeout?: number
+}
+export interface PragmaOptions {
+ simple: boolean
}
export declare class Database {
memory: boolean
readonly: boolean
- inTransaction: boolean
open: boolean
name: string
- constructor(path: string, options?: Options | undefined | null)
+ constructor(path: string, options?: OpenDatabaseOptions | undefined | null)
prepare(sql: string): Statement
- transaction(): void
- pragma(): void
+ pragma(pragmaName: string, options?: PragmaOptions | undefined | null): unknown
backup(): void
serialize(): void
function(): void
aggregate(): void
table(): void
- loadExtension(): void
+ loadExtension(path: string): void
+ exec(sql: string): void
+ close(): void
}
export declare class Statement {
- database: Database
source: string
- reader: boolean
- readonly: boolean
- busy: boolean
- get(): unknown
- all(): NapiResult
- run(args: Array): void
- static iterate(): void
- static pluck(): void
+ get(args?: Array | undefined | null): unknown
+ run(args?: Array | undefined | null): unknown
+ iterate(args?: Array | undefined | null): IteratorStatement
+ all(args?: Array | undefined | null): unknown
+ pluck(pluck?: boolean | undefined | null): void
static expand(): void
- static raw(): void
+ raw(raw?: boolean | undefined | null): void
static columns(): void
- static bind(): void
+ bind(args?: Array | undefined | null): Statement
}
+export declare class IteratorStatement { }
diff --git a/bindings/javascript/index.js b/bindings/javascript/index.js
index 4e9bf54a7..c1f087ea5 100644
--- a/bindings/javascript/index.js
+++ b/bindings/javascript/index.js
@@ -5,325 +5,313 @@
/* auto-generated by NAPI-RS */
const { existsSync, readFileSync } = require('fs')
-const { join } = require("path");
+const { join } = require('path')
-const { platform, arch } = process;
+const { platform, arch } = process
-let nativeBinding = null;
-let localFileExisted = false;
-let loadError = null;
+let nativeBinding = null
+let localFileExisted = false
+let loadError = null
function isMusl() {
// For Node 10
- if (!process.report || typeof process.report.getReport !== "function") {
+ if (!process.report || typeof process.report.getReport !== 'function') {
try {
- const lddPath = require("child_process")
- .execSync("which ldd")
- .toString()
- .trim();
- return readFileSync(lddPath, "utf8").includes("musl");
+ const lddPath = require('child_process').execSync('which ldd').toString().trim()
+ return readFileSync(lddPath, 'utf8').includes('musl')
} catch (e) {
- return true;
+ return true
}
} else {
- const { glibcVersionRuntime } = process.report.getReport().header;
- return !glibcVersionRuntime;
+ const { glibcVersionRuntime } = process.report.getReport().header
+ return !glibcVersionRuntime
}
}
switch (platform) {
- case "android":
+ case 'android':
switch (arch) {
- case "arm64":
- localFileExisted = existsSync(
- join(__dirname, "turso.android-arm64.node"),
- );
+ case 'arm64':
+ localFileExisted = existsSync(join(__dirname, 'turso.android-arm64.node'))
try {
if (localFileExisted) {
- nativeBinding = require("./turso.android-arm64.node");
+ nativeBinding = require('./turso.android-arm64.node')
} else {
- nativeBinding = require("@tursodatabase/turso-android-arm64");
+ nativeBinding = require('@tursodatabase/turso-android-arm64')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
- case "arm":
- localFileExisted = existsSync(
- join(__dirname, "turso.android-arm-eabi.node"),
- );
+ break
+ case 'arm':
+ localFileExisted = existsSync(join(__dirname, 'turso.android-arm-eabi.node'))
try {
if (localFileExisted) {
- nativeBinding = require("./turso.android-arm-eabi.node");
+ nativeBinding = require('./turso.android-arm-eabi.node')
} else {
- nativeBinding = require("@tursodatabase/turso-android-arm-eabi");
+ nativeBinding = require('@tursodatabase/turso-android-arm-eabi')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
+ break
default:
- throw new Error(`Unsupported architecture on Android ${arch}`);
+ throw new Error(`Unsupported architecture on Android ${arch}`)
}
- break;
- case "win32":
+ break
+ case 'win32':
switch (arch) {
- case "x64":
+ case 'x64':
localFileExisted = existsSync(
- join(__dirname, "turso.win32-x64-msvc.node"),
- );
+ join(__dirname, 'turso.win32-x64-msvc.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.win32-x64-msvc.node");
+ nativeBinding = require('./turso.win32-x64-msvc.node')
} else {
- nativeBinding = require("@tursodatabase/turso-win32-x64-msvc");
+ nativeBinding = require('@tursodatabase/turso-win32-x64-msvc')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
- case "ia32":
+ break
+ case 'ia32':
localFileExisted = existsSync(
- join(__dirname, "turso.win32-ia32-msvc.node"),
- );
+ join(__dirname, 'turso.win32-ia32-msvc.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.win32-ia32-msvc.node");
+ nativeBinding = require('./turso.win32-ia32-msvc.node')
} else {
- nativeBinding = require("@tursodatabase/turso-win32-ia32-msvc");
+ nativeBinding = require('@tursodatabase/turso-win32-ia32-msvc')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
- case "arm64":
+ break
+ case 'arm64':
localFileExisted = existsSync(
- join(__dirname, "turso.win32-arm64-msvc.node"),
- );
+ join(__dirname, 'turso.win32-arm64-msvc.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.win32-arm64-msvc.node");
+ nativeBinding = require('./turso.win32-arm64-msvc.node')
} else {
- nativeBinding = require("@tursodatabase/turso-win32-arm64-msvc");
+ nativeBinding = require('@tursodatabase/turso-win32-arm64-msvc')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
+ break
default:
- throw new Error(`Unsupported architecture on Windows: ${arch}`);
+ throw new Error(`Unsupported architecture on Windows: ${arch}`)
}
- break;
- case "darwin":
- localFileExisted = existsSync(
- join(__dirname, "turso.darwin-universal.node"),
- );
+ break
+ case 'darwin':
+ localFileExisted = existsSync(join(__dirname, 'turso.darwin-universal.node'))
try {
if (localFileExisted) {
- nativeBinding = require("./turso.darwin-universal.node");
+ nativeBinding = require('./turso.darwin-universal.node')
} else {
- nativeBinding = require("@tursodatabase/turso-darwin-universal");
+ nativeBinding = require('@tursodatabase/turso-darwin-universal')
}
- break;
+ break
} catch {}
switch (arch) {
- case "x64":
- localFileExisted = existsSync(
- join(__dirname, "turso.darwin-x64.node"),
- );
+ case 'x64':
+ localFileExisted = existsSync(join(__dirname, 'turso.darwin-x64.node'))
try {
if (localFileExisted) {
- nativeBinding = require("./turso.darwin-x64.node");
+ nativeBinding = require('./turso.darwin-x64.node')
} else {
- nativeBinding = require("@tursodatabase/turso-darwin-x64");
+ nativeBinding = require('@tursodatabase/turso-darwin-x64')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
- case "arm64":
+ break
+ case 'arm64':
localFileExisted = existsSync(
- join(__dirname, "turso.darwin-arm64.node"),
- );
+ join(__dirname, 'turso.darwin-arm64.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.darwin-arm64.node");
+ nativeBinding = require('./turso.darwin-arm64.node')
} else {
- nativeBinding = require("@tursodatabase/turso-darwin-arm64");
+ nativeBinding = require('@tursodatabase/turso-darwin-arm64')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
+ break
default:
- throw new Error(`Unsupported architecture on macOS: ${arch}`);
+ throw new Error(`Unsupported architecture on macOS: ${arch}`)
}
- break;
- case "freebsd":
- if (arch !== "x64") {
- throw new Error(`Unsupported architecture on FreeBSD: ${arch}`);
+ break
+ case 'freebsd':
+ if (arch !== 'x64') {
+ throw new Error(`Unsupported architecture on FreeBSD: ${arch}`)
}
- localFileExisted = existsSync(
- join(__dirname, "turso.freebsd-x64.node"),
- );
+ localFileExisted = existsSync(join(__dirname, 'turso.freebsd-x64.node'))
try {
if (localFileExisted) {
- nativeBinding = require("./turso.freebsd-x64.node");
+ nativeBinding = require('./turso.freebsd-x64.node')
} else {
- nativeBinding = require("@tursodatabase/turso-freebsd-x64");
+ nativeBinding = require('@tursodatabase/turso-freebsd-x64')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
- case "linux":
+ break
+ case 'linux':
switch (arch) {
- case "x64":
+ case 'x64':
if (isMusl()) {
localFileExisted = existsSync(
- join(__dirname, "turso.linux-x64-musl.node"),
- );
+ join(__dirname, 'turso.linux-x64-musl.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-x64-musl.node");
+ nativeBinding = require('./turso.linux-x64-musl.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-x64-musl");
+ nativeBinding = require('@tursodatabase/turso-linux-x64-musl')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
} else {
localFileExisted = existsSync(
- join(__dirname, "turso.linux-x64-gnu.node"),
- );
+ join(__dirname, 'turso.linux-x64-gnu.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-x64-gnu.node");
+ nativeBinding = require('./turso.linux-x64-gnu.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-x64-gnu");
+ nativeBinding = require('@tursodatabase/turso-linux-x64-gnu')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
}
- break;
- case "arm64":
+ break
+ case 'arm64':
if (isMusl()) {
localFileExisted = existsSync(
- join(__dirname, "turso.linux-arm64-musl.node"),
- );
+ join(__dirname, 'turso.linux-arm64-musl.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-arm64-musl.node");
+ nativeBinding = require('./turso.linux-arm64-musl.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-arm64-musl");
+ nativeBinding = require('@tursodatabase/turso-linux-arm64-musl')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
} else {
localFileExisted = existsSync(
- join(__dirname, "turso.linux-arm64-gnu.node"),
- );
+ join(__dirname, 'turso.linux-arm64-gnu.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-arm64-gnu.node");
+ nativeBinding = require('./turso.linux-arm64-gnu.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-arm64-gnu");
+ nativeBinding = require('@tursodatabase/turso-linux-arm64-gnu')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
}
- break;
- case "arm":
+ break
+ case 'arm':
if (isMusl()) {
localFileExisted = existsSync(
- join(__dirname, "turso.linux-arm-musleabihf.node"),
- );
+ join(__dirname, 'turso.linux-arm-musleabihf.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-arm-musleabihf.node");
+ nativeBinding = require('./turso.linux-arm-musleabihf.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-arm-musleabihf");
+ nativeBinding = require('@tursodatabase/turso-linux-arm-musleabihf')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
} else {
localFileExisted = existsSync(
- join(__dirname, "turso.linux-arm-gnueabihf.node"),
- );
+ join(__dirname, 'turso.linux-arm-gnueabihf.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-arm-gnueabihf.node");
+ nativeBinding = require('./turso.linux-arm-gnueabihf.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-arm-gnueabihf");
+ nativeBinding = require('@tursodatabase/turso-linux-arm-gnueabihf')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
}
- break;
- case "riscv64":
+ break
+ case 'riscv64':
if (isMusl()) {
localFileExisted = existsSync(
- join(__dirname, "turso.linux-riscv64-musl.node"),
- );
+ join(__dirname, 'turso.linux-riscv64-musl.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-riscv64-musl.node");
+ nativeBinding = require('./turso.linux-riscv64-musl.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-riscv64-musl");
+ nativeBinding = require('@tursodatabase/turso-linux-riscv64-musl')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
} else {
localFileExisted = existsSync(
- join(__dirname, "turso.linux-riscv64-gnu.node"),
- );
+ join(__dirname, 'turso.linux-riscv64-gnu.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-riscv64-gnu.node");
+ nativeBinding = require('./turso.linux-riscv64-gnu.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-riscv64-gnu");
+ nativeBinding = require('@tursodatabase/turso-linux-riscv64-gnu')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
}
- break;
- case "s390x":
+ break
+ case 's390x':
localFileExisted = existsSync(
- join(__dirname, "turso.linux-s390x-gnu.node"),
- );
+ join(__dirname, 'turso.linux-s390x-gnu.node')
+ )
try {
if (localFileExisted) {
- nativeBinding = require("./turso.linux-s390x-gnu.node");
+ nativeBinding = require('./turso.linux-s390x-gnu.node')
} else {
- nativeBinding = require("@tursodatabase/turso-linux-s390x-gnu");
+ nativeBinding = require('@tursodatabase/turso-linux-s390x-gnu')
}
} catch (e) {
- loadError = e;
+ loadError = e
}
- break;
+ break
default:
- throw new Error(`Unsupported architecture on Linux: ${arch}`);
+ throw new Error(`Unsupported architecture on Linux: ${arch}`)
}
- break;
+ break
default:
- throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`);
+ throw new Error(`Unsupported OS: ${platform}, architecture: ${arch}`)
}
if (!nativeBinding) {
if (loadError) {
- throw loadError;
+ throw loadError
}
- throw new Error(`Failed to load native binding`);
+ throw new Error(`Failed to load native binding`)
}
-const { Database, Statement } = nativeBinding;
+const { Database, Statement, IteratorStatement } = nativeBinding
-module.exports.Database = Database;
-module.exports.Statement = Statement;
+module.exports.Database = Database
+module.exports.Statement = Statement
+module.exports.IteratorStatement = IteratorStatement
diff --git a/bindings/javascript/npm/darwin-universal/package.json b/bindings/javascript/npm/darwin-universal/package.json
index c5047c2bb..d0d67e532 100644
--- a/bindings/javascript/npm/darwin-universal/package.json
+++ b/bindings/javascript/npm/darwin-universal/package.json
@@ -1,6 +1,6 @@
{
"name": "@tursodatabase/turso-darwin-universal",
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"repository": {
"type": "git",
"url": "https://github.com/tursodatabase/turso"
diff --git a/bindings/javascript/npm/linux-x64-gnu/package.json b/bindings/javascript/npm/linux-x64-gnu/package.json
index aa3d65f33..41f793ddb 100644
--- a/bindings/javascript/npm/linux-x64-gnu/package.json
+++ b/bindings/javascript/npm/linux-x64-gnu/package.json
@@ -1,6 +1,6 @@
{
"name": "@tursodatabase/turso-linux-x64-gnu",
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"repository": {
"type": "git",
"url": "https://github.com/tursodatabase/turso"
diff --git a/bindings/javascript/npm/win32-x64-msvc/package.json b/bindings/javascript/npm/win32-x64-msvc/package.json
index 0b4bac4cb..f5339ea01 100644
--- a/bindings/javascript/npm/win32-x64-msvc/package.json
+++ b/bindings/javascript/npm/win32-x64-msvc/package.json
@@ -1,6 +1,6 @@
{
"name": "@tursodatabase/turso-win32-x64-msvc",
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"repository": {
"type": "git",
"url": "https://github.com/tursodatabase/turso"
diff --git a/bindings/javascript/package-lock.json b/bindings/javascript/package-lock.json
index 98014d52b..93b39fe9c 100644
--- a/bindings/javascript/package-lock.json
+++ b/bindings/javascript/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@tursodatabase/turso",
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@tursodatabase/turso",
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"license": "MIT",
"devDependencies": {
"@napi-rs/cli": "^2.18.4",
diff --git a/bindings/javascript/package.json b/bindings/javascript/package.json
index 809015934..fbacd7543 100644
--- a/bindings/javascript/package.json
+++ b/bindings/javascript/package.json
@@ -1,6 +1,6 @@
{
"name": "@tursodatabase/turso",
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"repository": {
"type": "git",
"url": "https://github.com/tursodatabase/turso"
@@ -42,4 +42,4 @@
"version": "napi version"
},
"packageManager": "yarn@4.6.0"
-}
+}
\ No newline at end of file
diff --git a/bindings/javascript/sqlite-error.js b/bindings/javascript/sqlite-error.js
new file mode 100644
index 000000000..82356bc36
--- /dev/null
+++ b/bindings/javascript/sqlite-error.js
@@ -0,0 +1,22 @@
+'use strict';
+const descriptor = { value: 'SqliteError', writable: true, enumerable: false, configurable: true };
+
+function SqliteError(message, code, rawCode) {
+ if (new.target !== SqliteError) {
+ return new SqliteError(message, code);
+ }
+ if (typeof code !== 'string') {
+ throw new TypeError('Expected second argument to be a string');
+ }
+ Error.call(this, message);
+ descriptor.value = '' + message;
+ Object.defineProperty(this, 'message', descriptor);
+ Error.captureStackTrace(this, SqliteError);
+ this.code = code;
+ this.rawCode = rawCode
+}
+Object.setPrototypeOf(SqliteError, Error);
+Object.setPrototypeOf(SqliteError.prototype, Error.prototype);
+Object.defineProperty(SqliteError.prototype, 'name', descriptor);
+module.exports = SqliteError;
+
diff --git a/bindings/javascript/src/lib.rs b/bindings/javascript/src/lib.rs
index 248c240b4..15c32940f 100644
--- a/bindings/javascript/src/lib.rs
+++ b/bindings/javascript/src/lib.rs
@@ -14,12 +14,18 @@ use turso_core::{LimboError, StepResult};
#[derive(Default)]
#[napi(object)]
pub struct OpenDatabaseOptions {
- pub readonly: bool,
- pub file_must_exist: bool,
- pub timeout: u32,
+ pub readonly: Option,
+ pub file_must_exist: Option,
+ pub timeout: Option,
// verbose => Callback,
}
+impl OpenDatabaseOptions {
+ fn readonly(&self) -> bool {
+ self.readonly.unwrap_or(false)
+ }
+}
+
#[napi(object)]
pub struct PragmaOptions {
pub simple: bool,
@@ -41,7 +47,7 @@ pub struct Database {
pub name: String,
_db: Arc,
conn: Arc,
- io: Arc,
+ _io: Arc,
}
impl ObjectFinalize for Database {
@@ -55,34 +61,36 @@ impl ObjectFinalize for Database {
#[napi]
impl Database {
#[napi(constructor)]
- pub fn new(path: String, options: Option) -> napi::Result {
+ pub fn new(path: String, options: Option) -> napi::Result {
let memory = path == ":memory:";
let io: Arc = if memory {
Arc::new(turso_core::MemoryIO::new())
} else {
- Arc::new(turso_core::PlatformIO::new().map_err(into_napi_error)?)
+ Arc::new(turso_core::PlatformIO::new().map_err(into_napi_sqlite_error)?)
};
let opts = options.unwrap_or_default();
- let flag = if opts.readonly {
+ let flag = if opts.readonly() {
turso_core::OpenFlags::ReadOnly
} else {
turso_core::OpenFlags::Create
};
- let file = io.open_file(&path, flag, false).map_err(into_napi_error)?;
+ let file = io
+ .open_file(&path, flag, false)
+ .map_err(|err| into_napi_error_with_message("SQLITE_CANTOPEN".to_owned(), err))?;
let db_file = Arc::new(DatabaseFile::new(file));
let db = turso_core::Database::open(io.clone(), &path, db_file, false, false)
- .map_err(into_napi_error)?;
- let conn = db.connect().map_err(into_napi_error)?;
+ .map_err(into_napi_sqlite_error)?;
+ let conn = db.connect().map_err(into_napi_sqlite_error)?;
Ok(Self {
- readonly: opts.readonly,
+ readonly: opts.readonly(),
memory,
_db: db,
conn,
open: true,
name: path,
- io,
+ _io: io,
})
}
@@ -114,7 +122,7 @@ impl Database {
return Ok(env.get_undefined()?.into_unknown())
}
turso_core::StepResult::IO => {
- self.io.run_once().map_err(into_napi_error)?;
+ stmt.run_once().map_err(into_napi_error)?;
continue;
}
step @ turso_core::StepResult::Interrupt
@@ -131,16 +139,6 @@ impl Database {
}
}
- #[napi]
- pub fn readonly(&self) -> bool {
- self.readonly
- }
-
- #[napi]
- pub fn open(&self) -> bool {
- self.open
- }
-
#[napi]
pub fn backup(&self) {
todo!()
@@ -176,7 +174,7 @@ impl Database {
}
#[napi]
- pub fn exec(&self, sql: String) -> napi::Result<()> {
+ pub fn exec(&self, sql: String) -> napi::Result<(), String> {
let query_runner = self.conn.query_runner(sql.as_bytes());
// Since exec doesn't return any values, we can just iterate over the results
@@ -185,17 +183,17 @@ impl Database {
Ok(Some(mut stmt)) => loop {
match stmt.step() {
Ok(StepResult::Row) => continue,
- Ok(StepResult::IO) => self.io.run_once().map_err(into_napi_error)?,
+ Ok(StepResult::IO) => stmt.run_once().map_err(into_napi_sqlite_error)?,
Ok(StepResult::Done) => break,
Ok(StepResult::Interrupt | StepResult::Busy) => {
return Err(napi::Error::new(
- napi::Status::GenericFailure,
+ "SQLITE_ERROR".to_owned(),
"Statement execution interrupted or busy".to_string(),
));
}
Err(err) => {
return Err(napi::Error::new(
- napi::Status::GenericFailure,
+ "SQLITE_ERROR".to_owned(),
format!("Error executing SQL: {}", err),
));
}
@@ -204,7 +202,7 @@ impl Database {
Ok(None) => continue,
Err(err) => {
return Err(napi::Error::new(
- napi::Status::GenericFailure,
+ "SQLITE_ERROR".to_owned(),
format!("Error executing SQL: {}", err),
));
}
@@ -263,7 +261,7 @@ impl Statement {
#[napi]
pub fn get(&self, env: Env, args: Option>) -> napi::Result {
- let mut stmt = self.check_and_bind(args)?;
+ let mut stmt = self.check_and_bind(env, args)?;
loop {
let step = stmt.step().map_err(into_napi_error)?;
@@ -308,7 +306,7 @@ impl Statement {
}
turso_core::StepResult::Done => return Ok(env.get_undefined()?.into_unknown()),
turso_core::StepResult::IO => {
- self.database.io.run_once().map_err(into_napi_error)?;
+ stmt.run_once().map_err(into_napi_error)?;
continue;
}
turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => {
@@ -324,7 +322,7 @@ impl Statement {
// TODO: Return Info object (https://github.com/WiseLibs/better-sqlite3/blob/master/docs/api.md#runbindparameters---object)
#[napi]
pub fn run(&self, env: Env, args: Option>) -> napi::Result {
- let stmt = self.check_and_bind(args)?;
+ let stmt = self.check_and_bind(env, args)?;
self.internal_all(env, stmt)
}
@@ -335,10 +333,15 @@ impl Statement {
env: Env,
args: Option>,
) -> napi::Result {
- self.check_and_bind(args)?;
+ if let Some(some_args) = args.as_ref() {
+ if some_args.iter().len() != 0 {
+ self.check_and_bind(env, args)?;
+ }
+ }
+
Ok(IteratorStatement {
stmt: Rc::clone(&self.inner),
- database: self.database.clone(),
+ _database: self.database.clone(),
env,
presentation_mode: self.presentation_mode.clone(),
})
@@ -346,7 +349,7 @@ impl Statement {
#[napi]
pub fn all(&self, env: Env, args: Option>) -> napi::Result {
- let stmt = self.check_and_bind(args)?;
+ let stmt = self.check_and_bind(env, args)?;
self.internal_all(env, stmt)
}
@@ -401,7 +404,7 @@ impl Statement {
break;
}
turso_core::StepResult::IO => {
- self.database.io.run_once().map_err(into_napi_error)?;
+ stmt.run_once().map_err(into_napi_error)?;
}
turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => {
return Err(napi::Error::new(
@@ -444,8 +447,9 @@ impl Statement {
}
#[napi]
- pub fn bind(&mut self, args: Option>) -> napi::Result {
- self.check_and_bind(args)?;
+ pub fn bind(&mut self, env: Env, args: Option>) -> napi::Result {
+ self.check_and_bind(env, args)
+ .map_err(with_sqlite_error_message)?;
self.binded = true;
Ok(self.clone())
@@ -455,16 +459,22 @@ impl Statement {
/// and bind values do variables. The expected type for args is `Option>`
fn check_and_bind(
&self,
+ env: Env,
args: Option>,
) -> napi::Result> {
let mut stmt = self.inner.borrow_mut();
stmt.reset();
if let Some(args) = args {
if self.binded {
- return Err(napi::Error::new(
- napi::Status::InvalidArg,
- "This statement already has bound parameters",
- ));
+ let err = napi::Error::new(
+ into_convertible_type_error_message("TypeError"),
+ "The bind() method can only be invoked once per statement object",
+ );
+ unsafe {
+ napi::JsTypeError::from(err).throw_into(env.raw());
+ }
+
+ return Err(napi::Error::from_status(napi::Status::PendingException));
}
for (i, elem) in args.into_iter().enumerate() {
@@ -480,7 +490,7 @@ impl Statement {
#[napi(iterator)]
pub struct IteratorStatement {
stmt: Rc>,
- database: Database,
+ _database: Database,
env: Env,
presentation_mode: PresentationMode,
}
@@ -528,7 +538,7 @@ impl Generator for IteratorStatement {
}
turso_core::StepResult::Done => return None,
turso_core::StepResult::IO => {
- self.database.io.run_once().ok()?;
+ stmt.run_once().ok()?;
continue;
}
turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => return None,
@@ -630,6 +640,29 @@ impl turso_core::DatabaseStorage for DatabaseFile {
}
#[inline]
-pub fn into_napi_error(limbo_error: LimboError) -> napi::Error {
+fn into_napi_error(limbo_error: LimboError) -> napi::Error {
napi::Error::new(napi::Status::GenericFailure, format!("{limbo_error}"))
}
+
+#[inline]
+fn into_napi_sqlite_error(limbo_error: LimboError) -> napi::Error {
+ napi::Error::new(String::from("SQLITE_ERROR"), format!("{limbo_error}"))
+}
+
+#[inline]
+fn into_napi_error_with_message(
+ error_code: String,
+ limbo_error: LimboError,
+) -> napi::Error {
+ napi::Error::new(error_code, format!("{limbo_error}"))
+}
+
+#[inline]
+fn with_sqlite_error_message(err: napi::Error) -> napi::Error {
+ napi::Error::new("SQLITE_ERROR".to_owned(), err.reason)
+}
+
+#[inline]
+fn into_convertible_type_error_message(error_type: &str) -> String {
+ "[TURSO_CONVERT_TYPE]".to_owned() + error_type
+}
diff --git a/bindings/javascript/wrapper.js b/bindings/javascript/wrapper.js
index c42e1246d..0d4c53c96 100644
--- a/bindings/javascript/wrapper.js
+++ b/bindings/javascript/wrapper.js
@@ -2,6 +2,28 @@
const { Database: NativeDB } = require("./index.js");
+const SqliteError = require("./sqlite-error.js");
+
+const convertibleErrorTypes = { TypeError };
+const CONVERTIBLE_ERROR_PREFIX = '[TURSO_CONVERT_TYPE]';
+
+function convertError(err) {
+ if ((err.code ?? '').startsWith(CONVERTIBLE_ERROR_PREFIX)) {
+ return createErrorByName(err.code.substring(CONVERTIBLE_ERROR_PREFIX.length), err.message);
+ }
+
+ return new SqliteError(err.message, err.code, err.rawCode);
+}
+
+function createErrorByName(name, message) {
+ const ErrorConstructor = convertibleErrorTypes[name];
+ if (!ErrorConstructor) {
+ throw new Error(`unknown error type ${name} from Turso`);
+ }
+
+ return new ErrorConstructor(message);
+}
+
/**
* Database represents a connection that can prepare and execute SQL statements.
*/
@@ -145,7 +167,11 @@ class Database {
* @param {string} sql - The SQL statement string to execute.
*/
exec(sql) {
- this.db.exec(sql);
+ try {
+ this.db.exec(sql);
+ } catch (err) {
+ throw convertError(err);
+ }
}
/**
@@ -264,8 +290,13 @@ class Statement {
* @returns this - Statement with binded parameters
*/
bind(...bindParameters) {
- return this.stmt.bind(bindParameters.flat());
+ try {
+ return new Statement(this.stmt.bind(bindParameters.flat()), this.db);
+ } catch (err) {
+ throw convertError(err);
+ }
}
}
module.exports = Database;
+module.exports.SqliteError = SqliteError;
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index 83adf54c3..61693fb51 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -93,17 +93,24 @@ impl Cursor {
Ok::<(), anyhow::Error>(())
})?;
+ if stmt_is_dml && self.conn.conn.get_auto_commit() {
+ self.conn.conn.execute("BEGIN").map_err(|e| {
+ PyErr::new::(format!(
+ "Failed to start transaction after DDL: {:?}",
+ e
+ ))
+ })?;
+ }
+
// For DDL and DML statements,
// we need to execute the statement immediately
if stmt_is_ddl || stmt_is_dml || stmt_is_tx {
+ let mut stmt = stmt.borrow_mut();
while let turso_core::StepResult::IO = stmt
- .borrow_mut()
.step()
.map_err(|e| PyErr::new::(format!("Step error: {:?}", e)))?
{
- self.conn
- .io
- .run_once()
+ stmt.run_once()
.map_err(|e| PyErr::new::(format!("IO error: {:?}", e)))?;
}
}
@@ -132,7 +139,7 @@ impl Cursor {
return Ok(Some(py_row));
}
turso_core::StepResult::IO => {
- self.conn.io.run_once().map_err(|e| {
+ stmt.run_once().map_err(|e| {
PyErr::new::(format!("IO error: {:?}", e))
})?;
}
@@ -168,7 +175,7 @@ impl Cursor {
results.push(py_row);
}
turso_core::StepResult::IO => {
- self.conn.io.run_once().map_err(|e| {
+ stmt.run_once().map_err(|e| {
PyErr::new::(format!("IO error: {:?}", e))
})?;
}
@@ -233,7 +240,7 @@ fn stmt_is_tx(sql: &str) -> bool {
#[derive(Clone)]
pub struct Connection {
conn: Arc,
- io: Arc,
+ _io: Arc,
}
#[pymethods]
@@ -298,9 +305,11 @@ impl Connection {
impl Drop for Connection {
fn drop(&mut self) {
- self.conn
- .close()
- .expect("Failed to drop (close) connection");
+ if Arc::strong_count(&self.conn) == 1 {
+ self.conn
+ .close()
+ .expect("Failed to drop (close) connection");
+ }
}
}
@@ -308,7 +317,7 @@ impl Drop for Connection {
#[pyfunction]
pub fn connect(path: &str) -> Result {
match turso_core::Connection::from_uri(path, false, false) {
- Ok((io, conn)) => Ok(Connection { conn, io }),
+ Ok((io, conn)) => Ok(Connection { conn, _io: io }),
Err(e) => Err(PyErr::new::(format!(
"Failed to create connection: {:?}",
e
diff --git a/bindings/python/tests/test_database.py b/bindings/python/tests/test_database.py
index c9e1209dd..78c6987d0 100644
--- a/bindings/python/tests/test_database.py
+++ b/bindings/python/tests/test_database.py
@@ -158,6 +158,25 @@ def test_commit(provider):
assert record
+# Test case for: https://github.com/tursodatabase/turso/issues/2002
+@pytest.mark.parametrize("provider", ["sqlite3", "turso"])
+def test_first_rollback(provider, tmp_path):
+ db_file = tmp_path / "test_first_rollback.db"
+
+ conn = connect(provider, str(db_file))
+ cur = conn.cursor()
+ cur.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT)")
+ cur.execute("INSERT INTO users VALUES (1, 'alice')")
+ cur.execute("INSERT INTO users VALUES (2, 'bob')")
+
+ conn.rollback()
+
+ cur.execute("SELECT * FROM users")
+ users = cur.fetchall()
+
+ assert users == []
+ conn.close()
+
@pytest.mark.parametrize("provider", ["sqlite3", "turso"])
def test_with_statement(provider):
with connect(provider, "tests/database.db") as conn:
diff --git a/bindings/wasm/package-lock.json b/bindings/wasm/package-lock.json
index 93ef4f5fd..4ddc87922 100644
--- a/bindings/wasm/package-lock.json
+++ b/bindings/wasm/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "limbo-wasm",
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "limbo-wasm",
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"license": "MIT",
"devDependencies": {
"@playwright/test": "^1.49.1",
diff --git a/bindings/wasm/package.json b/bindings/wasm/package.json
index fa71211c2..463313751 100644
--- a/bindings/wasm/package.json
+++ b/bindings/wasm/package.json
@@ -3,7 +3,7 @@
"collaborators": [
"the Limbo authors"
],
- "version": "0.1.1",
+ "version": "0.1.2-pre.2",
"license": "MIT",
"repository": {
"type": "git",
diff --git a/cli/app.rs b/cli/app.rs
index a5a65e138..8136f16a5 100644
--- a/cli/app.rs
+++ b/cli/app.rs
@@ -24,6 +24,7 @@ use std::{
},
time::{Duration, Instant},
};
+use tracing::level_filters::LevelFilter;
use tracing_appender::non_blocking::WorkerGuard;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
use turso_core::{Connection, Database, LimboError, OpenFlags, Statement, StepResult, Value};
@@ -95,7 +96,7 @@ macro_rules! query_internal {
$body(row)?;
}
StepResult::IO => {
- $self.io.run_once()?;
+ rows.run_once()?;
}
StepResult::Interrupt => break,
StepResult::Done => break,
@@ -175,7 +176,6 @@ impl Limbo {
pub fn with_readline(mut self, mut rl: Editor) -> Self {
let h = LimboHelper::new(
self.conn.clone(),
- self.io.clone(),
self.config.as_ref().map(|c| c.highlight.clone()),
);
rl.set_helper(Some(h));
@@ -644,8 +644,7 @@ impl Limbo {
let _ = self.show_info();
}
Command::Import(args) => {
- let mut import_file =
- ImportFile::new(self.conn.clone(), self.io.clone(), &mut self.writer);
+ let mut import_file = ImportFile::new(self.conn.clone(), &mut self.writer);
import_file.import(args)
}
Command::LoadExtension(args) => {
@@ -740,7 +739,7 @@ impl Limbo {
}
Ok(StepResult::IO) => {
let start = Instant::now();
- self.io.run_once()?;
+ rows.run_once()?;
if let Some(ref mut stats) = statistics {
stats.io_time_elapsed_samples.push(start.elapsed());
}
@@ -833,7 +832,7 @@ impl Limbo {
}
Ok(StepResult::IO) => {
let start = Instant::now();
- self.io.run_once()?;
+ rows.run_once()?;
if let Some(ref mut stats) = statistics {
stats.io_time_elapsed_samples.push(start.elapsed());
}
@@ -908,7 +907,12 @@ impl Limbo {
.with_thread_ids(true)
.with_ansi(should_emit_ansi),
)
- .with(EnvFilter::from_default_env().add_directive("rustyline=off".parse().unwrap()))
+ .with(
+ EnvFilter::builder()
+ .with_default_directive(LevelFilter::OFF.into())
+ .from_env_lossy()
+ .add_directive("rustyline=off".parse().unwrap()),
+ )
.try_init()
{
println!("Unable to setup tracing appender: {:?}", e);
@@ -940,7 +944,7 @@ impl Limbo {
}
}
StepResult::IO => {
- self.io.run_once()?;
+ rows.run_once()?;
}
StepResult::Interrupt => break,
StepResult::Done => break,
@@ -996,7 +1000,7 @@ impl Limbo {
}
}
StepResult::IO => {
- self.io.run_once()?;
+ rows.run_once()?;
}
StepResult::Interrupt => break,
StepResult::Done => break,
@@ -1047,7 +1051,7 @@ impl Limbo {
}
}
StepResult::IO => {
- self.io.run_once()?;
+ rows.run_once()?;
}
StepResult::Interrupt => break,
StepResult::Done => break,
diff --git a/cli/commands/import.rs b/cli/commands/import.rs
index eee0b57d1..536dbcb24 100644
--- a/cli/commands/import.rs
+++ b/cli/commands/import.rs
@@ -21,17 +21,12 @@ pub struct ImportArgs {
pub struct ImportFile<'a> {
conn: Arc,
- io: Arc,
writer: &'a mut dyn Write,
}
impl<'a> ImportFile<'a> {
- pub fn new(
- conn: Arc,
- io: Arc,
- writer: &'a mut dyn Write,
- ) -> Self {
- Self { conn, io, writer }
+ pub fn new(conn: Arc, writer: &'a mut dyn Write) -> Self {
+ Self { conn, writer }
}
pub fn import(&mut self, args: ImportArgs) {
@@ -79,7 +74,7 @@ impl<'a> ImportFile<'a> {
while let Ok(x) = rows.step() {
match x {
turso_core::StepResult::IO => {
- self.io.run_once().unwrap();
+ rows.run_once().unwrap();
}
turso_core::StepResult::Done => break,
turso_core::StepResult::Interrupt => break,
diff --git a/cli/helper.rs b/cli/helper.rs
index 6076e1d0f..aee154662 100644
--- a/cli/helper.rs
+++ b/cli/helper.rs
@@ -40,11 +40,7 @@ pub struct LimboHelper {
}
impl LimboHelper {
- pub fn new(
- conn: Arc,
- io: Arc,
- syntax_config: Option,
- ) -> Self {
+ pub fn new(conn: Arc, syntax_config: Option) -> Self {
// Load only predefined syntax
let ps = from_uncompressed_data(include_bytes!(concat!(
env!("OUT_DIR"),
@@ -59,7 +55,7 @@ impl LimboHelper {
}
}
LimboHelper {
- completer: SqlCompleter::new(conn, io),
+ completer: SqlCompleter::new(conn),
syntax_set: ps,
theme_set: ts,
syntax_config: syntax_config.unwrap_or_default(),
@@ -141,7 +137,6 @@ impl Highlighter for LimboHelper {
pub struct SqlCompleter {
conn: Arc,
- io: Arc,
// Has to be a ref cell as Rustyline takes immutable reference to self
// This problem would be solved with Reedline as it uses &mut self for completions
cmd: RefCell,
@@ -149,10 +144,9 @@ pub struct SqlCompleter {
}
impl SqlCompleter {
- pub fn new(conn: Arc, io: Arc) -> Self {
+ pub fn new(conn: Arc) -> Self {
Self {
conn,
- io,
cmd: C::command().into(),
_cmd_phantom: PhantomData,
}
@@ -228,7 +222,7 @@ impl SqlCompleter {
candidates.push(pair);
}
StepResult::IO => {
- try_result!(self.io.run_once(), (prefix_pos, candidates));
+ try_result!(rows.run_once(), (prefix_pos, candidates));
}
StepResult::Interrupt => break,
StepResult::Done => break,
diff --git a/core/benches/benchmark.rs b/core/benches/benchmark.rs
index 5318a33c2..5ff69cba1 100644
--- a/core/benches/benchmark.rs
+++ b/core/benches/benchmark.rs
@@ -1,7 +1,7 @@
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use pprof::criterion::{Output, PProfProfiler};
use std::sync::Arc;
-use turso_core::{Database, PlatformIO, IO};
+use turso_core::{Database, PlatformIO};
fn rusqlite_open() -> rusqlite::Connection {
let sqlite_conn = rusqlite::Connection::open("../testing/testing.db").unwrap();
@@ -79,7 +79,6 @@ fn bench_execute_select_rows(criterion: &mut Criterion) {
let mut stmt = limbo_conn
.prepare(format!("SELECT * FROM users LIMIT {}", *i))
.unwrap();
- let io = io.clone();
b.iter(|| {
loop {
match stmt.step().unwrap() {
@@ -87,7 +86,7 @@ fn bench_execute_select_rows(criterion: &mut Criterion) {
black_box(stmt.row());
}
turso_core::StepResult::IO => {
- let _ = io.run_once();
+ stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
@@ -141,7 +140,6 @@ fn bench_execute_select_1(criterion: &mut Criterion) {
group.bench_function("limbo_execute_select_1", |b| {
let mut stmt = limbo_conn.prepare("SELECT 1").unwrap();
- let io = io.clone();
b.iter(|| {
loop {
match stmt.step().unwrap() {
@@ -149,7 +147,7 @@ fn bench_execute_select_1(criterion: &mut Criterion) {
black_box(stmt.row());
}
turso_core::StepResult::IO => {
- let _ = io.run_once();
+ stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
@@ -194,7 +192,6 @@ fn bench_execute_select_count(criterion: &mut Criterion) {
group.bench_function("limbo_execute_select_count", |b| {
let mut stmt = limbo_conn.prepare("SELECT count() FROM users").unwrap();
- let io = io.clone();
b.iter(|| {
loop {
match stmt.step().unwrap() {
@@ -202,7 +199,7 @@ fn bench_execute_select_count(criterion: &mut Criterion) {
black_box(stmt.row());
}
turso_core::StepResult::IO => {
- let _ = io.run_once();
+ stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
diff --git a/core/benches/json_benchmark.rs b/core/benches/json_benchmark.rs
index 3caa4e3bb..d458d60ea 100644
--- a/core/benches/json_benchmark.rs
+++ b/core/benches/json_benchmark.rs
@@ -4,7 +4,7 @@ use pprof::{
flamegraph::Options,
};
use std::sync::Arc;
-use turso_core::{Database, PlatformIO, IO};
+use turso_core::{Database, PlatformIO};
// Title: JSONB Function Benchmarking
@@ -447,13 +447,12 @@ fn bench(criterion: &mut Criterion) {
group.bench_function("Limbo", |b| {
let mut stmt = limbo_conn.prepare(&query).unwrap();
- let io = io.clone();
b.iter(|| {
loop {
match stmt.step().unwrap() {
turso_core::StepResult::Row => {}
turso_core::StepResult::IO => {
- let _ = io.run_once();
+ stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
@@ -606,13 +605,12 @@ fn bench_sequential_jsonb(criterion: &mut Criterion) {
group.bench_function("Limbo - Sequential", |b| {
let mut stmt = limbo_conn.prepare(&query).unwrap();
- let io = io.clone();
b.iter(|| {
loop {
match stmt.step().unwrap() {
turso_core::StepResult::Row => {}
turso_core::StepResult::IO => {
- let _ = io.run_once();
+ stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
@@ -899,13 +897,12 @@ fn bench_json_patch(criterion: &mut Criterion) {
group.bench_function("Limbo", |b| {
let mut stmt = limbo_conn.prepare(&query).unwrap();
- let io = io.clone();
b.iter(|| {
loop {
match stmt.step().unwrap() {
turso_core::StepResult::Row => {}
turso_core::StepResult::IO => {
- let _ = io.run_once();
+ stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
diff --git a/core/benches/tpc_h_benchmark.rs b/core/benches/tpc_h_benchmark.rs
index b976b5917..16bf857a5 100644
--- a/core/benches/tpc_h_benchmark.rs
+++ b/core/benches/tpc_h_benchmark.rs
@@ -2,7 +2,7 @@ use std::sync::Arc;
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, SamplingMode};
use pprof::criterion::{Output, PProfProfiler};
-use turso_core::{Database, PlatformIO, IO as _};
+use turso_core::{Database, PlatformIO};
const TPC_H_PATH: &str = "../perf/tpc-h/TPC-H.db";
@@ -97,7 +97,7 @@ fn bench_tpc_h_queries(criterion: &mut Criterion) {
black_box(stmt.row());
}
turso_core::StepResult::IO => {
- let _ = io.run_once();
+ stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
diff --git a/core/ext/vtab_xconnect.rs b/core/ext/vtab_xconnect.rs
index 2a5993f38..6d29613c3 100644
--- a/core/ext/vtab_xconnect.rs
+++ b/core/ext/vtab_xconnect.rs
@@ -65,7 +65,10 @@ pub unsafe extern "C" fn execute(
return ResultCode::OK;
}
Ok(StepResult::IO) => {
- let _ = conn.pager.io.run_once();
+ let res = stmt.run_once();
+ if res.is_err() {
+ return ResultCode::Error;
+ }
continue;
}
Ok(StepResult::Interrupt) => return ResultCode::Interrupt,
@@ -154,7 +157,6 @@ pub unsafe extern "C" fn stmt_step(stmt: *mut Stmt) -> ResultCode {
tracing::error!("stmt_step: null connection or context");
return ResultCode::Error;
}
- let conn: &Connection = unsafe { &*(stmt._conn as *const Connection) };
let stmt_ctx: &mut Statement = unsafe { &mut *(stmt._ctx as *mut Statement) };
while let Ok(res) = stmt_ctx.step() {
match res {
@@ -162,7 +164,10 @@ pub unsafe extern "C" fn stmt_step(stmt: *mut Stmt) -> ResultCode {
StepResult::Done => return ResultCode::EOF,
StepResult::IO => {
// always handle IO step result internally.
- let _ = conn.pager.io.run_once();
+ let res = stmt_ctx.run_once();
+ if res.is_err() {
+ return ResultCode::Error;
+ }
continue;
}
StepResult::Interrupt => return ResultCode::Interrupt,
diff --git a/core/function.rs b/core/function.rs
index 03c731f86..3467dd738 100644
--- a/core/function.rs
+++ b/core/function.rs
@@ -619,7 +619,8 @@ impl Func {
}
}
pub fn resolve_function(name: &str, arg_count: usize) -> Result {
- match name {
+ let normalized_name = crate::util::normalize_ident(name);
+ match normalized_name.as_str() {
"avg" => {
if arg_count != 1 {
crate::bail_parse_error!("wrong number of arguments to function {}()", name)
diff --git a/core/io/unix.rs b/core/io/unix.rs
index 76dfe3c05..235df10d0 100644
--- a/core/io/unix.rs
+++ b/core/io/unix.rs
@@ -18,7 +18,7 @@ use std::{
io::{ErrorKind, Read, Seek, Write},
sync::Arc,
};
-use tracing::{debug, trace};
+use tracing::{debug, instrument, trace, Level};
struct OwnedCallbacks(UnsafeCell);
// We assume we locking on IO level is done by user.
@@ -219,6 +219,7 @@ impl IO for UnixIO {
Ok(unix_file)
}
+ #[instrument(err, skip_all, level = Level::INFO)]
fn run_once(&self) -> Result<()> {
if self.callbacks.is_empty() {
return Ok(());
@@ -333,6 +334,7 @@ impl File for UnixFile<'_> {
Ok(())
}
+ #[instrument(err, skip_all, level = Level::INFO)]
fn pread(&self, pos: usize, c: Completion) -> Result> {
let file = self.file.borrow();
let result = {
@@ -366,6 +368,7 @@ impl File for UnixFile<'_> {
}
}
+ #[instrument(err, skip_all, level = Level::INFO)]
fn pwrite(
&self,
pos: usize,
@@ -401,6 +404,7 @@ impl File for UnixFile<'_> {
}
}
+ #[instrument(err, skip_all, level = Level::INFO)]
fn sync(&self, c: Completion) -> Result> {
let file = self.file.borrow();
let result = fs::fsync(file.as_fd());
@@ -415,6 +419,7 @@ impl File for UnixFile<'_> {
}
}
+ #[instrument(err, skip_all, level = Level::INFO)]
fn size(&self) -> Result {
let file = self.file.borrow();
Ok(file.metadata()?.len())
diff --git a/core/lib.rs b/core/lib.rs
index 4067aac15..f5cb80b57 100644
--- a/core/lib.rs
+++ b/core/lib.rs
@@ -43,6 +43,7 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
use crate::storage::{header_accessor, wal::DummyWAL};
use crate::translate::optimizer::optimize_plan;
+use crate::translate::pragma::TURSO_CDC_DEFAULT_TABLE_NAME;
use crate::util::{OpenMode, OpenOptions};
use crate::vtab::VirtualTable;
use core::str;
@@ -97,7 +98,7 @@ pub type Result = std::result::Result;
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum TransactionState {
- Write { change_schema: bool },
+ Write { schema_did_change: bool },
Read,
None,
}
@@ -217,7 +218,7 @@ impl Database {
if is_empty == 2 {
// parse schema
let conn = db.connect()?;
- let schema_version = get_schema_version(&conn, &io)?;
+ let schema_version = get_schema_version(&conn)?;
schema.write().schema_version = schema_version;
let rows = conn.query("SELECT * FROM sqlite_schema")?;
let mut schema = schema
@@ -225,7 +226,7 @@ impl Database {
.expect("lock on schema should succeed first try");
let syms = conn.syms.borrow();
if let Err(LimboError::ExtensionError(e)) =
- parse_schema_rows(rows, &mut schema, io, &syms, None)
+ parse_schema_rows(rows, &mut schema, &syms, None)
{
// this means that a vtab exists and we no longer have the module loaded. we print
// a warning to the user to load the module
@@ -278,6 +279,8 @@ impl Database {
cache_size: Cell::new(default_cache_size),
readonly: Cell::new(false),
wal_checkpoint_disabled: Cell::new(false),
+ capture_data_changes: RefCell::new(CaptureDataChangesMode::Off),
+ closed: Cell::new(false),
});
if let Err(e) = conn.register_builtins() {
return Err(LimboError::ExtensionError(e));
@@ -330,6 +333,8 @@ impl Database {
cache_size: Cell::new(default_cache_size),
readonly: Cell::new(false),
wal_checkpoint_disabled: Cell::new(false),
+ capture_data_changes: RefCell::new(CaptureDataChangesMode::Off),
+ closed: Cell::new(false),
});
if let Err(e) = conn.register_builtins() {
@@ -390,7 +395,7 @@ impl Database {
}
}
-fn get_schema_version(conn: &Arc, io: &Arc) -> Result {
+fn get_schema_version(conn: &Arc) -> Result {
let mut rows = conn
.query("PRAGMA schema_version")?
.ok_or(LimboError::InternalError(
@@ -409,7 +414,7 @@ fn get_schema_version(conn: &Arc, io: &Arc) -> Result {
schema_version = Some(row.get::(0)? as u32);
}
StepResult::IO => {
- io.run_once()?;
+ rows.run_once()?;
}
StepResult::Interrupt => {
return Err(LimboError::InternalError(
@@ -434,6 +439,39 @@ fn get_schema_version(conn: &Arc, io: &Arc) -> Result {
}
}
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum CaptureDataChangesMode {
+ Off,
+ RowidOnly { table: String },
+}
+
+impl CaptureDataChangesMode {
+ pub fn parse(value: &str) -> Result {
+ let (mode, table) = value
+ .split_once(",")
+ .unwrap_or((value, TURSO_CDC_DEFAULT_TABLE_NAME));
+ match mode {
+ "off" => Ok(CaptureDataChangesMode::Off),
+ "rowid-only" => Ok(CaptureDataChangesMode::RowidOnly { table: table.to_string() }),
+ _ => Err(LimboError::InvalidArgument(
+ "unexpected pragma value: expected '' or ',' parameter where mode is one of off|rowid-only".to_string(),
+ ))
+ }
+ }
+ pub fn mode_name(&self) -> &str {
+ match self {
+ CaptureDataChangesMode::Off => "off",
+ CaptureDataChangesMode::RowidOnly { .. } => "rowid-only",
+ }
+ }
+ pub fn table(&self) -> Option<&str> {
+ match self {
+ CaptureDataChangesMode::Off => None,
+ CaptureDataChangesMode::RowidOnly { table } => Some(table.as_str()),
+ }
+ }
+}
+
pub struct Connection {
_db: Arc,
pager: Rc,
@@ -450,11 +488,16 @@ pub struct Connection {
cache_size: Cell,
readonly: Cell,
wal_checkpoint_disabled: Cell,
+ capture_data_changes: RefCell,
+ closed: Cell,
}
impl Connection {
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
pub fn prepare(self: &Arc, sql: impl AsRef) -> Result {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
if sql.as_ref().is_empty() {
return Err(LimboError::InvalidArgument(
"The supplied SQL string contains no statements".to_string(),
@@ -494,8 +537,11 @@ impl Connection {
}
}
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
pub fn query(self: &Arc, sql: impl AsRef) -> Result> {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
let sql = sql.as_ref();
tracing::trace!("Querying: {}", sql);
let mut parser = Parser::new(sql.as_bytes());
@@ -510,12 +556,15 @@ impl Connection {
}
}
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
pub(crate) fn run_cmd(
self: &Arc,
cmd: Cmd,
input: &str,
) -> Result> {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
let syms = self.syms.borrow();
match cmd {
Cmd::Stmt(ref stmt) | Cmd::Explain(ref stmt) => {
@@ -563,8 +612,11 @@ impl Connection {
/// Execute will run a query from start to finish taking ownership of I/O because it will run pending I/Os if it didn't finish.
/// TODO: make this api async
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
pub fn execute(self: &Arc, sql: impl AsRef) -> Result<()> {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
let sql = sql.as_ref();
let mut parser = Parser::new(sql.as_bytes());
while let Some(cmd) = parser.next()? {
@@ -610,7 +662,7 @@ impl Connection {
if matches!(res, StepResult::Done) {
break;
}
- self._db.io.run_once()?;
+ self.run_once()?;
}
}
}
@@ -618,6 +670,20 @@ impl Connection {
Ok(())
}
+ fn run_once(&self) -> Result<()> {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
+ let res = self._db.io.run_once();
+ if res.is_err() {
+ let state = self.transaction_state.get();
+ if let TransactionState::Write { schema_did_change } = state {
+ self.pager.rollback(schema_did_change, self)?
+ }
+ }
+ res
+ }
+
#[cfg(feature = "fs")]
pub fn from_uri(
uri: &str,
@@ -676,6 +742,9 @@ impl Connection {
/// If the WAL size is over the checkpoint threshold, it will checkpoint the WAL to
/// the database file and then fsync the database file.
pub fn cacheflush(&self) -> Result {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
self.pager.cacheflush(self.wal_checkpoint_disabled.get())
}
@@ -685,12 +754,19 @@ impl Connection {
}
pub fn checkpoint(&self) -> Result {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
self.pager
.wal_checkpoint(self.wal_checkpoint_disabled.get())
}
/// Close a connection and checkpoint.
pub fn close(&self) -> Result<()> {
+ if self.closed.get() {
+ return Ok(());
+ }
+ self.closed.set(true);
self.pager
.checkpoint_shutdown(self.wal_checkpoint_disabled.get())
}
@@ -724,6 +800,13 @@ impl Connection {
self.cache_size.set(size);
}
+ pub fn get_capture_data_changes(&self) -> std::cell::Ref<'_, CaptureDataChangesMode> {
+ self.capture_data_changes.borrow()
+ }
+ pub fn set_capture_data_changes(&self, opts: CaptureDataChangesMode) {
+ self.capture_data_changes.replace(opts);
+ }
+
#[cfg(feature = "fs")]
pub fn open_new(&self, path: &str, vfs: &str) -> Result<(Arc, Arc)> {
Database::open_with_vfs(&self._db, path, vfs)
@@ -751,12 +834,15 @@ impl Connection {
}
pub fn parse_schema_rows(self: &Arc) -> Result<()> {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
let rows = self.query("SELECT * FROM sqlite_schema")?;
let mut schema = self.schema.borrow_mut();
{
let syms = self.syms.borrow();
if let Err(LimboError::ExtensionError(e)) =
- parse_schema_rows(rows, &mut schema, self.pager.io.clone(), &syms, None)
+ parse_schema_rows(rows, &mut schema, &syms, None)
{
// this means that a vtab exists and we no longer have the module loaded. we print
// a warning to the user to load the module
@@ -769,6 +855,9 @@ impl Connection {
// Clearly there is something to improve here, Vec> isn't a couple of tea
/// Query the current rows/values of `pragma_name`.
pub fn pragma_query(self: &Arc, pragma_name: &str) -> Result>> {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
let pragma = format!("PRAGMA {}", pragma_name);
let mut stmt = self.prepare(pragma)?;
let mut results = Vec::new();
@@ -797,6 +886,9 @@ impl Connection {
pragma_name: &str,
pragma_value: V,
) -> Result>> {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
let pragma = format!("PRAGMA {} = {}", pragma_name, pragma_value);
let mut stmt = self.prepare(pragma)?;
let mut results = Vec::new();
@@ -827,6 +919,9 @@ impl Connection {
pragma_name: &str,
pragma_value: V,
) -> Result>> {
+ if self.closed.get() {
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
let pragma = format!("PRAGMA {}({})", pragma_name, pragma_value);
let mut stmt = self.prepare(pragma)?;
let mut results = Vec::new();
@@ -883,7 +978,15 @@ impl Statement {
}
pub fn run_once(&self) -> Result<()> {
- self.pager.io.run_once()
+ let res = self.pager.io.run_once();
+ if res.is_err() {
+ let state = self.program.connection.transaction_state.get();
+ if let TransactionState::Write { schema_did_change } = state {
+ self.pager
+ .rollback(schema_did_change, &self.program.connection)?
+ }
+ }
+ res
}
pub fn num_columns(&self) -> usize {
diff --git a/core/pragma.rs b/core/pragma.rs
index a65aefff4..38d33a3fd 100644
--- a/core/pragma.rs
+++ b/core/pragma.rs
@@ -7,21 +7,21 @@ use turso_sqlite3_parser::ast::PragmaName;
bitflags! {
// Flag names match those used in SQLite:
// https://github.com/sqlite/sqlite/blob/b3c1884b65400da85636458298bd77cbbfdfb401/tool/mkpragmatab.tcl#L22-L29
- struct PragmaFlags: u8 {
- const NeedSchema = 0x01;
- const NoColumns = 0x02;
- const NoColumns1 = 0x04;
- const ReadOnly = 0x08;
- const Result0 = 0x10;
- const Result1 = 0x20;
- const SchemaOpt = 0x40;
- const SchemaReq = 0x80;
+ pub struct PragmaFlags: u8 {
+ const NeedSchema = 0x01; /* Force schema load before running */
+ const NoColumns = 0x02; /* OP_ResultRow called with zero columns */
+ const NoColumns1 = 0x04; /* zero columns if RHS argument is present */
+ const ReadOnly = 0x08; /* Read-only HEADER_VALUE */
+ const Result0 = 0x10; /* Acts as query when no argument */
+ const Result1 = 0x20; /* Acts as query when has one argument */
+ const SchemaOpt = 0x40; /* Schema restricts name search if present */
+ const SchemaReq = 0x80; /* Schema required - "main" is default */
}
}
-struct Pragma {
- flags: PragmaFlags,
- columns: &'static [&'static str],
+pub struct Pragma {
+ pub flags: PragmaFlags,
+ pub columns: &'static [&'static str],
}
impl Pragma {
@@ -30,7 +30,7 @@ impl Pragma {
}
}
-fn pragma_for(pragma: PragmaName) -> Pragma {
+pub fn pragma_for(pragma: PragmaName) -> Pragma {
use PragmaName::*;
match pragma {
@@ -77,6 +77,10 @@ fn pragma_for(pragma: PragmaName) -> Pragma {
PragmaFlags::NeedSchema | PragmaFlags::ReadOnly | PragmaFlags::Result0,
&["message"],
),
+ UnstableCaptureDataChangesConn => Pragma::new(
+ PragmaFlags::NeedSchema | PragmaFlags::Result0 | PragmaFlags::SchemaReq,
+ &["mode", "table"],
+ ),
}
}
diff --git a/core/storage/btree.rs b/core/storage/btree.rs
index c650d791d..5ae9d713c 100644
--- a/core/storage/btree.rs
+++ b/core/storage/btree.rs
@@ -22,6 +22,13 @@ use crate::{
LimboError, Result,
};
+use super::{
+ pager::PageRef,
+ sqlite3_ondisk::{
+ write_varint_to_vec, IndexInteriorCell, IndexLeafCell, OverflowCell, DATABASE_HEADER_SIZE,
+ MINIMUM_CELL_SIZE,
+ },
+};
#[cfg(debug_assertions)]
use std::collections::HashSet;
use std::{
@@ -34,13 +41,6 @@ use std::{
sync::Arc,
};
-use super::{
- pager::PageRef,
- sqlite3_ondisk::{
- write_varint_to_vec, IndexInteriorCell, IndexLeafCell, OverflowCell, DATABASE_HEADER_SIZE,
- },
-};
-
/// The B-Tree page header is 12 bytes for interior pages and 8 bytes for leaf pages.
///
/// +--------+-----------------+-----------------+-----------------+--------+----- ..... ----+
@@ -68,7 +68,14 @@ pub mod offset {
/// The number of cells in the page (u16).
pub const BTREE_CELL_COUNT: usize = 3;
- /// A pointer to first byte of cell allocated content from top (u16).
+ /// A pointer to the first byte of cell allocated content from top (u16).
+ ///
+ /// A zero value for this integer is interpreted as 65,536.
+ /// If a page contains no cells (which is only possible for a root page of a table that
+ /// contains no rows) then the offset to the cell content area will equal the page size minus
+ /// the bytes of reserved space. If the database uses a 65536-byte page size and the
+ /// reserved space is zero (the usual value for reserved space) then the cell content offset of
+ /// an empty page wants to be 6,5536
///
/// SQLite strives to place cells as far toward the end of the b-tree page as it can, in
/// order to leave space for future growth of the cell pointer array. This means that the
@@ -445,16 +452,16 @@ pub enum CursorSeekState {
}
#[derive(Debug)]
-struct FindCellState(Option);
+struct FindCellState(Option<(usize, usize)>); // low, high
impl FindCellState {
#[inline]
- fn set(&mut self, cell_idx: isize) {
- self.0 = Some(cell_idx)
+ fn set(&mut self, lowhigh: (usize, usize)) {
+ self.0 = Some(lowhigh);
}
#[inline]
- fn get_cell_idx(&mut self) -> isize {
+ fn get_state(&mut self) -> (usize, usize) {
self.0.expect("get can only be called after a set")
}
@@ -500,7 +507,7 @@ pub struct BTreeCursor {
/// Colations for Index Btree constraint checks
/// Contains the Collation Seq for the whole Index
/// This Vec should be empty for Table Btree
- collations: Vec,
+ pub collations: Vec,
seek_state: CursorSeekState,
/// Separate state to read a record with overflow pages. This separation from `state` is necessary as
/// we can be in a function that relies on `state`, but also needs to process overflow pages
@@ -592,6 +599,7 @@ impl BTreeCursor {
/// Check if the table is empty.
/// This is done by checking if the root page has no cells.
+ #[instrument(skip_all, level = Level::INFO)]
fn is_empty_table(&self) -> Result> {
if let Some(mv_cursor) = &self.mv_cursor {
let mv_cursor = mv_cursor.borrow();
@@ -606,7 +614,7 @@ impl BTreeCursor {
/// Move the cursor to the previous record and return it.
/// Used in backwards iteration.
- #[instrument(skip(self), level = Level::TRACE, name = "prev")]
+ #[instrument(skip(self), level = Level::INFO, name = "prev")]
fn get_prev_record(&mut self) -> Result> {
loop {
let page = self.stack.top();
@@ -666,19 +674,13 @@ impl BTreeCursor {
}
let cell_idx = self.stack.current_cell_index() as usize;
- let cell = contents.cell_get(
- cell_idx,
- payload_overflow_threshold_max(contents.page_type(), self.usable_space() as u16),
- payload_overflow_threshold_min(contents.page_type(), self.usable_space() as u16),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cell_idx, self.usable_space())?;
match cell {
BTreeCell::TableInteriorCell(TableInteriorCell {
- _left_child_page,
- _rowid,
+ left_child_page, ..
}) => {
- let mem_page = self.read_page(_left_child_page as usize)?;
+ let mem_page = self.read_page(left_child_page as usize)?;
self.stack.push_backwards(mem_page);
continue;
}
@@ -717,7 +719,7 @@ impl BTreeCursor {
/// Reads the record of a cell that has overflow pages. This is a state machine that requires to be called until completion so everything
/// that calls this function should be reentrant.
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
fn process_overflow_read(
&self,
payload: &'static [u8],
@@ -835,6 +837,7 @@ impl BTreeCursor {
///
/// If the cell has overflow pages, it will skip till the overflow page which
/// is at the offset given.
+ #[instrument(skip_all, level = Level::INFO)]
pub fn read_write_payload_with_offset(
&mut self,
mut offset: u32,
@@ -863,18 +866,11 @@ impl BTreeCursor {
}
let usable_size = self.usable_space();
- let cell = contents
- .cell_get(
- cell_idx,
- payload_overflow_threshold_max(contents.page_type(), usable_size as u16),
- payload_overflow_threshold_min(contents.page_type(), usable_size as u16),
- usable_size,
- )
- .unwrap();
+ let cell = contents.cell_get(cell_idx, usable_size).unwrap();
let (payload, payload_size, first_overflow_page) = match cell {
BTreeCell::TableLeafCell(cell) => {
- (cell._payload, cell.payload_size, cell.first_overflow_page)
+ (cell.payload, cell.payload_size, cell.first_overflow_page)
}
BTreeCell::IndexLeafCell(cell) => {
(cell.payload, cell.payload_size, cell.first_overflow_page)
@@ -945,6 +941,7 @@ impl BTreeCursor {
Ok(CursorResult::Ok(()))
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn continue_payload_overflow_with_offset(
&mut self,
buffer: &mut Vec,
@@ -1121,7 +1118,7 @@ impl BTreeCursor {
/// Move the cursor to the next record and return it.
/// Used in forwards iteration, which is the default.
- #[instrument(skip(self), level = Level::TRACE, name = "next")]
+ #[instrument(skip(self), level = Level::INFO, name = "next")]
fn get_next_record(&mut self) -> Result> {
if let Some(mv_cursor) = &self.mv_cursor {
let mut mv_cursor = mv_cursor.borrow_mut();
@@ -1206,18 +1203,12 @@ impl BTreeCursor {
}
turso_assert!(cell_idx < contents.cell_count(), "cell index out of bounds");
- let cell = contents.cell_get(
- cell_idx,
- payload_overflow_threshold_max(contents.page_type(), self.usable_space() as u16),
- payload_overflow_threshold_min(contents.page_type(), self.usable_space() as u16),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cell_idx, self.usable_space())?;
match &cell {
BTreeCell::TableInteriorCell(TableInteriorCell {
- _left_child_page,
- _rowid,
+ left_child_page, ..
}) => {
- let mem_page = self.read_page(*_left_child_page as usize)?;
+ let mem_page = self.read_page(*left_child_page as usize)?;
self.stack.push(mem_page);
continue;
}
@@ -1261,20 +1252,21 @@ impl BTreeCursor {
}
/// Move the cursor to the root page of the btree.
- #[instrument(skip_all, level = Level::TRACE)]
- fn move_to_root(&mut self) {
+ #[instrument(skip_all, level = Level::INFO)]
+ fn move_to_root(&mut self) -> Result<()> {
self.seek_state = CursorSeekState::Start;
self.going_upwards = false;
tracing::trace!(root_page = self.root_page);
- let mem_page = self.read_page(self.root_page).unwrap();
+ let mem_page = self.read_page(self.root_page)?;
self.stack.clear();
self.stack.push(mem_page);
+ Ok(())
}
/// Move the cursor to the rightmost record in the btree.
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
fn move_to_rightmost(&mut self) -> Result> {
- self.move_to_root();
+ self.move_to_root()?;
loop {
let mem_page = self.stack.top();
@@ -1307,7 +1299,7 @@ impl BTreeCursor {
}
/// Specialized version of move_to() for table btrees.
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
fn tablebtree_move_to(&mut self, rowid: i64, seek_op: SeekOp) -> Result> {
'outer: loop {
let page = self.stack.top();
@@ -1358,8 +1350,8 @@ impl BTreeCursor {
let max = max_cell_idx.get();
if min > max {
if let Some(nearest_matching_cell) = nearest_matching_cell.get() {
- let left_child_page = contents
- .cell_table_interior_read_left_child_page(nearest_matching_cell)?;
+ let left_child_page =
+ contents.cell_interior_read_left_child_page(nearest_matching_cell);
self.stack.set_cell_index(nearest_matching_cell as i32);
let mem_page = self.read_page(left_child_page as usize)?;
self.stack.push(mem_page);
@@ -1425,7 +1417,7 @@ impl BTreeCursor {
}
/// Specialized version of move_to() for index btrees.
- #[instrument(skip(self, index_key), level = Level::TRACE)]
+ #[instrument(skip(self, index_key), level = Level::INFO)]
fn indexbtree_move_to(
&mut self,
index_key: &ImmutableRecord,
@@ -1504,18 +1496,8 @@ impl BTreeCursor {
}
}
};
- let matching_cell = contents.cell_get(
- leftmost_matching_cell,
- payload_overflow_threshold_max(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )?;
+ let matching_cell =
+ contents.cell_get(leftmost_matching_cell, self.usable_space())?;
self.stack.set_cell_index(leftmost_matching_cell as i32);
// we don't advance in case of forward iteration and index tree internal nodes because we will visit this node going up.
// in backwards iteration, we must retreat because otherwise we would unnecessarily visit this node again.
@@ -1543,18 +1525,7 @@ impl BTreeCursor {
let cur_cell_idx = (min + max) >> 1; // rustc generates extra insns for (min+max)/2 due to them being isize. we know min&max are >=0 here.
self.stack.set_cell_index(cur_cell_idx as i32);
- let cell = contents.cell_get(
- cur_cell_idx as usize,
- payload_overflow_threshold_max(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cur_cell_idx as usize, self.usable_space())?;
let BTreeCell::IndexInteriorCell(IndexInteriorCell {
payload,
payload_size,
@@ -1641,7 +1612,7 @@ impl BTreeCursor {
/// Specialized version of do_seek() for table btrees that uses binary search instead
/// of iterating cells in order.
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
fn tablebtree_seek(&mut self, rowid: i64, seek_op: SeekOp) -> Result> {
turso_assert!(
self.mv_cursor.is_none(),
@@ -1761,7 +1732,7 @@ impl BTreeCursor {
}
}
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
fn indexbtree_seek(
&mut self,
key: &ImmutableRecord,
@@ -1836,12 +1807,7 @@ impl BTreeCursor {
let page = page.get();
let contents = page.get().contents.as_ref().unwrap();
let cur_cell_idx = self.stack.current_cell_index() as usize;
- let cell = contents.cell_get(
- cur_cell_idx,
- payload_overflow_threshold_max(contents.page_type(), self.usable_space() as u16),
- payload_overflow_threshold_min(contents.page_type(), self.usable_space() as u16),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cur_cell_idx, self.usable_space())?;
let BTreeCell::IndexInteriorCell(IndexInteriorCell {
payload,
first_overflow_page,
@@ -1935,12 +1901,7 @@ impl BTreeCursor {
let cur_cell_idx = (min + max) >> 1; // rustc generates extra insns for (min+max)/2 due to them being isize. we know min&max are >=0 here.
self.stack.set_cell_index(cur_cell_idx as i32);
- let cell = contents.cell_get(
- cur_cell_idx as usize,
- payload_overflow_threshold_max(contents.page_type(), self.usable_space() as u16),
- payload_overflow_threshold_min(contents.page_type(), self.usable_space() as u16),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cur_cell_idx as usize, self.usable_space())?;
let BTreeCell::IndexLeafCell(IndexLeafCell {
payload,
first_overflow_page,
@@ -2032,7 +1993,7 @@ impl BTreeCursor {
}
}
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
pub fn move_to(&mut self, key: SeekKey<'_>, cmp: SeekOp) -> Result> {
turso_assert!(
self.mv_cursor.is_none(),
@@ -2072,7 +2033,7 @@ impl BTreeCursor {
self.seek_state = CursorSeekState::Start;
}
if matches!(self.seek_state, CursorSeekState::Start) {
- self.move_to_root();
+ self.move_to_root()?;
}
let ret = match key {
@@ -2085,7 +2046,7 @@ impl BTreeCursor {
/// Insert a record into the btree.
/// If the insert operation overflows the page, it will be split and the btree will be balanced.
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
fn insert_into_page(&mut self, bkey: &BTreeKey) -> Result> {
let record = bkey
.get_record()
@@ -2130,15 +2091,13 @@ impl BTreeCursor {
// if the cell index is less than the total cells, check: if its an existing
// rowid, we are going to update / overwrite the cell
if cell_idx < page.get().get_contents().cell_count() {
- let cell = page.get().get_contents().cell_get(
- cell_idx,
- payload_overflow_threshold_max(page_type, self.usable_space() as u16),
- payload_overflow_threshold_min(page_type, self.usable_space() as u16),
- self.usable_space(),
- )?;
+ let cell = page
+ .get()
+ .get_contents()
+ .cell_get(cell_idx, self.usable_space())?;
match cell {
BTreeCell::TableLeafCell(tbl_leaf) => {
- if tbl_leaf._rowid == bkey.to_rowid() {
+ if tbl_leaf.rowid == bkey.to_rowid() {
tracing::debug!("TableLeafCell: found exact match with cell_idx={cell_idx}, overwriting");
self.overwrite_cell(page.clone(), cell_idx, record)?;
let write_info = self
@@ -2214,10 +2173,10 @@ impl BTreeCursor {
cell_idx,
self.usable_space() as u16,
)?;
- contents.overflow_cells.len()
+ !contents.overflow_cells.is_empty()
};
self.stack.set_cell_index(cell_idx as i32);
- if overflow > 0 {
+ if overflow {
// A balance will happen so save the key we were inserting
tracing::debug!(page = page.get().get().id, cell_idx, "balance triggered:");
self.save_context(match bkey {
@@ -2266,7 +2225,7 @@ impl BTreeCursor {
/// This is a naive algorithm that doesn't try to distribute cells evenly by content.
/// It will try to split the page in half by keys not by content.
/// Sqlite tries to have a page at least 40% full.
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
fn balance(&mut self) -> Result> {
turso_assert!(
matches!(self.state, CursorState::Write(_)),
@@ -2310,7 +2269,7 @@ impl BTreeCursor {
}
if !self.stack.has_parent() {
- self.balance_root();
+ self.balance_root()?;
}
let write_info = self.state.mut_write_info().unwrap();
@@ -2328,6 +2287,7 @@ impl BTreeCursor {
}
/// Balance a non root page by trying to balance cells between a maximum of 3 siblings that should be neighboring the page that overflowed/underflowed.
+ #[instrument(skip_all, level = Level::INFO)]
fn balance_non_root(&mut self) -> Result> {
turso_assert!(
matches!(self.state, CursorState::Write(_)),
@@ -2419,14 +2379,6 @@ impl BTreeCursor {
} else {
let (start_of_cell, _) = parent_contents.cell_get_raw_region(
first_cell_divider + sibling_pointer,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
self.usable_space(),
);
let buf = parent_contents.as_ptr().as_mut_ptr();
@@ -2462,20 +2414,9 @@ impl BTreeCursor {
break;
}
let next_cell_divider = i + first_cell_divider - 1;
- pgno = match parent_contents.cell_get(
- next_cell_divider,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )? {
+ pgno = match parent_contents.cell_get(next_cell_divider, self.usable_space())? {
BTreeCell::TableInteriorCell(table_interior_cell) => {
- table_interior_cell._left_child_page
+ table_interior_cell.left_child_page
}
BTreeCell::IndexInteriorCell(index_interior_cell) => {
index_interior_cell.left_child_page
@@ -2561,18 +2502,8 @@ impl BTreeCursor {
}
// Since we know we have a left sibling, take the divider that points to left sibling of this page
let cell_idx = balance_info.first_divider_cell + i;
- let (cell_start, cell_len) = parent_contents.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- );
+ let (cell_start, cell_len) =
+ parent_contents.cell_get_raw_region(cell_idx, self.usable_space());
let buf = parent_contents.as_ptr();
let cell_buf = &buf[cell_start..cell_start + cell_len];
max_cells += 1;
@@ -2624,18 +2555,8 @@ impl BTreeCursor {
let old_page_contents = old_page.get_contents();
debug_validate_cells!(&old_page_contents, self.usable_space() as u16);
for cell_idx in 0..old_page_contents.cell_count() {
- let (cell_start, cell_len) = old_page_contents.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(
- old_page_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- old_page_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- );
+ let (cell_start, cell_len) =
+ old_page_contents.cell_get_raw_region(cell_idx, self.usable_space());
let buf = old_page_contents.as_ptr();
let cell_buf = &mut buf[cell_start..cell_start + cell_len];
// TODO(pere): make this reference and not copy
@@ -2885,7 +2806,7 @@ impl BTreeCursor {
pages_to_balance_new[i].replace(page.clone());
} else {
// FIXME: handle page cache is full
- let page = self.allocate_page(page_type, 0);
+ let page = self.allocate_page(page_type, 0)?;
pages_to_balance_new[i].replace(page);
// Since this page didn't exist before, we can set it to cells length as it
// marks them as empty since it is a prefix sum of cells.
@@ -3257,18 +3178,8 @@ impl BTreeCursor {
page: &std::sync::Arc,
) {
let left_pointer = if parent_contents.overflow_cells.is_empty() {
- let (cell_start, cell_len) = parent_contents.cell_get_raw_region(
- balance_info.first_divider_cell + i,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- );
+ let (cell_start, cell_len) = parent_contents
+ .cell_get_raw_region(balance_info.first_divider_cell + i, self.usable_space());
tracing::debug!(
"balance_non_root(cell_start={}, cell_len={})",
cell_start,
@@ -3313,22 +3224,11 @@ impl BTreeCursor {
let mut current_index_cell = 0;
for cell_idx in 0..parent_contents.cell_count() {
let cell = parent_contents
- .cell_get(
- cell_idx,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )
+ .cell_get(cell_idx, self.usable_space())
.unwrap();
match cell {
BTreeCell::TableInteriorCell(table_interior_cell) => {
- let left_child_page = table_interior_cell._left_child_page;
+ let left_child_page = table_interior_cell.left_child_page;
if left_child_page == parent_page.get().get().id as u32 {
tracing::error!("balance_non_root(parent_divider_points_to_same_page, page_id={}, cell_left_child_page={})",
parent_page.get().get().id,
@@ -3362,18 +3262,8 @@ impl BTreeCursor {
debug_validate_cells!(contents, self.usable_space() as u16);
// Cells are distributed in order
for cell_idx in 0..contents.cell_count() {
- let (cell_start, cell_len) = contents.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- );
+ let (cell_start, cell_len) =
+ contents.cell_get_raw_region(cell_idx, self.usable_space());
let buf = contents.as_ptr();
let cell_buf = to_static_buf(&mut buf[cell_start..cell_start + cell_len]);
let cell_buf_in_array = &cells_debug[current_index_cell];
@@ -3387,22 +3277,14 @@ impl BTreeCursor {
let cell = crate::storage::sqlite3_ondisk::read_btree_cell(
cell_buf,
- &page_type,
+ contents,
0,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
self.usable_space(),
)
.unwrap();
match &cell {
BTreeCell::TableInteriorCell(table_interior_cell) => {
- let left_child_page = table_interior_cell._left_child_page;
+ let left_child_page = table_interior_cell.left_child_page;
if left_child_page == page.get().id as u32 {
tracing::error!("balance_non_root(child_page_points_same_page, page_id={}, cell_left_child_page={}, page_idx={})",
page.get().id,
@@ -3530,31 +3412,11 @@ impl BTreeCursor {
for (parent_cell_idx, cell_buf_in_array) in
cells_debug.iter().enumerate().take(contents.cell_count())
{
- let (parent_cell_start, parent_cell_len) = parent_contents.cell_get_raw_region(
- parent_cell_idx,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- );
+ let (parent_cell_start, parent_cell_len) =
+ parent_contents.cell_get_raw_region(parent_cell_idx, self.usable_space());
- let (cell_start, cell_len) = contents.cell_get_raw_region(
- parent_cell_idx,
- payload_overflow_threshold_max(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- );
+ let (cell_start, cell_len) =
+ contents.cell_get_raw_region(parent_cell_idx, self.usable_space());
let buf = contents.as_ptr();
let cell_buf = to_static_buf(&mut buf[cell_start..cell_start + cell_len]);
@@ -3612,18 +3474,8 @@ impl BTreeCursor {
}
// check if overflow
// check if right pointer, this is the last page. Do we update rightmost pointer and defragment moves it?
- let (cell_start, cell_len) = parent_contents.cell_get_raw_region(
- cell_divider_idx,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- );
+ let (cell_start, cell_len) =
+ parent_contents.cell_get_raw_region(cell_divider_idx, self.usable_space());
let cell_left_pointer = read_u32(&parent_buf[cell_start..cell_start + cell_len], 0);
if cell_left_pointer != page.get().id as u32 {
tracing::error!("balance_non_root(cell_divider_left_pointer, should point to page_id={}, but points to {}, divider_cell={}, overflow_cells_parent={})",
@@ -3645,40 +3497,21 @@ impl BTreeCursor {
to_static_buf(&mut cells_debug[current_index_cell - 1]);
let cell = crate::storage::sqlite3_ondisk::read_btree_cell(
cell_buf,
- &page_type,
+ contents,
0,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
self.usable_space(),
)
.unwrap();
let parent_cell = parent_contents
- .cell_get(
- cell_divider_idx,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )
+ .cell_get(cell_divider_idx, self.usable_space())
.unwrap();
let rowid = match cell {
- BTreeCell::TableLeafCell(table_leaf_cell) => table_leaf_cell._rowid,
+ BTreeCell::TableLeafCell(table_leaf_cell) => table_leaf_cell.rowid,
_ => unreachable!(),
};
let rowid_parent = match parent_cell {
BTreeCell::TableInteriorCell(table_interior_cell) => {
- table_interior_cell._rowid
+ table_interior_cell.rowid
}
_ => unreachable!(),
};
@@ -3717,18 +3550,8 @@ impl BTreeCursor {
}
continue;
}
- let (parent_cell_start, parent_cell_len) = parent_contents.cell_get_raw_region(
- cell_divider_idx,
- payload_overflow_threshold_max(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- parent_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- );
+ let (parent_cell_start, parent_cell_len) =
+ parent_contents.cell_get_raw_region(cell_divider_idx, self.usable_space());
let cell_buf_in_array = &cells_debug[current_index_cell];
let left_pointer = read_u32(
&parent_buf[parent_cell_start..parent_cell_start + parent_cell_len],
@@ -3780,7 +3603,7 @@ impl BTreeCursor {
/// Balance the root page.
/// This is done when the root page overflows, and we need to create a new root page.
/// See e.g. https://en.wikipedia.org/wiki/B-tree
- fn balance_root(&mut self) {
+ fn balance_root(&mut self) -> Result<()> {
/* todo: balance deeper, create child and copy contents of root there. Then split root */
/* if we are in root page then we just need to create a new root and push key there */
@@ -3797,7 +3620,7 @@ impl BTreeCursor {
// FIXME: handle page cache is full
let child_btree =
self.pager
- .do_allocate_page(root_contents.page_type(), 0, BtreePageAllocMode::Any);
+ .do_allocate_page(root_contents.page_type(), 0, BtreePageAllocMode::Any)?;
tracing::debug!(
"balance_root(root={}, rightmost={}, page_type={:?})",
@@ -3855,6 +3678,7 @@ impl BTreeCursor {
self.stack.push(root_btree.clone());
self.stack.set_cell_index(0); // leave parent pointing at the rightmost pointer (in this case 0, as there are no cells), since we will be balancing the rightmost child page.
self.stack.push(child_btree.clone());
+ Ok(())
}
fn usable_space(&self) -> usize {
@@ -3862,33 +3686,27 @@ impl BTreeCursor {
}
/// Find the index of the cell in the page that contains the given rowid.
+ #[instrument( skip_all, level = Level::INFO)]
fn find_cell(&mut self, page: &PageContent, key: &BTreeKey) -> Result> {
- if self.find_cell_state.0.is_none() {
- self.find_cell_state.set(0);
- }
let cell_count = page.cell_count();
- while self.find_cell_state.get_cell_idx() < cell_count as isize {
- assert!(self.find_cell_state.get_cell_idx() >= 0);
- let cell_idx = self.find_cell_state.get_cell_idx() as usize;
- match page
- .cell_get(
- cell_idx,
- payload_overflow_threshold_max(page.page_type(), self.usable_space() as u16),
- payload_overflow_threshold_min(page.page_type(), self.usable_space() as u16),
- self.usable_space(),
- )
- .unwrap()
- {
- BTreeCell::TableLeafCell(cell) => {
- if key.to_rowid() <= cell._rowid {
- break;
- }
- }
- BTreeCell::TableInteriorCell(cell) => {
- if key.to_rowid() <= cell._rowid {
- break;
- }
- }
+ let mut low = 0;
+ let mut high = if cell_count > 0 { cell_count - 1 } else { 0 };
+ let mut result_index = cell_count;
+ if self.find_cell_state.0.is_some() {
+ (low, high) = self.find_cell_state.get_state();
+ }
+
+ while low <= high && cell_count > 0 {
+ let mid = low + (high - low) / 2;
+ self.find_cell_state.set((low, high));
+ let cell = match page.cell_get(mid, self.usable_space()) {
+ Ok(c) => c,
+ Err(e) => return Err(e),
+ };
+
+ let comparison_result = match cell {
+ BTreeCell::TableLeafCell(cell) => key.to_rowid().cmp(&cell.rowid),
+ BTreeCell::TableInteriorCell(cell) => key.to_rowid().cmp(&cell.rowid),
BTreeCell::IndexInteriorCell(IndexInteriorCell {
payload,
first_overflow_page,
@@ -3899,46 +3717,57 @@ impl BTreeCursor {
payload,
first_overflow_page,
payload_size,
+ ..
}) => {
// TODO: implement efficient comparison of records
// e.g. https://github.com/sqlite/sqlite/blob/master/src/vdbeaux.c#L4719
return_if_io!(self.read_record_w_possible_overflow(
payload,
first_overflow_page,
- payload_size,
+ payload_size
));
+
let key_values = key.to_index_key_values();
let record = self.get_immutable_record();
let record = record.as_ref().unwrap();
let record_same_number_cols = &record.get_values()[..key_values.len()];
- let order = compare_immutable(
+ compare_immutable(
key_values,
record_same_number_cols,
self.key_sort_order(),
&self.collations,
- );
- match order {
- Ordering::Less | Ordering::Equal => {
- break;
- }
- Ordering::Greater => {}
+ )
+ }
+ };
+
+ match comparison_result {
+ Ordering::Equal => {
+ result_index = mid;
+ break;
+ }
+ Ordering::Greater => {
+ low = mid + 1;
+ }
+ Ordering::Less => {
+ result_index = mid;
+ if mid == 0 {
+ break;
}
+ high = mid - 1;
}
}
- let cell_idx = self.find_cell_state.get_cell_idx();
- self.find_cell_state.set(cell_idx + 1);
}
- let cell_idx = self.find_cell_state.get_cell_idx();
- assert!(cell_idx >= 0);
- let cell_idx = cell_idx as usize;
- assert!(cell_idx <= cell_count);
+
self.find_cell_state.reset();
- Ok(CursorResult::Ok(cell_idx))
+ assert!(result_index <= cell_count);
+
+ Ok(CursorResult::Ok(result_index))
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn seek_end(&mut self) -> Result> {
assert!(self.mv_cursor.is_none()); // unsure about this -_-
- self.move_to_root();
+ self.move_to_root()?;
loop {
let mem_page = self.stack.top();
let page_id = mem_page.get().get().id;
@@ -3964,6 +3793,7 @@ impl BTreeCursor {
}
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn seek_to_last(&mut self) -> Result> {
let has_record = return_if_io!(self.move_to_rightmost());
self.invalidate_record();
@@ -3984,13 +3814,14 @@ impl BTreeCursor {
self.root_page
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn rewind(&mut self) -> Result> {
if self.mv_cursor.is_some() {
let cursor_has_record = return_if_io!(self.get_next_record());
self.invalidate_record();
self.has_record.replace(cursor_has_record);
} else {
- self.move_to_root();
+ self.move_to_root()?;
let cursor_has_record = return_if_io!(self.get_next_record());
self.invalidate_record();
@@ -3999,6 +3830,7 @@ impl BTreeCursor {
Ok(CursorResult::Ok(()))
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn last(&mut self) -> Result> {
assert!(self.mv_cursor.is_none());
let cursor_has_record = return_if_io!(self.move_to_rightmost());
@@ -4007,6 +3839,7 @@ impl BTreeCursor {
Ok(CursorResult::Ok(()))
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn next(&mut self) -> Result> {
return_if_io!(self.restore_context());
let cursor_has_record = return_if_io!(self.get_next_record());
@@ -4022,6 +3855,7 @@ impl BTreeCursor {
.invalidate();
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn prev(&mut self) -> Result> {
assert!(self.mv_cursor.is_none());
return_if_io!(self.restore_context());
@@ -4031,7 +3865,7 @@ impl BTreeCursor {
Ok(CursorResult::Ok(cursor_has_record))
}
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
pub fn rowid(&mut self) -> Result>> {
if let Some(mv_cursor) = &self.mv_cursor {
let mv_cursor = mv_cursor.borrow();
@@ -4048,23 +3882,15 @@ impl BTreeCursor {
let page = page.get();
let contents = page.get_contents();
let cell_idx = self.stack.current_cell_index();
- let cell = contents.cell_get(
- cell_idx as usize,
- payload_overflow_threshold_max(contents.page_type(), self.usable_space() as u16),
- payload_overflow_threshold_min(contents.page_type(), self.usable_space() as u16),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cell_idx as usize, self.usable_space())?;
if page_type.is_table() {
- let BTreeCell::TableLeafCell(TableLeafCell {
- _rowid, _payload, ..
- }) = cell
- else {
+ let BTreeCell::TableLeafCell(TableLeafCell { rowid, .. }) = cell else {
unreachable!(
"BTreeCursor::rowid(): unexpected page_type: {:?}",
page_type
);
};
- Ok(CursorResult::Ok(Some(_rowid)))
+ Ok(CursorResult::Ok(Some(rowid)))
} else {
Ok(CursorResult::Ok(self.get_index_rowid_from_record()))
}
@@ -4073,7 +3899,7 @@ impl BTreeCursor {
}
}
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
pub fn seek(&mut self, key: SeekKey<'_>, op: SeekOp) -> Result> {
assert!(self.mv_cursor.is_none());
// Empty trace to capture the span information
@@ -4094,7 +3920,7 @@ impl BTreeCursor {
/// Return a reference to the record the cursor is currently pointing to.
/// If record was not parsed yet, then we have to parse it and in case of I/O we yield control
/// back.
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
pub fn record(&self) -> Result>>> {
if !self.has_record.get() {
return Ok(CursorResult::Ok(None));
@@ -4121,24 +3947,19 @@ impl BTreeCursor {
let page = page.get();
let contents = page.get_contents();
let cell_idx = self.stack.current_cell_index();
- let cell = contents.cell_get(
- cell_idx as usize,
- payload_overflow_threshold_max(contents.page_type(), self.usable_space() as u16),
- payload_overflow_threshold_min(contents.page_type(), self.usable_space() as u16),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cell_idx as usize, self.usable_space())?;
let (payload, payload_size, first_overflow_page) = match cell {
BTreeCell::TableLeafCell(TableLeafCell {
- _rowid,
- _payload,
- payload_size,
- first_overflow_page,
- }) => (_payload, payload_size, first_overflow_page),
- BTreeCell::IndexInteriorCell(IndexInteriorCell {
- left_child_page: _,
payload,
payload_size,
first_overflow_page,
+ ..
+ }) => (payload, payload_size, first_overflow_page),
+ BTreeCell::IndexInteriorCell(IndexInteriorCell {
+ payload,
+ payload_size,
+ first_overflow_page,
+ ..
}) => (payload, payload_size, first_overflow_page),
BTreeCell::IndexLeafCell(IndexLeafCell {
payload,
@@ -4162,7 +3983,7 @@ impl BTreeCursor {
Ok(CursorResult::Ok(Some(record_ref)))
}
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
pub fn insert(
&mut self,
key: &BTreeKey,
@@ -4232,7 +4053,7 @@ impl BTreeCursor {
/// 7. WaitForBalancingToComplete -> perform balancing
/// 8. SeekAfterBalancing -> adjust the cursor to a node that is closer to the deleted value. go to Finish
/// 9. Finish -> Delete operation is done. Return CursorResult(Ok())
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
pub fn delete(&mut self) -> Result> {
assert!(self.mv_cursor.is_none());
@@ -4259,13 +4080,10 @@ impl BTreeCursor {
page.get().get_contents().page_type(),
PageType::TableLeaf | PageType::TableInterior
) {
- let _target_rowid = match return_if_io!(self.rowid()) {
- Some(rowid) => rowid,
- _ => {
- self.state = CursorState::None;
- return Ok(CursorResult::Ok(()));
- }
- };
+ if return_if_io!(self.rowid()).is_none() {
+ self.state = CursorState::None;
+ return Ok(CursorResult::Ok(()));
+ }
} else if self.reusable_immutable_record.borrow().is_none() {
self.state = CursorState::None;
return Ok(CursorResult::Ok(()));
@@ -4334,21 +4152,10 @@ impl BTreeCursor {
cell_idx
);
- let cell = contents.cell_get(
- cell_idx,
- payload_overflow_threshold_max(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cell_idx, self.usable_space())?;
let original_child_pointer = match &cell {
- BTreeCell::TableInteriorCell(interior) => Some(interior._left_child_page),
+ BTreeCell::TableInteriorCell(interior) => Some(interior.left_child_page),
BTreeCell::IndexInteriorCell(interior) => Some(interior.left_child_page),
_ => None,
};
@@ -4374,8 +4181,6 @@ impl BTreeCursor {
let page = page.get();
let contents = page.get_contents();
- let is_last_cell = cell_idx == contents.cell_count().saturating_sub(1);
-
let delete_info = self.state.mut_delete_info().unwrap();
if !contents.is_leaf() {
delete_info.state = DeleteState::InteriorNodeReplacement {
@@ -4384,7 +4189,7 @@ impl BTreeCursor {
post_balancing_seek_key,
};
} else {
- let contents = page.get().contents.as_mut().unwrap();
+ let is_last_cell = cell_idx == contents.cell_count().saturating_sub(1);
drop_cell(contents, cell_idx, self.usable_space() as u16)?;
let delete_info = self.state.mut_delete_info().unwrap();
@@ -4414,18 +4219,8 @@ impl BTreeCursor {
assert!(leaf_contents.is_leaf());
assert!(leaf_contents.cell_count() > 0);
let leaf_cell_idx = leaf_contents.cell_count() - 1;
- let last_cell_on_child_page = leaf_contents.cell_get(
- leaf_cell_idx,
- payload_overflow_threshold_max(
- leaf_contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- leaf_contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )?;
+ let last_cell_on_child_page =
+ leaf_contents.cell_get(leaf_cell_idx, self.usable_space())?;
let mut cell_payload: Vec = Vec::new();
let child_pointer =
@@ -4435,7 +4230,7 @@ impl BTreeCursor {
BTreeCell::TableLeafCell(leaf_cell) => {
// Table interior cells contain the left child pointer and the rowid as varint.
cell_payload.extend_from_slice(&child_pointer.to_be_bytes());
- write_varint_to_vec(leaf_cell._rowid as u64, &mut cell_payload);
+ write_varint_to_vec(leaf_cell.rowid as u64, &mut cell_payload);
}
BTreeCell::IndexLeafCell(leaf_cell) => {
// Index interior cells contain:
@@ -4608,36 +4403,7 @@ impl BTreeCursor {
self.null_flag
}
- /// Search for a key in an Index Btree. Looking up indexes that need to be unique, we cannot compare the rowid
- pub fn key_exists_in_index(&mut self, key: &ImmutableRecord) -> Result> {
- return_if_io!(self.seek(SeekKey::IndexKey(key), SeekOp::GE { eq_only: true }));
-
- let record_opt = return_if_io!(self.record());
- match record_opt.as_ref() {
- Some(record) => {
- // Existing record found — compare prefix
- let existing_key = &record.get_values()[..record.count().saturating_sub(1)];
- let inserted_key_vals = &key.get_values();
- // Need this check because .all returns True on an empty iterator,
- // So when record_opt is invalidated, it would always indicate show up as a duplicate key
- if existing_key.len() != inserted_key_vals.len() {
- return Ok(CursorResult::Ok(false));
- }
-
- Ok(CursorResult::Ok(
- existing_key
- .iter()
- .zip(inserted_key_vals.iter())
- .all(|(a, b)| a == b),
- ))
- }
- None => {
- // Cursor not pointing at a record — table is empty or past last
- Ok(CursorResult::Ok(false))
- }
- }
- }
-
+ #[instrument(skip_all, level = Level::INFO)]
pub fn exists(&mut self, key: &Value) -> Result> {
assert!(self.mv_cursor.is_none());
let int_key = match key {
@@ -4654,6 +4420,7 @@ impl BTreeCursor {
/// Clear the overflow pages linked to a specific page provided by the leaf cell
/// Uses a state machine to keep track of it's operations so that traversal can be
/// resumed from last point after IO interruption
+ #[instrument(skip_all, level = Level::INFO)]
fn clear_overflow_pages(&mut self, cell: &BTreeCell) -> Result> {
loop {
let state = self.overflow_state.take().unwrap_or(OverflowState::Start);
@@ -4722,10 +4489,10 @@ impl BTreeCursor {
/// ```
///
/// The destruction order would be: [4',4,5,2,6,7,3,1]
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
pub fn btree_destroy(&mut self) -> Result>> {
if let CursorState::None = &self.state {
- self.move_to_root();
+ self.move_to_root()?;
self.state = CursorState::Destroy(DestroyInfo {
state: DestroyState::Start,
});
@@ -4808,18 +4575,7 @@ impl BTreeCursor {
// We have not yet processed all cells in this page
// Get the current cell
- let cell = contents.cell_get(
- cell_idx as usize,
- payload_overflow_threshold_max(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cell_idx as usize, self.usable_space())?;
match contents.is_leaf() {
// For a leaf cell, clear the overflow pages associated with this cell
@@ -4844,7 +4600,7 @@ impl BTreeCursor {
// For all other interior cells, load the left child page
_ => {
let child_page_id = match &cell {
- BTreeCell::TableInteriorCell(cell) => cell._left_child_page,
+ BTreeCell::TableInteriorCell(cell) => cell.left_child_page,
BTreeCell::IndexInteriorCell(cell) => cell.left_child_page,
_ => panic!("expected interior cell"),
};
@@ -4936,12 +4692,7 @@ impl BTreeCursor {
let (old_offset, old_local_size) = {
let page_ref = page_ref.get();
let page = page_ref.get().contents.as_ref().unwrap();
- page.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(page_type, self.usable_space() as u16),
- payload_overflow_threshold_min(page_type, self.usable_space() as u16),
- self.usable_space(),
- )
+ page.cell_get_raw_region(cell_idx, self.usable_space())
};
// if it all fits in local space and old_local_size is enough, do an in-place overwrite
@@ -4998,10 +4749,10 @@ impl BTreeCursor {
/// Count the number of entries in the b-tree
///
/// Only supposed to be used in the context of a simple Count Select Statement
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
pub fn count(&mut self) -> Result> {
if self.count == 0 {
- self.move_to_root();
+ self.move_to_root()?;
}
if let Some(_mv_cursor) = &self.mv_cursor {
@@ -5034,7 +4785,7 @@ impl BTreeCursor {
loop {
if !self.stack.has_parent() {
// All pages of the b-tree have been visited. Return successfully
- self.move_to_root();
+ self.move_to_root()?;
return Ok(CursorResult::Ok(self.count));
}
@@ -5069,23 +4820,11 @@ impl BTreeCursor {
self.stack.push(mem_page);
} else {
// Move to child left page
- let cell = contents.cell_get(
- cell_idx,
- payload_overflow_threshold_max(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- payload_overflow_threshold_min(
- contents.page_type(),
- self.usable_space() as u16,
- ),
- self.usable_space(),
- )?;
+ let cell = contents.cell_get(cell_idx, self.usable_space())?;
match cell {
BTreeCell::TableInteriorCell(TableInteriorCell {
- _left_child_page: left_child_page,
- ..
+ left_child_page, ..
})
| BTreeCell::IndexInteriorCell(IndexInteriorCell {
left_child_page, ..
@@ -5107,6 +4846,7 @@ impl BTreeCursor {
}
/// If context is defined, restore it and set it None on success
+ #[instrument(skip_all, level = Level::INFO)]
fn restore_context(&mut self) -> Result> {
if self.context.is_none() || !matches!(self.valid_state, CursorValidState::RequireSeek) {
return Ok(CursorResult::Ok(()));
@@ -5137,7 +4877,7 @@ impl BTreeCursor {
btree_read_page(&self.pager, page_idx)
}
- pub fn allocate_page(&self, page_type: PageType, offset: usize) -> BTreePage {
+ pub fn allocate_page(&self, page_type: PageType, offset: usize) -> Result {
self.pager
.do_allocate_page(page_type, offset, BtreePageAllocMode::Any)
}
@@ -5277,12 +5017,8 @@ pub fn integrity_check(
// have seen.
let mut next_rowid = max_intkey;
for cell_idx in (0..contents.cell_count()).rev() {
- let (cell_start, cell_length) = contents.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(contents.page_type(), usable_space),
- payload_overflow_threshold_min(contents.page_type(), usable_space),
- usable_space as usize,
- );
+ let (cell_start, cell_length) =
+ contents.cell_get_raw_region(cell_idx, usable_space as usize);
if cell_start < contents.cell_content_area() as usize
|| cell_start > usable_space as usize - 4
{
@@ -5306,20 +5042,15 @@ pub fn integrity_check(
});
}
coverage_checker.add_cell(cell_start, cell_start + cell_length);
- let cell = contents.cell_get(
- cell_idx,
- payload_overflow_threshold_max(contents.page_type(), usable_space),
- payload_overflow_threshold_min(contents.page_type(), usable_space),
- usable_space as usize,
- )?;
+ let cell = contents.cell_get(cell_idx, usable_space as usize)?;
match cell {
BTreeCell::TableInteriorCell(table_interior_cell) => {
state.page_stack.push(IntegrityCheckPageEntry {
- page_idx: table_interior_cell._left_child_page as usize,
+ page_idx: table_interior_cell.left_child_page as usize,
level: level + 1,
- max_intkey: table_interior_cell._rowid,
+ max_intkey: table_interior_cell.rowid,
});
- let rowid = table_interior_cell._rowid;
+ let rowid = table_interior_cell.rowid;
if rowid > max_intkey || rowid > next_rowid {
errors.push(IntegrityCheckError::CellRowidOutOfRange {
page_id: page.get().id,
@@ -5344,7 +5075,7 @@ pub fn integrity_check(
} else {
state.first_leaf_level = Some(level);
}
- let rowid = table_leaf_cell._rowid;
+ let rowid = table_leaf_cell.rowid;
if rowid > max_intkey || rowid > next_rowid {
errors.push(IntegrityCheckError::CellRowidOutOfRange {
page_id: page.get().id,
@@ -5545,7 +5276,7 @@ impl PageStack {
}
/// Push a new page onto the stack.
/// This effectively means traversing to a child page.
- #[instrument(skip_all, level = Level::TRACE, name = "pagestack::push")]
+ #[instrument(skip_all, level = Level::INFO, name = "pagestack::push")]
fn _push(&self, page: BTreePage, starting_cell_idx: i32) {
tracing::trace!(
current = self.current_page.get(),
@@ -5572,7 +5303,7 @@ impl PageStack {
/// Pop a page off the stack.
/// This effectively means traversing back up to a parent page.
- #[instrument(skip_all, level = Level::TRACE, name = "pagestack::pop")]
+ #[instrument(skip_all, level = Level::INFO, name = "pagestack::pop")]
fn pop(&self) {
let current = self.current_page.get();
assert!(current >= 0);
@@ -5584,7 +5315,7 @@ impl PageStack {
/// Get the top page on the stack.
/// This is the page that is currently being traversed.
- #[instrument(skip(self), level = Level::TRACE, name = "pagestack::top", )]
+ #[instrument(skip(self), level = Level::INFO, name = "pagestack::top", )]
fn top(&self) -> BTreePage {
let page = self.stack.borrow()[self.current()]
.as_ref()
@@ -5616,7 +5347,7 @@ impl PageStack {
/// Advance the current cell index of the current page to the next cell.
/// We usually advance after going traversing a new page
- #[instrument(skip(self), level = Level::TRACE, name = "pagestack::advance",)]
+ #[instrument(skip(self), level = Level::INFO, name = "pagestack::advance",)]
fn advance(&self) {
let current = self.current();
tracing::trace!(
@@ -5626,7 +5357,7 @@ impl PageStack {
self.cell_indices.borrow_mut()[current] += 1;
}
- #[instrument(skip(self), level = Level::TRACE, name = "pagestack::retreat")]
+ #[instrument(skip(self), level = Level::INFO, name = "pagestack::retreat")]
fn retreat(&self) {
let current = self.current();
tracing::trace!(
@@ -6068,8 +5799,8 @@ fn free_cell_range(
pc
};
- if offset <= page.cell_content_area() {
- if offset < page.cell_content_area() {
+ if (offset as u32) <= page.cell_content_area() {
+ if (offset as u32) < page.cell_content_area() {
return_corrupt!("Free block before content area");
}
if pointer_to_pc != page.offset as u16 + offset::BTREE_FIRST_FREEBLOCK as u16 {
@@ -6114,12 +5845,7 @@ fn defragment_page(page: &PageContent, usable_space: u16) {
assert!(pc <= last_cell);
- let (_, size) = cloned_page.cell_get_raw_region(
- i,
- payload_overflow_threshold_max(page.page_type(), usable_space),
- payload_overflow_threshold_min(page.page_type(), usable_space),
- usable_space as usize,
- );
+ let (_, size) = cloned_page.cell_get_raw_region(i, usable_space as usize);
let size = size as u16;
cbrk -= size;
if cbrk < first_cell || pc + size > usable_space {
@@ -6152,12 +5878,7 @@ fn defragment_page(page: &PageContent, usable_space: u16) {
/// Only enabled in debug mode, where we ensure that all cells are valid.
fn debug_validate_cells_core(page: &PageContent, usable_space: u16) {
for i in 0..page.cell_count() {
- let (offset, size) = page.cell_get_raw_region(
- i,
- payload_overflow_threshold_max(page.page_type(), usable_space),
- payload_overflow_threshold_min(page.page_type(), usable_space),
- usable_space as usize,
- );
+ let (offset, size) = page.cell_get_raw_region(i, usable_space as usize);
let buf = &page.as_ptr()[offset..offset + size];
// E.g. the following table btree cell may just have two bytes:
// Payload size 0 (stored as SerialTypeKind::ConstInt0)
@@ -6244,8 +5965,13 @@ fn insert_into_cell(
Ok(())
}
-/// Free blocks can be zero, meaning the "real free space" that can be used to allocate is expected to be between first cell byte
-/// and end of cell pointer area.
+/// The amount of free space is the sum of:
+/// #1. The size of the unallocated region
+/// #2. Fragments (isolated 1-3 byte chunks of free space within the cell content area)
+/// #3. freeblocks (linked list of blocks of at least 4 bytes within the cell content area that
+/// are not in use due to e.g. deletions)
+/// Free blocks can be zero, meaning the "real free space" that can be used to allocate is expected
+/// to be between first cell byte and end of cell pointer area.
#[allow(unused_assignments)]
fn compute_free_space(page: &PageContent, usable_space: u16) -> u16 {
// TODO(pere): maybe free space is not calculated correctly with offset
@@ -6254,38 +5980,14 @@ fn compute_free_space(page: &PageContent, usable_space: u16) -> u16 {
// space that is not reserved for extensions by sqlite. Usually reserved_space is 0.
let usable_space = usable_space as usize;
- let mut cell_content_area_start = page.cell_content_area();
- // A zero value for the cell content area pointer is interpreted as 65536.
- // See https://www.sqlite.org/fileformat.html
- // The max page size for a sqlite database is 64kiB i.e. 65536 bytes.
- // 65536 is u16::MAX + 1, and since cell content grows from right to left, this means
- // the cell content area pointer is at the end of the page,
- // i.e.
- // 1. the page size is 64kiB
- // 2. there are no cells on the page
- // 3. there is no reserved space at the end of the page
- if cell_content_area_start == 0 {
- cell_content_area_start = u16::MAX;
- }
-
- // The amount of free space is the sum of:
- // #1. the size of the unallocated region
- // #2. fragments (isolated 1-3 byte chunks of free space within the cell content area)
- // #3. freeblocks (linked list of blocks of at least 4 bytes within the cell content area that are not in use due to e.g. deletions)
-
- let pointer_size = if matches!(page.page_type(), PageType::TableLeaf | PageType::IndexLeaf) {
- 0
- } else {
- 4
- };
- let first_cell = page.offset + 8 + pointer_size + (2 * page.cell_count());
- let mut free_space_bytes =
- cell_content_area_start as usize + page.num_frag_free_bytes() as usize;
+ let first_cell = page.offset + page.header_size() + (2 * page.cell_count());
+ let cell_content_area_start = page.cell_content_area() as usize;
+ let mut free_space_bytes = cell_content_area_start + page.num_frag_free_bytes() as usize;
// #3 is computed by iterating over the freeblocks linked list
let mut cur_freeblock_ptr = page.first_freeblock() as usize;
if cur_freeblock_ptr > 0 {
- if cur_freeblock_ptr < cell_content_area_start as usize {
+ if cur_freeblock_ptr < cell_content_area_start {
// Freeblocks exist in the cell content area e.g. after deletions
// They should never exist in the unused area of the page.
todo!("corrupted page");
@@ -6299,7 +6001,7 @@ fn compute_free_space(page: &PageContent, usable_space: u16) -> u16 {
size = page.read_u16_no_offset(cur_freeblock_ptr + 2) as usize; // next 2 bytes in freeblock = size of current freeblock
free_space_bytes += size;
// Freeblocks are in order from left to right on the page,
- // so next pointer should > current pointer + its size, or 0 if no next block exists.
+ // so the next pointer should > current pointer + its size, or 0 if no next block exists.
if next <= cur_freeblock_ptr + size + 3 {
break;
}
@@ -6307,8 +6009,8 @@ fn compute_free_space(page: &PageContent, usable_space: u16) -> u16 {
}
// Next should always be 0 (NULL) at this point since we have reached the end of the freeblocks linked list
- assert!(
- next == 0,
+ assert_eq!(
+ next, 0,
"corrupted page: freeblocks list not in ascending order"
);
@@ -6323,16 +6025,15 @@ fn compute_free_space(page: &PageContent, usable_space: u16) -> u16 {
"corrupted page: free space is greater than usable space"
);
- // if( nFree>usableSize || nFree Result {
- let amount = amount as usize;
+ let mut amount = amount as usize;
+ if amount < MINIMUM_CELL_SIZE {
+ amount = MINIMUM_CELL_SIZE;
+ }
let (cell_offset, _) = page_ref.cell_pointer_array_offset_and_size();
let gap = cell_offset + 2 * page_ref.cell_count();
@@ -6417,7 +6118,6 @@ fn fill_cell_payload(
cell_payload.resize(prev_size + space_left + 4, 0);
let mut pointer = unsafe { cell_payload.as_mut_ptr().add(prev_size) };
let mut pointer_to_next = unsafe { cell_payload.as_mut_ptr().add(prev_size + space_left) };
- let mut overflow_pages = Vec::new();
loop {
let to_copy = space_left.min(to_copy_buffer.len());
@@ -6431,7 +6131,6 @@ fn fill_cell_payload(
// we still have bytes to add, we will need to allocate new overflow page
// FIXME: handle page cache is full
let overflow_page = pager.allocate_overflow_page();
- overflow_pages.push(overflow_page.clone());
{
let id = overflow_page.get().id as u32;
let contents = overflow_page.get().contents.as_mut().unwrap();
@@ -6463,7 +6162,7 @@ fn fill_cell_payload(
/// - Give a minimum fanout of 4 for index b-trees
/// - Ensure enough payload is on the b-tree page that the record header can usually be accessed
/// without consulting an overflow page
-fn payload_overflow_threshold_max(page_type: PageType, usable_space: u16) -> usize {
+pub fn payload_overflow_threshold_max(page_type: PageType, usable_space: u16) -> usize {
match page_type {
PageType::IndexInterior | PageType::IndexLeaf => {
((usable_space as usize - 12) * 64 / 255) - 23 // Index page formula
@@ -6483,7 +6182,7 @@ fn payload_overflow_threshold_max(page_type: PageType, usable_space: u16) -> usi
/// - Otherwise: store M bytes on page
///
/// The remaining bytes are stored on overflow pages in both cases.
-fn payload_overflow_threshold_min(_page_type: PageType, usable_space: u16) -> usize {
+pub fn payload_overflow_threshold_min(_page_type: PageType, usable_space: u16) -> usize {
// Same formula for all page types
((usable_space as usize - 12) * 32 / 255) - 23
}
@@ -6491,12 +6190,7 @@ fn payload_overflow_threshold_min(_page_type: PageType, usable_space: u16) -> us
/// Drop a cell from a page.
/// This is done by freeing the range of bytes that the cell occupies.
fn drop_cell(page: &mut PageContent, cell_idx: usize, usable_space: u16) -> Result<()> {
- let (cell_start, cell_len) = page.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(page.page_type(), usable_space),
- payload_overflow_threshold_min(page.page_type(), usable_space),
- usable_space as usize,
- );
+ let (cell_start, cell_len) = page.cell_get_raw_region(cell_idx, usable_space as usize);
free_cell_range(page, cell_start as u16, cell_len as u16, usable_space)?;
if page.cell_count() > 1 {
shift_pointers_left(page, cell_idx);
@@ -6555,10 +6249,7 @@ mod tests {
use crate::{
io::BufferData,
storage::{
- btree::{
- compute_free_space, fill_cell_payload, payload_overflow_threshold_max,
- payload_overflow_threshold_min,
- },
+ btree::{compute_free_space, fill_cell_payload, payload_overflow_threshold_max},
sqlite3_ondisk::{BTreeCell, PageContent, PageType},
},
types::Value,
@@ -6605,12 +6296,7 @@ mod tests {
}
fn ensure_cell(page: &mut PageContent, cell_idx: usize, payload: &Vec) {
- let cell = page.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(page.page_type(), 4096),
- payload_overflow_threshold_min(page.page_type(), 4096),
- 4096,
- );
+ let cell = page.cell_get_raw_region(cell_idx, 4096);
tracing::trace!("cell idx={} start={} len={}", cell_idx, cell.0, cell.1);
let buf = &page.as_ptr()[cell.0..cell.0 + cell.1];
assert_eq!(buf.len(), payload.len());
@@ -6706,41 +6392,33 @@ mod tests {
// Pin page in order to not drop it in between
page.set_dirty();
let contents = page.get().contents.as_ref().unwrap();
- let page_type = contents.page_type();
let mut previous_key = None;
let mut valid = true;
let mut depth = None;
debug_validate_cells!(contents, pager.usable_space() as u16);
let mut child_pages = Vec::new();
for cell_idx in 0..contents.cell_count() {
- let cell = contents
- .cell_get(
- cell_idx,
- payload_overflow_threshold_max(page_type, 4096),
- payload_overflow_threshold_min(page_type, 4096),
- cursor.usable_space(),
- )
- .unwrap();
+ let cell = contents.cell_get(cell_idx, cursor.usable_space()).unwrap();
let current_depth = match cell {
BTreeCell::TableLeafCell(..) => 1,
BTreeCell::TableInteriorCell(TableInteriorCell {
- _left_child_page, ..
+ left_child_page, ..
}) => {
- let child_page = cursor.read_page(_left_child_page as usize).unwrap();
+ let child_page = cursor.read_page(left_child_page as usize).unwrap();
while child_page.get().is_locked() {
pager.io.run_once().unwrap();
}
child_pages.push(child_page);
- if _left_child_page == page.get().id as u32 {
+ if left_child_page == page.get().id as u32 {
valid = false;
tracing::error!(
"left child page is the same as parent {}",
- _left_child_page
+ left_child_page
);
continue;
}
let (child_depth, child_valid) =
- validate_btree(pager.clone(), _left_child_page as usize);
+ validate_btree(pager.clone(), left_child_page as usize);
valid &= child_valid;
child_depth
}
@@ -6757,17 +6435,17 @@ mod tests {
valid = false;
}
match cell {
- BTreeCell::TableInteriorCell(TableInteriorCell { _rowid, .. })
- | BTreeCell::TableLeafCell(TableLeafCell { _rowid, .. }) => {
- if previous_key.is_some() && previous_key.unwrap() >= _rowid {
+ BTreeCell::TableInteriorCell(TableInteriorCell { rowid, .. })
+ | BTreeCell::TableLeafCell(TableLeafCell { rowid, .. }) => {
+ if previous_key.is_some() && previous_key.unwrap() >= rowid {
tracing::error!(
"keys are in bad order: prev={:?}, current={}",
previous_key,
- _rowid
+ rowid
);
valid = false;
}
- previous_key = Some(_rowid);
+ previous_key = Some(rowid);
}
_ => panic!("unsupported btree cell: {:?}", cell),
}
@@ -6823,35 +6501,27 @@ mod tests {
// Pin page in order to not drop it in between loading of different pages. If not contents will be a dangling reference.
page.set_dirty();
let contents = page.get().contents.as_ref().unwrap();
- let page_type = contents.page_type();
let mut current = Vec::new();
let mut child = Vec::new();
for cell_idx in 0..contents.cell_count() {
- let cell = contents
- .cell_get(
- cell_idx,
- payload_overflow_threshold_max(page_type, 4096),
- payload_overflow_threshold_min(page_type, 4096),
- cursor.usable_space(),
- )
- .unwrap();
+ let cell = contents.cell_get(cell_idx, cursor.usable_space()).unwrap();
match cell {
BTreeCell::TableInteriorCell(cell) => {
current.push(format!(
"node[rowid:{}, ptr(<=):{}]",
- cell._rowid, cell._left_child_page
+ cell.rowid, cell.left_child_page
));
child.push(format_btree(
pager.clone(),
- cell._left_child_page as usize,
+ cell.left_child_page as usize,
depth + 2,
));
}
BTreeCell::TableLeafCell(cell) => {
current.push(format!(
"leaf[rowid:{}, len(payload):{}, overflow:{}]",
- cell._rowid,
- cell._payload.len(),
+ cell.rowid,
+ cell.payload.len(),
cell.first_overflow_page.is_some()
));
}
@@ -7066,10 +6736,10 @@ mod tests {
}
run_until_done(|| pager.begin_read_tx(), &pager).unwrap();
// FIXME: add sorted vector instead, should be okay for small amounts of keys for now :P, too lazy to fix right now
- cursor.move_to_root();
+ cursor.move_to_root().unwrap();
let mut valid = true;
if do_validate {
- cursor.move_to_root();
+ cursor.move_to_root().unwrap();
for key in keys.iter() {
tracing::trace!("seeking key: {}", key);
run_until_done(|| cursor.next(), pager.deref()).unwrap();
@@ -7102,7 +6772,7 @@ mod tests {
if matches!(validate_btree(pager.clone(), root_page), (_, false)) {
panic!("invalid btree");
}
- cursor.move_to_root();
+ cursor.move_to_root().unwrap();
for key in keys.iter() {
tracing::trace!("seeking key: {}", key);
run_until_done(|| cursor.next(), pager.deref()).unwrap();
@@ -7181,7 +6851,7 @@ mod tests {
pager.deref(),
)
.unwrap();
- cursor.move_to_root();
+ cursor.move_to_root().unwrap();
loop {
match pager.end_tx(false, false, &conn, false).unwrap() {
crate::PagerCacheflushStatus::Done(_) => break,
@@ -7194,7 +6864,7 @@ mod tests {
// Check that all keys can be found by seeking
pager.begin_read_tx().unwrap();
- cursor.move_to_root();
+ cursor.move_to_root().unwrap();
for (i, key) in keys.iter().enumerate() {
tracing::info!("seeking key {}/{}: {:?}", i + 1, keys.len(), key);
let exists = run_until_done(
@@ -7214,7 +6884,7 @@ mod tests {
assert!(exists, "key {:?} is not found", key);
}
// Check that key count is right
- cursor.move_to_root();
+ cursor.move_to_root().unwrap();
let mut count = 0;
while run_until_done(|| cursor.next(), pager.deref()).unwrap() {
count += 1;
@@ -7227,7 +6897,7 @@ mod tests {
keys.len()
);
// Check that all keys can be found in-order, by iterating the btree
- cursor.move_to_root();
+ cursor.move_to_root().unwrap();
let mut prev = None;
for (i, key) in keys.iter().enumerate() {
tracing::info!("iterating key {}/{}: {:?}", i + 1, keys.len(), key);
@@ -7474,8 +7144,8 @@ mod tests {
// Create leaf cell pointing to start of overflow chain
let leaf_cell = BTreeCell::TableLeafCell(TableLeafCell {
- _rowid: 1,
- _payload: unsafe { transmute::<&[u8], &'static [u8]>(large_payload.as_slice()) },
+ rowid: 1,
+ payload: unsafe { transmute::<&[u8], &'static [u8]>(large_payload.as_slice()) },
first_overflow_page: Some(2), // Point to first overflow page
payload_size: large_payload.len() as u64,
});
@@ -7530,8 +7200,8 @@ mod tests {
// Create leaf cell with no overflow pages
let leaf_cell = BTreeCell::TableLeafCell(TableLeafCell {
- _rowid: 1,
- _payload: unsafe { transmute::<&[u8], &'static [u8]>(small_payload.as_slice()) },
+ rowid: 1,
+ payload: unsafe { transmute::<&[u8], &'static [u8]>(small_payload.as_slice()) },
first_overflow_page: None,
payload_size: small_payload.len() as u64,
});
@@ -7571,11 +7241,11 @@ mod tests {
let mut cursor = BTreeCursor::new_table(None, pager.clone(), 2);
// Initialize page 2 as a root page (interior)
- let root_page = cursor.allocate_page(PageType::TableInterior, 0);
+ let root_page = cursor.allocate_page(PageType::TableInterior, 0)?;
// Allocate two leaf pages
- let page3 = cursor.allocate_page(PageType::TableLeaf, 0);
- let page4 = cursor.allocate_page(PageType::TableLeaf, 0);
+ let page3 = cursor.allocate_page(PageType::TableLeaf, 0)?;
+ let page4 = cursor.allocate_page(PageType::TableLeaf, 0)?;
// Configure the root page to point to the two leaf pages
{
@@ -7775,12 +7445,7 @@ mod tests {
continue;
}
let cell_idx = rng.next_u64() as usize % page.cell_count();
- let (_, len) = page.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(page.page_type(), 4096),
- payload_overflow_threshold_min(page.page_type(), 4096),
- usable_space as usize,
- );
+ let (_, len) = page.cell_get_raw_region(cell_idx, usable_space as usize);
drop_cell(page, cell_idx, usable_space).unwrap();
total_size -= len as u16 + 2;
cells.remove(cell_idx);
@@ -7856,12 +7521,7 @@ mod tests {
continue;
}
let cell_idx = rng.next_u64() as usize % page.cell_count();
- let (_, len) = page.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(page.page_type(), 4096),
- payload_overflow_threshold_min(page.page_type(), 4096),
- usable_space as usize,
- );
+ let (_, len) = page.cell_get_raw_region(cell_idx, usable_space as usize);
drop_cell(page, cell_idx, usable_space).unwrap();
total_size -= len as u16 + 2;
cells.remove(cell_idx);
@@ -8011,12 +7671,7 @@ mod tests {
assert_eq!(page.cell_count(), 1);
defragment_page(page, usable_space);
assert_eq!(page.cell_count(), 1);
- let (start, len) = page.cell_get_raw_region(
- 0,
- payload_overflow_threshold_max(page.page_type(), 4096),
- payload_overflow_threshold_min(page.page_type(), 4096),
- usable_space as usize,
- );
+ let (start, len) = page.cell_get_raw_region(0, usable_space as usize);
let buf = page.as_ptr();
assert_eq!(&payload, &buf[start..start + len]);
}
@@ -8047,12 +7702,7 @@ mod tests {
let payload = add_record(0, 0, page, record, &conn);
assert_eq!(page.cell_count(), 1);
- let (start, len) = page.cell_get_raw_region(
- 0,
- payload_overflow_threshold_max(page.page_type(), 4096),
- payload_overflow_threshold_min(page.page_type(), 4096),
- usable_space as usize,
- );
+ let (start, len) = page.cell_get_raw_region(0, usable_space as usize);
let buf = page.as_ptr();
assert_eq!(&payload, &buf[start..start + len]);
}
@@ -8084,12 +7734,7 @@ mod tests {
let payload = add_record(0, 0, page, record, &conn);
assert_eq!(page.cell_count(), 1);
- let (start, len) = page.cell_get_raw_region(
- 0,
- payload_overflow_threshold_max(page.page_type(), 4096),
- payload_overflow_threshold_min(page.page_type(), 4096),
- usable_space as usize,
- );
+ let (start, len) = page.cell_get_raw_region(0, usable_space as usize);
let buf = page.as_ptr();
assert_eq!(&payload, &buf[start..start + len]);
}
@@ -8429,7 +8074,7 @@ mod tests {
);
}
let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page);
- cursor.move_to_root();
+ cursor.move_to_root().unwrap();
for i in 0..iterations {
let has_next = run_until_done(|| cursor.next(), pager.deref()).unwrap();
if !has_next {
@@ -8650,12 +8295,7 @@ mod tests {
let contents = page.get_contents();
for cell_idx in 0..contents.cell_count() {
let buf = contents.as_ptr();
- let (start, len) = contents.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(contents.page_type(), 4096),
- payload_overflow_threshold_min(contents.page_type(), 4096),
- pager.usable_space(),
- );
+ let (start, len) = contents.cell_get_raw_region(cell_idx, pager.usable_space());
cell_array
.cells
.push(to_static_buf(&mut buf[start..start + len]));
@@ -8694,12 +8334,7 @@ mod tests {
let mut cell_idx_cloned = if prefix { size } else { 0 };
for cell_idx in 0..contents.cell_count() {
let buf = contents.as_ptr();
- let (start, len) = contents.cell_get_raw_region(
- cell_idx,
- payload_overflow_threshold_max(contents.page_type(), 4096),
- payload_overflow_threshold_min(contents.page_type(), 4096),
- pager.usable_space(),
- );
+ let (start, len) = contents.cell_get_raw_region(cell_idx, pager.usable_space());
let cell_in_page = &buf[start..start + len];
let cell_in_array = &cells_cloned[cell_idx_cloned];
assert_eq!(cell_in_page, cell_in_array);
diff --git a/core/storage/database.rs b/core/storage/database.rs
index dd5c9263f..c2ad2c57a 100644
--- a/core/storage/database.rs
+++ b/core/storage/database.rs
@@ -2,6 +2,7 @@ use crate::error::LimboError;
use crate::io::CompletionType;
use crate::{io::Completion, Buffer, Result};
use std::{cell::RefCell, sync::Arc};
+use tracing::{instrument, Level};
/// DatabaseStorage is an interface a database file that consists of pages.
///
@@ -32,6 +33,7 @@ unsafe impl Sync for DatabaseFile {}
#[cfg(feature = "fs")]
impl DatabaseStorage for DatabaseFile {
+ #[instrument(skip_all, level = Level::INFO)]
fn read_page(&self, page_idx: usize, c: Completion) -> Result<()> {
let r = c.as_read();
let size = r.buf().len();
@@ -44,6 +46,7 @@ impl DatabaseStorage for DatabaseFile {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn write_page(
&self,
page_idx: usize,
@@ -60,11 +63,13 @@ impl DatabaseStorage for DatabaseFile {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn sync(&self, c: Completion) -> Result<()> {
let _ = self.file.sync(c)?;
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn size(&self) -> Result {
self.file.size()
}
@@ -85,6 +90,7 @@ unsafe impl Send for FileMemoryStorage {}
unsafe impl Sync for FileMemoryStorage {}
impl DatabaseStorage for FileMemoryStorage {
+ #[instrument(skip_all, level = Level::INFO)]
fn read_page(&self, page_idx: usize, c: Completion) -> Result<()> {
let r = match c.completion_type {
CompletionType::Read(ref r) => r,
@@ -100,6 +106,7 @@ impl DatabaseStorage for FileMemoryStorage {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn write_page(
&self,
page_idx: usize,
@@ -115,11 +122,13 @@ impl DatabaseStorage for FileMemoryStorage {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn sync(&self, c: Completion) -> Result<()> {
let _ = self.file.sync(c)?;
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn size(&self) -> Result {
self.file.size()
}
diff --git a/core/storage/pager.rs b/core/storage/pager.rs
index 008c154e9..1a4e3f755 100644
--- a/core/storage/pager.rs
+++ b/core/storage/pager.rs
@@ -14,7 +14,7 @@ use std::collections::HashSet;
use std::rc::Rc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
-use tracing::{trace, Level};
+use tracing::{instrument, trace, Level};
use super::btree::{btree_init_page, BTreePage};
use super::page_cache::{CacheError, CacheResizeResult, DumbLruPageCache, PageCacheKey};
@@ -471,6 +471,7 @@ impl Pager {
/// This method is used to allocate a new root page for a btree, both for tables and indexes
/// FIXME: handle no room in page cache
+ #[instrument(skip_all, level = Level::INFO)]
pub fn btree_create(&self, flags: &CreateBTreeFlags) -> Result> {
let page_type = match flags {
_ if flags.is_table() => PageType::TableLeaf,
@@ -479,7 +480,7 @@ impl Pager {
};
#[cfg(feature = "omit_autovacuum")]
{
- let page = self.do_allocate_page(page_type, 0, BtreePageAllocMode::Any);
+ let page = self.do_allocate_page(page_type, 0, BtreePageAllocMode::Any)?;
let page_id = page.get().get().id;
Ok(CursorResult::Ok(page_id as u32))
}
@@ -490,7 +491,7 @@ impl Pager {
let auto_vacuum_mode = self.auto_vacuum_mode.borrow();
match *auto_vacuum_mode {
AutoVacuumMode::None => {
- let page = self.do_allocate_page(page_type, 0, BtreePageAllocMode::Any);
+ let page = self.do_allocate_page(page_type, 0, BtreePageAllocMode::Any)?;
let page_id = page.get().get().id;
Ok(CursorResult::Ok(page_id as u32))
}
@@ -514,7 +515,7 @@ impl Pager {
page_type,
0,
BtreePageAllocMode::Exact(root_page_num),
- );
+ )?;
let allocated_page_id = page.get().get().id as u32;
if allocated_page_id != root_page_num {
// TODO(Zaid): Handle swapping the allocated page with the desired root page
@@ -558,8 +559,8 @@ impl Pager {
page_type: PageType,
offset: usize,
_alloc_mode: BtreePageAllocMode,
- ) -> BTreePage {
- let page = self.allocate_page().unwrap();
+ ) -> Result {
+ let page = self.allocate_page()?;
let page = Arc::new(BTreePageInner {
page: RefCell::new(page),
});
@@ -569,7 +570,7 @@ impl Pager {
page.get().get().id,
page.get().get_contents().page_type()
);
- page
+ Ok(page)
}
/// The "usable size" of a database page is the page size specified by the 2-byte integer at offset 16
@@ -589,6 +590,7 @@ impl Pager {
}
#[inline(always)]
+ #[instrument(skip_all, level = Level::INFO)]
pub fn begin_read_tx(&self) -> Result> {
// We allocate the first page lazily in the first transaction
match self.maybe_allocate_page1()? {
@@ -598,6 +600,7 @@ impl Pager {
Ok(CursorResult::Ok(self.wal.borrow_mut().begin_read_tx()?))
}
+ #[instrument(skip_all, level = Level::INFO)]
fn maybe_allocate_page1(&self) -> Result> {
if self.is_empty.load(Ordering::SeqCst) < DB_STATE_INITIALIZED {
if let Ok(_lock) = self.init_lock.try_lock() {
@@ -621,6 +624,7 @@ impl Pager {
}
#[inline(always)]
+ #[instrument(skip_all, level = Level::INFO)]
pub fn begin_write_tx(&self) -> Result> {
// TODO(Diego): The only possibly allocate page1 here is because OpenEphemeral needs a write transaction
// we should have a unique API to begin transactions, something like sqlite3BtreeBeginTrans
@@ -631,10 +635,11 @@ impl Pager {
Ok(CursorResult::Ok(self.wal.borrow_mut().begin_write_tx()?))
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn end_tx(
&self,
rollback: bool,
- change_schema: bool,
+ schema_did_change: bool,
connection: &Connection,
wal_checkpoint_disabled: bool,
) -> Result {
@@ -648,7 +653,7 @@ impl Pager {
match cacheflush_status {
PagerCacheflushStatus::IO => Ok(PagerCacheflushStatus::IO),
PagerCacheflushStatus::Done(_) => {
- let maybe_schema_pair = if change_schema {
+ let maybe_schema_pair = if schema_did_change {
let schema = connection.schema.borrow().clone();
// Lock first before writing to the database schema in case someone tries to read the schema before it's updated
let db_schema = connection._db.schema.write();
@@ -666,13 +671,14 @@ impl Pager {
}
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn end_read_tx(&self) -> Result<()> {
self.wal.borrow().end_read_tx()?;
Ok(())
}
/// Reads a page from the database.
- #[tracing::instrument(skip_all, level = Level::DEBUG)]
+ #[tracing::instrument(skip_all, level = Level::INFO)]
pub fn read_page(&self, page_idx: usize) -> Result {
tracing::trace!("read_page(page_idx = {})", page_idx);
let mut page_cache = self.page_cache.write();
@@ -759,11 +765,12 @@ impl Pager {
/// In the base case, it will write the dirty pages to the WAL and then fsync the WAL.
/// If the WAL size is over the checkpoint threshold, it will checkpoint the WAL to
/// the database file and then fsync the database file.
+ #[instrument(skip_all, level = Level::INFO)]
pub fn cacheflush(&self, wal_checkpoint_disabled: bool) -> Result {
let mut checkpoint_result = CheckpointResult::default();
- loop {
+ let res = loop {
let state = self.flush_info.borrow().state;
- trace!("cacheflush {:?}", state);
+ trace!(?state);
match state {
FlushState::Start => {
let db_size = header_accessor::get_database_size(self)?;
@@ -795,7 +802,6 @@ impl Pager {
let in_flight = *self.flush_info.borrow().in_flight_writes.borrow();
if in_flight == 0 {
self.flush_info.borrow_mut().state = FlushState::SyncWal;
- self.wal.borrow_mut().finish_append_frames_commit()?;
} else {
return Ok(PagerCacheflushStatus::IO);
}
@@ -807,9 +813,7 @@ impl Pager {
if wal_checkpoint_disabled || !self.wal.borrow().should_checkpoint() {
self.flush_info.borrow_mut().state = FlushState::Start;
- return Ok(PagerCacheflushStatus::Done(
- PagerCacheflushResult::WalWritten,
- ));
+ break PagerCacheflushResult::WalWritten;
}
self.flush_info.borrow_mut().state = FlushState::Checkpoint;
}
@@ -831,16 +835,17 @@ impl Pager {
return Ok(PagerCacheflushStatus::IO);
} else {
self.flush_info.borrow_mut().state = FlushState::Start;
- break;
+ break PagerCacheflushResult::Checkpointed(checkpoint_result);
}
}
}
- }
- Ok(PagerCacheflushStatus::Done(
- PagerCacheflushResult::Checkpointed(checkpoint_result),
- ))
+ };
+ // We should only signal that we finished appenind frames after wal sync to avoid inconsistencies when sync fails
+ self.wal.borrow_mut().finish_append_frames_commit()?;
+ Ok(PagerCacheflushStatus::Done(res))
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn wal_get_frame(
&self,
frame_no: u32,
@@ -856,11 +861,12 @@ impl Pager {
)
}
+ #[instrument(skip_all, level = Level::INFO, target = "pager_checkpoint",)]
pub fn checkpoint(&self) -> Result {
let mut checkpoint_result = CheckpointResult::default();
loop {
let state = *self.checkpoint_state.borrow();
- trace!("pager_checkpoint(state={:?})", state);
+ trace!(?state);
match state {
CheckpointState::Checkpoint => {
let in_flight = self.checkpoint_inflight.clone();
@@ -932,6 +938,7 @@ impl Pager {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn wal_checkpoint(&self, wal_checkpoint_disabled: bool) -> Result {
if wal_checkpoint_disabled {
return Ok(CheckpointResult {
@@ -947,7 +954,7 @@ impl Pager {
CheckpointMode::Passive,
) {
Ok(CheckpointStatus::IO) => {
- let _ = self.io.run_once();
+ self.io.run_once()?;
}
Ok(CheckpointStatus::Done(res)) => {
checkpoint_result = res;
@@ -965,6 +972,7 @@ impl Pager {
// Providing a page is optional, if provided it will be used to avoid reading the page from disk.
// This is implemented in accordance with sqlite freepage2() function.
+ #[instrument(skip_all, level = Level::INFO)]
pub fn free_page(&self, page: Option, page_id: usize) -> Result<()> {
tracing::trace!("free_page(page_id={})", page_id);
const TRUNK_PAGE_HEADER_SIZE: usize = 8;
@@ -1036,6 +1044,7 @@ impl Pager {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
pub fn allocate_page1(&self) -> Result> {
let state = self.allocate_page1_state.borrow().clone();
match state {
@@ -1111,6 +1120,7 @@ impl Pager {
*/
// FIXME: handle no room in page cache
#[allow(clippy::readonly_write_lock)]
+ #[instrument(skip_all, level = Level::INFO)]
pub fn allocate_page(&self) -> Result {
let old_db_size = header_accessor::get_database_size(self)?;
#[allow(unused_mut)]
@@ -1195,12 +1205,18 @@ impl Pager {
(page_size - reserved_space) as usize
}
- pub fn rollback(&self, change_schema: bool, connection: &Connection) -> Result<(), LimboError> {
+ #[instrument(skip_all, level = Level::INFO)]
+ pub fn rollback(
+ &self,
+ schema_did_change: bool,
+ connection: &Connection,
+ ) -> Result<(), LimboError> {
+ tracing::debug!(schema_did_change);
self.dirty_pages.borrow_mut().clear();
let mut cache = self.page_cache.write();
cache.unset_dirty_all_pages();
cache.clear().expect("failed to clear page cache");
- if change_schema {
+ if schema_did_change {
let prev_schema = connection._db.schema.read().clone();
connection.schema.replace(prev_schema);
}
diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs
index 1985b4e92..4c694c56c 100644
--- a/core/storage/sqlite3_ondisk.rs
+++ b/core/storage/sqlite3_ondisk.rs
@@ -45,11 +45,18 @@
use tracing::{instrument, Level};
+use super::pager::PageRef;
+use super::wal::LimboRwLock;
use crate::error::LimboError;
use crate::fast_lock::SpinLock;
use crate::io::{
Buffer, Complete, Completion, CompletionType, ReadCompletion, SyncCompletion, WriteCompletion,
};
+use crate::storage::btree::offset::{
+ BTREE_CELL_CONTENT_AREA, BTREE_CELL_COUNT, BTREE_FIRST_FREEBLOCK, BTREE_FRAGMENTED_BYTES_COUNT,
+ BTREE_PAGE_TYPE, BTREE_RIGHTMOST_PTR,
+};
+use crate::storage::btree::{payload_overflow_threshold_max, payload_overflow_threshold_min};
use crate::storage::buffer_pool::BufferPool;
use crate::storage::database::DatabaseStorage;
use crate::storage::pager::Pager;
@@ -65,9 +72,6 @@ use std::rc::Rc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
-use super::pager::PageRef;
-use super::wal::LimboRwLock;
-
/// The size of the database header in bytes.
pub const DATABASE_HEADER_SIZE: usize = 100;
// DEFAULT_CACHE_SIZE negative values mean that we store the amount of pages a XKiB of memory can hold.
@@ -88,6 +92,9 @@ pub const DEFAULT_PAGE_SIZE: u16 = 4096;
pub const DATABASE_HEADER_PAGE_ID: usize = 1;
+/// The minimum size of a cell in bytes.
+pub const MINIMUM_CELL_SIZE: usize = 4;
+
/// The database header.
/// The first 100 bytes of the database file comprise the database file header.
/// The database file header is divided into fields as shown by the table below.
@@ -357,6 +364,8 @@ pub struct OverflowCell {
#[derive(Debug)]
pub struct PageContent {
+ /// the position where page content starts. it's 100 for page 1(database file header is 100 bytes),
+ /// 0 for all other pages.
pub offset: usize,
pub buffer: Arc>,
pub overflow_cells: Vec,
@@ -373,6 +382,7 @@ impl Clone for PageContent {
}
}
+const CELL_POINTER_SIZE_BYTES: usize = 2;
impl PageContent {
pub fn new(offset: usize, buffer: Arc>) -> Self {
Self {
@@ -383,7 +393,7 @@ impl PageContent {
}
pub fn page_type(&self) -> PageType {
- self.read_u8(0).try_into().unwrap()
+ self.read_u8(BTREE_PAGE_TYPE).try_into().unwrap()
}
pub fn maybe_page_type(&self) -> Option {
@@ -452,19 +462,14 @@ impl PageContent {
buf[self.offset + pos..self.offset + pos + 4].copy_from_slice(&value.to_be_bytes());
}
- /// The second field of the b-tree page header is the offset of the first freeblock, or zero if there are no freeblocks on the page.
- /// A freeblock is a structure used to identify unallocated space within a b-tree page.
- /// Freeblocks are organized as a chain.
- ///
- /// To be clear, freeblocks do not mean the regular unallocated free space to the left of the cell content area pointer, but instead
- /// blocks of at least 4 bytes WITHIN the cell content area that are not in use due to e.g. deletions.
+ /// The offset of the first freeblock, or zero if there are no freeblocks on the page.
pub fn first_freeblock(&self) -> u16 {
- self.read_u16(1)
+ self.read_u16(BTREE_FIRST_FREEBLOCK)
}
/// The number of cells on the page.
pub fn cell_count(&self) -> usize {
- self.read_u16(3) as usize
+ self.read_u16(BTREE_CELL_COUNT) as usize
}
/// The size of the cell pointer array in bytes.
@@ -486,11 +491,13 @@ impl PageContent {
}
/// The start of the cell content area.
- /// SQLite strives to place cells as far toward the end of the b-tree page as it can,
- /// in order to leave space for future growth of the cell pointer array.
- /// = the cell content area pointer moves leftward as cells are added to the page
- pub fn cell_content_area(&self) -> u16 {
- self.read_u16(5)
+ pub fn cell_content_area(&self) -> u32 {
+ let offset = self.read_u16(BTREE_CELL_CONTENT_AREA);
+ if offset == 0 {
+ MAX_PAGE_SIZE
+ } else {
+ offset as u32
+ }
}
/// The size of the page header in bytes.
@@ -504,16 +511,15 @@ impl PageContent {
}
}
- /// The total number of bytes in all fragments is stored in the fifth field of the b-tree page header.
- /// Fragments are isolated groups of 1, 2, or 3 unused bytes within the cell content area.
+ /// The total number of bytes in all fragments
pub fn num_frag_free_bytes(&self) -> u8 {
- self.read_u8(7)
+ self.read_u8(BTREE_FRAGMENTED_BYTES_COUNT)
}
pub fn rightmost_pointer(&self) -> Option {
match self.page_type() {
- PageType::IndexInterior => Some(self.read_u32(8)),
- PageType::TableInterior => Some(self.read_u32(8)),
+ PageType::IndexInterior => Some(self.read_u32(BTREE_RIGHTMOST_PTR)),
+ PageType::TableInterior => Some(self.read_u32(BTREE_RIGHTMOST_PTR)),
PageType::IndexLeaf => None,
PageType::TableLeaf => None,
}
@@ -521,48 +527,35 @@ impl PageContent {
pub fn rightmost_pointer_raw(&self) -> Option<*mut u8> {
match self.page_type() {
- PageType::IndexInterior | PageType::TableInterior => {
- Some(unsafe { self.as_ptr().as_mut_ptr().add(self.offset + 8) })
- }
+ PageType::IndexInterior | PageType::TableInterior => Some(unsafe {
+ self.as_ptr()
+ .as_mut_ptr()
+ .add(self.offset + BTREE_RIGHTMOST_PTR)
+ }),
PageType::IndexLeaf => None,
PageType::TableLeaf => None,
}
}
- pub fn cell_get(
- &self,
- idx: usize,
- payload_overflow_threshold_max: usize,
- payload_overflow_threshold_min: usize,
- usable_size: usize,
- ) -> Result {
+ pub fn cell_get(&self, idx: usize, usable_size: usize) -> Result {
tracing::trace!("cell_get(idx={})", idx);
let buf = self.as_ptr();
let ncells = self.cell_count();
- // the page header is 12 bytes for interior pages, 8 bytes for leaf pages
- // this is because the 4 last bytes in the interior page's header are used for the rightmost pointer.
- let cell_pointer_array_start = self.header_size();
assert!(
idx < ncells,
"cell_get: idx out of bounds: idx={}, ncells={}",
idx,
ncells
);
- let cell_pointer = cell_pointer_array_start + (idx * 2);
+ let cell_pointer_array_start = self.header_size();
+ let cell_pointer = cell_pointer_array_start + (idx * CELL_POINTER_SIZE_BYTES);
let cell_pointer = self.read_u16(cell_pointer) as usize;
// SAFETY: this buffer is valid as long as the page is alive. We could store the page in the cell and do some lifetime magic
// but that is extra memory for no reason at all. Just be careful like in the old times :).
let static_buf: &'static [u8] = unsafe { std::mem::transmute::<&[u8], &'static [u8]>(buf) };
- read_btree_cell(
- static_buf,
- &self.page_type(),
- cell_pointer,
- payload_overflow_threshold_max,
- payload_overflow_threshold_min,
- usable_size,
- )
+ read_btree_cell(static_buf, self, cell_pointer, usable_size)
}
/// Read the rowid of a table interior cell.
@@ -570,30 +563,31 @@ impl PageContent {
pub fn cell_table_interior_read_rowid(&self, idx: usize) -> Result {
debug_assert!(self.page_type() == PageType::TableInterior);
let buf = self.as_ptr();
- const INTERIOR_PAGE_HEADER_SIZE_BYTES: usize = 12;
- let cell_pointer_array_start = INTERIOR_PAGE_HEADER_SIZE_BYTES;
- let cell_pointer = cell_pointer_array_start + (idx * 2);
+ let cell_pointer_array_start = self.header_size();
+ let cell_pointer = cell_pointer_array_start + (idx * CELL_POINTER_SIZE_BYTES);
let cell_pointer = self.read_u16(cell_pointer) as usize;
const LEFT_CHILD_PAGE_SIZE_BYTES: usize = 4;
let (rowid, _) = read_varint(&buf[cell_pointer + LEFT_CHILD_PAGE_SIZE_BYTES..])?;
Ok(rowid as i64)
}
- /// Read the left child page of a table interior cell.
+ /// Read the left child page of a table interior cell or an index interior cell.
#[inline(always)]
- pub fn cell_table_interior_read_left_child_page(&self, idx: usize) -> Result {
- debug_assert!(self.page_type() == PageType::TableInterior);
+ pub fn cell_interior_read_left_child_page(&self, idx: usize) -> u32 {
+ debug_assert!(
+ self.page_type() == PageType::TableInterior
+ || self.page_type() == PageType::IndexInterior
+ );
let buf = self.as_ptr();
- const INTERIOR_PAGE_HEADER_SIZE_BYTES: usize = 12;
- let cell_pointer_array_start = INTERIOR_PAGE_HEADER_SIZE_BYTES;
- let cell_pointer = cell_pointer_array_start + (idx * 2);
+ let cell_pointer_array_start = self.header_size();
+ let cell_pointer = cell_pointer_array_start + (idx * CELL_POINTER_SIZE_BYTES);
let cell_pointer = self.read_u16(cell_pointer) as usize;
- Ok(u32::from_be_bytes([
+ u32::from_be_bytes([
buf[cell_pointer],
buf[cell_pointer + 1],
buf[cell_pointer + 2],
buf[cell_pointer + 3],
- ]))
+ ])
}
/// Read the rowid of a table leaf cell.
@@ -601,9 +595,8 @@ impl PageContent {
pub fn cell_table_leaf_read_rowid(&self, idx: usize) -> Result {
debug_assert!(self.page_type() == PageType::TableLeaf);
let buf = self.as_ptr();
- const LEAF_PAGE_HEADER_SIZE_BYTES: usize = 8;
- let cell_pointer_array_start = LEAF_PAGE_HEADER_SIZE_BYTES;
- let cell_pointer = cell_pointer_array_start + (idx * 2);
+ let cell_pointer_array_start = self.header_size();
+ let cell_pointer = cell_pointer_array_start + (idx * CELL_POINTER_SIZE_BYTES);
let cell_pointer = self.read_u16(cell_pointer) as usize;
let mut pos = cell_pointer;
let (_, nr) = read_varint(&buf[pos..])?;
@@ -623,21 +616,19 @@ impl PageContent {
(self.offset + header_size, self.cell_pointer_array_size())
}
- /// Get region of a cell's payload
- pub fn cell_get_raw_region(
- &self,
- idx: usize,
- payload_overflow_threshold_max: usize,
- payload_overflow_threshold_min: usize,
- usable_size: usize,
- ) -> (usize, usize) {
+ /// Get region(start end length) of a cell's payload
+ pub fn cell_get_raw_region(&self, idx: usize, usable_size: usize) -> (usize, usize) {
let buf = self.as_ptr();
let ncells = self.cell_count();
let (cell_pointer_array_start, _) = self.cell_pointer_array_offset_and_size();
assert!(idx < ncells, "cell_get: idx out of bounds");
- let cell_pointer = cell_pointer_array_start + (idx * 2); // pointers are 2 bytes each
+ let cell_pointer = cell_pointer_array_start + (idx * CELL_POINTER_SIZE_BYTES);
let cell_pointer = self.read_u16_no_offset(cell_pointer) as usize;
let start = cell_pointer;
+ let payload_overflow_threshold_max =
+ payload_overflow_threshold_max(self.page_type(), usable_size as u16);
+ let payload_overflow_threshold_min =
+ payload_overflow_threshold_min(self.page_type(), usable_size as u16);
let len = match self.page_type() {
PageType::IndexInterior => {
let (len_payload, n_payload) = read_varint(&buf[cell_pointer + 4..]).unwrap();
@@ -668,7 +659,11 @@ impl PageContent {
if overflows {
to_read + n_payload
} else {
- len_payload as usize + n_payload
+ let mut size = len_payload as usize + n_payload;
+ if size < MINIMUM_CELL_SIZE {
+ size = MINIMUM_CELL_SIZE;
+ }
+ size
}
}
PageType::TableLeaf => {
@@ -683,7 +678,11 @@ impl PageContent {
if overflows {
to_read + n_payload + n_rowid
} else {
- len_payload as usize + n_payload + n_rowid
+ let mut size = len_payload as usize + n_payload + n_rowid;
+ if size < MINIMUM_CELL_SIZE {
+ size = MINIMUM_CELL_SIZE;
+ }
+ size
}
}
};
@@ -727,6 +726,7 @@ impl PageContent {
}
}
+#[instrument(skip_all, level = Level::INFO)]
pub fn begin_read_page(
db_file: Arc,
buffer_pool: Arc,
@@ -773,6 +773,7 @@ pub fn finish_read_page(
Ok(())
}
+#[instrument(skip_all, level = Level::INFO)]
pub fn begin_write_btree_page(
pager: &Pager,
page: &PageRef,
@@ -791,13 +792,14 @@ pub fn begin_write_btree_page(
};
*write_counter.borrow_mut() += 1;
+ let clone_counter = write_counter.clone();
let write_complete = {
let buf_copy = buffer.clone();
Box::new(move |bytes_written: i32| {
tracing::trace!("finish_write_btree_page");
let buf_copy = buf_copy.clone();
let buf_len = buf_copy.borrow().len();
- *write_counter.borrow_mut() -= 1;
+ *clone_counter.borrow_mut() -= 1;
page_finish.clear_dirty();
if bytes_written < buf_len as i32 {
@@ -806,10 +808,15 @@ pub fn begin_write_btree_page(
})
};
let c = Completion::new(CompletionType::Write(WriteCompletion::new(write_complete)));
- page_source.write_page(page_id, buffer.clone(), c)?;
- Ok(())
+ let res = page_source.write_page(page_id, buffer.clone(), c);
+ if res.is_err() {
+ // Avoid infinite loop if write page fails
+ *write_counter.borrow_mut() -= 1;
+ }
+ res
}
+#[instrument(skip_all, level = Level::INFO)]
pub fn begin_sync(db_file: Arc, syncing: Rc>) -> Result<()> {
assert!(!*syncing.borrow());
*syncing.borrow_mut() = true;
@@ -834,15 +841,15 @@ pub enum BTreeCell {
#[derive(Debug, Clone)]
pub struct TableInteriorCell {
- pub _left_child_page: u32,
- pub _rowid: i64,
+ pub left_child_page: u32,
+ pub rowid: i64,
}
#[derive(Debug, Clone)]
pub struct TableLeafCell {
- pub _rowid: i64,
+ pub rowid: i64,
/// Payload of cell, if it overflows it won't include overflowed payload.
- pub _payload: &'static [u8],
+ pub payload: &'static [u8],
/// This is the complete payload size including overflow pages.
pub payload_size: u64,
pub first_overflow_page: Option,
@@ -860,21 +867,22 @@ pub struct IndexInteriorCell {
#[derive(Debug, Clone)]
pub struct IndexLeafCell {
pub payload: &'static [u8],
- pub first_overflow_page: Option,
/// This is the complete payload size including overflow pages.
pub payload_size: u64,
+ pub first_overflow_page: Option,
}
/// read_btree_cell contructs a BTreeCell which is basically a wrapper around pointer to the payload of a cell.
/// buffer input "page" is static because we want the cell to point to the data in the page in case it has any payload.
pub fn read_btree_cell(
page: &'static [u8],
- page_type: &PageType,
+ page_content: &PageContent,
pos: usize,
- max_local: usize,
- min_local: usize,
usable_size: usize,
) -> Result {
+ let page_type = page_content.page_type();
+ let max_local = payload_overflow_threshold_max(page_type, usable_size as u16);
+ let min_local = payload_overflow_threshold_min(page_type, usable_size as u16);
match page_type {
PageType::IndexInterior => {
let mut pos = pos;
@@ -904,8 +912,8 @@ pub fn read_btree_cell(
pos += 4;
let (rowid, _) = read_varint(&page[pos..])?;
Ok(BTreeCell::TableInteriorCell(TableInteriorCell {
- _left_child_page: left_child_page,
- _rowid: rowid as i64,
+ left_child_page,
+ rowid: rowid as i64,
}))
}
PageType::IndexLeaf => {
@@ -939,8 +947,8 @@ pub fn read_btree_cell(
let (payload, first_overflow_page) =
read_payload(&page[pos..pos + to_read], payload_size as usize);
Ok(BTreeCell::TableLeafCell(TableLeafCell {
- _rowid: rowid as i64,
- _payload: payload,
+ rowid: rowid as i64,
+ payload,
first_overflow_page,
payload_size,
}))
@@ -1312,6 +1320,7 @@ pub fn read_entire_wal_dumb(file: &Arc) -> Result) -> Result) -> Result) -> Result 0;
if is_commit_record {
wfs_data.max_frame.store(frame_idx, Ordering::SeqCst);
- wfs_data.last_checksum = cumulative_checksum;
}
frame_idx += 1;
current_offset += WAL_FRAME_HEADER_SIZE + page_size;
}
+ wfs_data.last_checksum = cumulative_checksum;
wfs_data.loaded.store(true, Ordering::SeqCst);
});
let c = Completion::new(CompletionType::Read(ReadCompletion::new(
@@ -1481,7 +1498,7 @@ pub fn begin_read_wal_frame(
Ok(c)
}
-#[instrument(skip(io, page, write_counter, wal_header, checksums), level = Level::TRACE)]
+#[instrument(err,skip(io, page, write_counter, wal_header, checksums), level = Level::INFO)]
#[allow(clippy::too_many_arguments)]
pub fn begin_write_wal_frame(
io: &Arc,
@@ -1540,6 +1557,11 @@ pub fn begin_write_wal_frame(
);
header.checksum_1 = final_checksum.0;
header.checksum_2 = final_checksum.1;
+ tracing::trace!(
+ "begin_write_wal_frame(checksum=({}, {}))",
+ header.checksum_1,
+ header.checksum_2
+ );
buf[16..20].copy_from_slice(&header.checksum_1.to_be_bytes());
buf[20..24].copy_from_slice(&header.checksum_2.to_be_bytes());
@@ -1548,13 +1570,14 @@ pub fn begin_write_wal_frame(
(Arc::new(RefCell::new(buffer)), final_checksum)
};
+ let clone_counter = write_counter.clone();
*write_counter.borrow_mut() += 1;
let write_complete = {
let buf_copy = buffer.clone();
Box::new(move |bytes_written: i32| {
let buf_copy = buf_copy.clone();
let buf_len = buf_copy.borrow().len();
- *write_counter.borrow_mut() -= 1;
+ *clone_counter.borrow_mut() -= 1;
page_finish.clear_dirty();
if bytes_written < buf_len as i32 {
@@ -1564,12 +1587,18 @@ pub fn begin_write_wal_frame(
};
#[allow(clippy::arc_with_non_send_sync)]
let c = Completion::new(CompletionType::Write(WriteCompletion::new(write_complete)));
- io.pwrite(offset, buffer.clone(), c)?;
+ let res = io.pwrite(offset, buffer.clone(), c);
+ if res.is_err() {
+ // If we do not reduce the counter here on error, we incur an infinite loop when cacheflushing
+ *write_counter.borrow_mut() -= 1;
+ }
+ res?;
tracing::trace!("Frame written and synced");
Ok(checksums)
}
pub fn begin_write_wal_header(io: &Arc, header: &WalHeader) -> Result<()> {
+ tracing::trace!("begin_write_wal_header");
let buffer = {
let drop_fn = Rc::new(|_buf| {});
diff --git a/core/storage/wal.rs b/core/storage/wal.rs
index 699d70d8d..1836022fc 100644
--- a/core/storage/wal.rs
+++ b/core/storage/wal.rs
@@ -23,7 +23,7 @@ use crate::storage::sqlite3_ondisk::{
begin_read_wal_frame, begin_write_wal_frame, finish_read_page, WAL_FRAME_HEADER_SIZE,
WAL_HEADER_SIZE,
};
-use crate::{Buffer, Result};
+use crate::{Buffer, LimboError, Result};
use crate::{Completion, Page};
use self::sqlite3_ondisk::{checksum_wal, PageContent, WAL_MAGIC_BE, WAL_MAGIC_LE};
@@ -479,6 +479,7 @@ pub struct WalFileShared {
/// There is only one write allowed in WAL mode. This lock takes care of ensuring there is only
/// one used.
pub write_lock: LimboRwLock,
+ pub checkpoint_lock: LimboRwLock,
pub loaded: AtomicBool,
}
@@ -499,6 +500,7 @@ impl fmt::Debug for WalFileShared {
impl Wal for WalFile {
/// Begin a read transaction.
+ #[instrument(skip_all, level = Level::INFO)]
fn begin_read_tx(&mut self) -> Result {
let max_frame_in_wal = self.get_shared().max_frame.load(Ordering::SeqCst);
@@ -564,6 +566,7 @@ impl Wal for WalFile {
/// End a read transaction.
#[inline(always)]
+ #[instrument(skip_all, level = Level::INFO)]
fn end_read_tx(&self) -> Result {
tracing::debug!("end_read_tx(lock={})", self.max_frame_read_lock_index);
let read_lock = &mut self.get_shared().read_locks[self.max_frame_read_lock_index];
@@ -572,6 +575,7 @@ impl Wal for WalFile {
}
/// Begin a write transaction
+ #[instrument(skip_all, level = Level::INFO)]
fn begin_write_tx(&mut self) -> Result {
let busy = !self.get_shared().write_lock.write();
tracing::debug!("begin_write_transaction(busy={})", busy);
@@ -582,6 +586,7 @@ impl Wal for WalFile {
}
/// End a write transaction
+ #[instrument(skip_all, level = Level::INFO)]
fn end_write_tx(&self) -> Result {
tracing::debug!("end_write_txn");
self.get_shared().write_lock.unlock();
@@ -589,6 +594,7 @@ impl Wal for WalFile {
}
/// Find the latest frame containing a page.
+ #[instrument(skip_all, level = Level::INFO)]
fn find_frame(&self, page_id: u64) -> Result> {
let shared = self.get_shared();
let frames = shared.frame_cache.lock();
@@ -606,6 +612,7 @@ impl Wal for WalFile {
}
/// Read a frame from the WAL.
+ #[instrument(skip_all, level = Level::INFO)]
fn read_frame(&self, frame_id: u64, page: PageRef, buffer_pool: Arc) -> Result<()> {
tracing::debug!("read_frame({})", frame_id);
let offset = self.frame_offset(frame_id);
@@ -624,6 +631,7 @@ impl Wal for WalFile {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn read_frame_raw(
&self,
frame_id: u64,
@@ -650,6 +658,7 @@ impl Wal for WalFile {
}
/// Write a frame to the WAL.
+ #[instrument(skip_all, level = Level::INFO)]
fn append_frame(
&mut self,
page: PageRef,
@@ -660,12 +669,7 @@ impl Wal for WalFile {
let max_frame = self.max_frame;
let frame_id = if max_frame == 0 { 1 } else { max_frame + 1 };
let offset = self.frame_offset(frame_id);
- tracing::debug!(
- "append_frame(frame={}, offset={}, page_id={})",
- frame_id,
- offset,
- page_id
- );
+ tracing::debug!(frame_id, offset, page_id);
let checksums = {
let shared = self.get_shared();
let header = shared.wal_header.clone();
@@ -699,13 +703,14 @@ impl Wal for WalFile {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn should_checkpoint(&self) -> bool {
let shared = self.get_shared();
let frame_id = shared.max_frame.load(Ordering::SeqCst) as usize;
frame_id >= self.checkpoint_threshold
}
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
fn checkpoint(
&mut self,
pager: &Pager,
@@ -724,6 +729,10 @@ impl Wal for WalFile {
// TODO(pere): check what frames are safe to checkpoint between many readers!
self.ongoing_checkpoint.min_frame = self.min_frame;
let shared = self.get_shared();
+ let busy = !shared.checkpoint_lock.write();
+ if busy {
+ return Err(LimboError::Busy);
+ }
let mut max_safe_frame = shared.max_frame.load(Ordering::SeqCst);
for (read_lock_idx, read_lock) in shared.read_locks.iter_mut().enumerate() {
let this_mark = read_lock.value.load(Ordering::SeqCst);
@@ -747,8 +756,8 @@ impl Wal for WalFile {
self.ongoing_checkpoint.state = CheckpointState::ReadFrame;
tracing::trace!(
"checkpoint_start(min_frame={}, max_frame={})",
+ self.ongoing_checkpoint.min_frame,
self.ongoing_checkpoint.max_frame,
- self.ongoing_checkpoint.min_frame
);
}
CheckpointState::ReadFrame => {
@@ -831,6 +840,7 @@ impl Wal for WalFile {
return Ok(CheckpointStatus::IO);
}
let shared = self.get_shared();
+ shared.checkpoint_lock.unlock();
// Record two num pages fields to return as checkpoint result to caller.
// Ref: pnLog, pnCkpt on https://www.sqlite.org/c3ref/wal_checkpoint_v2.html
@@ -869,7 +879,7 @@ impl Wal for WalFile {
}
}
- #[instrument(skip_all, level = Level::DEBUG)]
+ #[instrument(err, skip_all, level = Level::INFO)]
fn sync(&mut self) -> Result {
match self.sync_state.get() {
SyncState::NotSyncing => {
@@ -911,6 +921,7 @@ impl Wal for WalFile {
self.min_frame
}
+ #[instrument(err, skip_all, level = Level::INFO)]
fn rollback(&mut self) -> Result<()> {
// TODO(pere): have to remove things from frame_cache because they are no longer valid.
// TODO(pere): clear page cache in pager.
@@ -918,7 +929,7 @@ impl Wal for WalFile {
// TODO(pere): implement proper hashmap, this sucks :).
let shared = self.get_shared();
let max_frame = shared.max_frame.load(Ordering::SeqCst);
- tracing::trace!("rollback(to_max_frame={})", max_frame);
+ tracing::debug!(to_max_frame = max_frame);
let mut frame_cache = shared.frame_cache.lock();
for (_, frames) in frame_cache.iter_mut() {
let mut last_valid_frame = frames.len();
@@ -936,14 +947,11 @@ impl Wal for WalFile {
Ok(())
}
+ #[instrument(skip_all, level = Level::INFO)]
fn finish_append_frames_commit(&mut self) -> Result<()> {
let shared = self.get_shared();
shared.max_frame.store(self.max_frame, Ordering::SeqCst);
- tracing::trace!(
- "finish_append_frames_commit(max_frame={}, last_checksum={:?})",
- self.max_frame,
- self.last_checksum
- );
+ tracing::trace!(self.max_frame, ?self.last_checksum);
shared.last_checksum = self.last_checksum;
Ok(())
}
@@ -969,6 +977,7 @@ impl WalFile {
}
let header = unsafe { shared.get().as_mut().unwrap().wal_header.lock() };
+ let last_checksum = unsafe { (*shared.get()).last_checksum };
Self {
io,
// default to max frame in WAL, so that when we read schema we can read from WAL too if it's there.
@@ -987,7 +996,7 @@ impl WalFile {
sync_state: Cell::new(SyncState::NotSyncing),
min_frame: 0,
max_frame_read_lock_index: 0,
- last_checksum: (0, 0),
+ last_checksum,
start_pages_in_frames: 0,
header: *header,
}
@@ -1075,6 +1084,7 @@ impl WalFileShared {
let checksum = header.lock();
(checksum.checksum_1, checksum.checksum_2)
};
+ tracing::debug!("new_shared(header={:?})", header);
let shared = WalFileShared {
wal_header: header,
min_frame: AtomicU64::new(0),
@@ -1094,6 +1104,7 @@ impl WalFileShared {
nreads: AtomicU32::new(0),
value: AtomicU32::new(READMARK_NOT_USED),
},
+ checkpoint_lock: LimboRwLock::new(),
loaded: AtomicBool::new(true),
};
Ok(Arc::new(UnsafeCell::new(shared)))
diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs
index fad19baec..58dcd5372 100644
--- a/core/translate/compound_select.rs
+++ b/core/translate/compound_select.rs
@@ -11,7 +11,7 @@ use turso_sqlite3_parser::ast::{CompoundOperator, SortOrder};
use tracing::Level;
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
pub fn emit_program_for_compound_select(
program: &mut ProgramBuilder,
plan: Plan,
@@ -150,9 +150,9 @@ fn emit_compound_select(
CompoundOperator::Union => {
let mut new_dedupe_index = false;
let dedupe_index = match right_most.query_destination {
- QueryDestination::EphemeralIndex { cursor_id, index } => {
- (cursor_id, index.clone())
- }
+ QueryDestination::EphemeralIndex {
+ cursor_id, index, ..
+ } => (cursor_id, index.clone()),
_ => {
new_dedupe_index = true;
create_dedupe_index(program, &right_most, schema)?
@@ -161,6 +161,7 @@ fn emit_compound_select(
plan.query_destination = QueryDestination::EphemeralIndex {
cursor_id: dedupe_index.0,
index: dedupe_index.1.clone(),
+ is_delete: false,
};
let compound_select = Plan::CompoundSelect {
left,
@@ -182,20 +183,18 @@ fn emit_compound_select(
right_most.query_destination = QueryDestination::EphemeralIndex {
cursor_id: dedupe_index.0,
index: dedupe_index.1.clone(),
+ is_delete: false,
};
emit_query(program, &mut right_most, &mut right_most_ctx)?;
if new_dedupe_index {
- let label_jump_over_dedupe = program.allocate_label();
- read_deduplicated_union_rows(
+ read_deduplicated_union_or_except_rows(
program,
dedupe_index.0,
dedupe_index.1.as_ref(),
limit_ctx,
- label_jump_over_dedupe,
yield_reg,
);
- program.preassign_label_to_next_insn(label_jump_over_dedupe);
}
}
CompoundOperator::Intersect => {
@@ -211,6 +210,7 @@ fn emit_compound_select(
plan.query_destination = QueryDestination::EphemeralIndex {
cursor_id: left_cursor_id,
index: left_index.clone(),
+ is_delete: false,
};
let compound_select = Plan::CompoundSelect {
left,
@@ -234,6 +234,7 @@ fn emit_compound_select(
right_most.query_destination = QueryDestination::EphemeralIndex {
cursor_id: right_cursor_id,
index: right_index,
+ is_delete: false,
};
emit_query(program, &mut right_most, &mut right_most_ctx)?;
read_intersect_rows(
@@ -246,8 +247,49 @@ fn emit_compound_select(
yield_reg,
);
}
- _ => {
- crate::bail_parse_error!("unimplemented compound select operator: {:?}", operator);
+ CompoundOperator::Except => {
+ let mut new_index = false;
+ let (cursor_id, index) = match right_most.query_destination {
+ QueryDestination::EphemeralIndex {
+ cursor_id, index, ..
+ } => (cursor_id, index),
+ _ => {
+ new_index = true;
+ create_dedupe_index(program, &right_most, schema)?
+ }
+ };
+ plan.query_destination = QueryDestination::EphemeralIndex {
+ cursor_id,
+ index: index.clone(),
+ is_delete: false,
+ };
+ let compound_select = Plan::CompoundSelect {
+ left,
+ right_most: plan,
+ limit,
+ offset,
+ order_by,
+ };
+ emit_compound_select(
+ program,
+ compound_select,
+ schema,
+ syms,
+ None,
+ yield_reg,
+ reg_result_cols_start,
+ )?;
+ right_most.query_destination = QueryDestination::EphemeralIndex {
+ cursor_id,
+ index: index.clone(),
+ is_delete: true,
+ };
+ emit_query(program, &mut right_most, &mut right_most_ctx)?;
+ if new_index {
+ read_deduplicated_union_or_except_rows(
+ program, cursor_id, &index, limit_ctx, yield_reg,
+ );
+ }
}
},
None => {
@@ -302,15 +344,16 @@ fn create_dedupe_index(
Ok((cursor_id, dedupe_index.clone()))
}
-/// Emits the bytecode for reading deduplicated rows from the ephemeral index created for UNION operators.
-fn read_deduplicated_union_rows(
+/// Emits the bytecode for reading deduplicated rows from the ephemeral index created for
+/// UNION or EXCEPT operators.
+fn read_deduplicated_union_or_except_rows(
program: &mut ProgramBuilder,
dedupe_cursor_id: usize,
dedupe_index: &Index,
limit_ctx: Option,
- label_limit_reached: BranchOffset,
yield_reg: Option,
) {
+ let label_close = program.allocate_label();
let label_dedupe_next = program.allocate_label();
let label_dedupe_loop_start = program.allocate_label();
let dedupe_cols_start_reg = program.alloc_registers(dedupe_index.columns.len());
@@ -348,7 +391,7 @@ fn read_deduplicated_union_rows(
if let Some(limit_ctx) = limit_ctx {
program.emit_insn(Insn::DecrJumpZero {
reg: limit_ctx.reg_limit,
- target_pc: label_limit_reached,
+ target_pc: label_close,
})
}
program.preassign_label_to_next_insn(label_dedupe_next);
@@ -356,6 +399,7 @@ fn read_deduplicated_union_rows(
cursor_id: dedupe_cursor_id,
pc_if_next: label_dedupe_loop_start,
});
+ program.preassign_label_to_next_insn(label_close);
program.emit_insn(Insn::Close {
cursor_id: dedupe_cursor_id,
});
diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs
index 17540ac3a..0b50c40a6 100644
--- a/core/translate/emitter.rs
+++ b/core/translate/emitter.rs
@@ -31,7 +31,7 @@ use crate::vdbe::builder::{CursorKey, CursorType, ProgramBuilder};
use crate::vdbe::insn::{CmpInsFlags, IdxInsertFlags, InsertFlags, RegisterOrLiteral};
use crate::vdbe::CursorID;
use crate::vdbe::{insn::Insn, BranchOffset};
-use crate::{Result, SymbolTable};
+use crate::{bail_parse_error, Result, SymbolTable};
pub struct Resolver<'a> {
pub schema: &'a Schema,
@@ -149,6 +149,8 @@ pub struct TranslateCtx<'a> {
/// - First: all `GROUP BY` expressions, in the order they appear in the `GROUP BY` clause.
/// - Then: remaining non-aggregate expressions that are not part of `GROUP BY`.
pub non_aggregate_expressions: Vec<(&'a Expr, bool)>,
+ /// Cursor id for turso_cdc table (if capture_data_changes=on is set and query can modify the data)
+ pub cdc_cursor_id: Option,
}
impl<'a> TranslateCtx<'a> {
@@ -175,6 +177,7 @@ impl<'a> TranslateCtx<'a> {
result_columns_to_skip_in_orderby_sorter: None,
resolver: Resolver::new(schema, syms),
non_aggregate_expressions: Vec::new(),
+ cdc_cursor_id: None,
}
}
}
@@ -198,7 +201,7 @@ pub enum TransactionMode {
/// Main entry point for emitting bytecode for a SQL query
/// Takes a query plan and generates the corresponding bytecode program
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
pub fn emit_program(
program: &mut ProgramBuilder,
plan: Plan,
@@ -216,7 +219,7 @@ pub fn emit_program(
}
}
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
fn emit_program_for_select(
program: &mut ProgramBuilder,
mut plan: SelectPlan,
@@ -255,7 +258,7 @@ fn emit_program_for_select(
Ok(())
}
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
pub fn emit_query<'a>(
program: &mut ProgramBuilder,
plan: &'a mut SelectPlan,
@@ -395,7 +398,7 @@ pub fn emit_query<'a>(
Ok(t_ctx.reg_result_cols_start.unwrap())
}
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
fn emit_program_for_delete(
program: &mut ProgramBuilder,
plan: DeletePlan,
@@ -562,10 +565,27 @@ fn emit_delete_insns(
start_reg,
num_regs,
cursor_id: index_cursor_id,
+ raise_error_if_no_matching_entry: true,
});
}
}
+ if let Some(turso_cdc_cursor_id) = t_ctx.cdc_cursor_id {
+ let rowid_reg = program.alloc_register();
+ program.emit_insn(Insn::RowId {
+ cursor_id: main_table_cursor_id,
+ dest: rowid_reg,
+ });
+ emit_cdc_insns(
+ program,
+ &t_ctx.resolver,
+ OperationMode::DELETE,
+ turso_cdc_cursor_id,
+ rowid_reg,
+ table_reference.table.get_name(),
+ )?;
+ }
+
program.emit_insn(Insn::Delete {
cursor_id: main_table_cursor_id,
});
@@ -580,7 +600,7 @@ fn emit_delete_insns(
Ok(())
}
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
fn emit_program_for_update(
program: &mut ProgramBuilder,
mut plan: UpdatePlan,
@@ -699,7 +719,7 @@ fn emit_program_for_update(
Ok(())
}
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
fn emit_update_insns(
plan: &UpdatePlan,
t_ctx: &TranslateCtx,
@@ -1064,6 +1084,7 @@ fn emit_update_insns(
start_reg,
num_regs,
cursor_id: idx_cursor_id,
+ raise_error_if_no_matching_entry: true,
});
// Insert new index key (filled further above with values from set_clauses)
@@ -1076,6 +1097,53 @@ fn emit_update_insns(
});
}
+ if let Some(cdc_cursor_id) = t_ctx.cdc_cursor_id {
+ let rowid_reg = program.alloc_register();
+ if has_user_provided_rowid {
+ program.emit_insn(Insn::RowId {
+ cursor_id,
+ dest: rowid_reg,
+ });
+ emit_cdc_insns(
+ program,
+ &t_ctx.resolver,
+ OperationMode::DELETE,
+ cdc_cursor_id,
+ rowid_reg,
+ table_ref.table.get_name(),
+ )?;
+ program.emit_insn(Insn::Copy {
+ src_reg: rowid_set_clause_reg.expect(
+ "rowid_set_clause_reg must be set because has_user_provided_rowid is true",
+ ),
+ dst_reg: rowid_reg,
+ amount: 1,
+ });
+ emit_cdc_insns(
+ program,
+ &t_ctx.resolver,
+ OperationMode::INSERT,
+ cdc_cursor_id,
+ rowid_reg,
+ table_ref.table.get_name(),
+ )?;
+ } else {
+ program.emit_insn(Insn::Copy {
+ src_reg: rowid_set_clause_reg.unwrap_or(beg),
+ dst_reg: rowid_reg,
+ amount: 1,
+ });
+ emit_cdc_insns(
+ program,
+ &t_ctx.resolver,
+ OperationMode::UPDATE,
+ cdc_cursor_id,
+ rowid_reg,
+ table_ref.table.get_name(),
+ )?;
+ }
+ }
+
// If we are updating the rowid, we cannot rely on overwrite on the
// Insert instruction to update the cell. We need to first delete the current cell
// and later insert the updated record
@@ -1115,6 +1183,79 @@ fn emit_update_insns(
Ok(())
}
+pub fn emit_cdc_insns(
+ program: &mut ProgramBuilder,
+ resolver: &Resolver,
+ operation_mode: OperationMode,
+ cdc_cursor_id: usize,
+ rowid_reg: usize,
+ table_name: &str,
+) -> Result<()> {
+ // (operation_id INTEGER PRIMARY KEY AUTOINCREMENT, operation_time INTEGER, operation_type INTEGER, table_name TEXT, id)
+ let turso_cdc_registers = program.alloc_registers(5);
+ program.emit_insn(Insn::Null {
+ dest: turso_cdc_registers,
+ dest_end: None,
+ });
+ program.mark_last_insn_constant();
+
+ let Some(unixepoch_fn) = resolver.resolve_function("unixepoch", 0) else {
+ bail_parse_error!("no function {}", "unixepoch");
+ };
+ let unixepoch_fn_ctx = crate::function::FuncCtx {
+ func: unixepoch_fn,
+ arg_count: 0,
+ };
+
+ program.emit_insn(Insn::Function {
+ constant_mask: 0,
+ start_reg: 0,
+ dest: turso_cdc_registers + 1,
+ func: unixepoch_fn_ctx,
+ });
+
+ let operation_type = match operation_mode {
+ OperationMode::INSERT => 1,
+ OperationMode::UPDATE | OperationMode::SELECT => 0,
+ OperationMode::DELETE => -1,
+ };
+ program.emit_int(operation_type, turso_cdc_registers + 2);
+ program.mark_last_insn_constant();
+
+ program.emit_string8(table_name.to_string(), turso_cdc_registers + 3);
+ program.mark_last_insn_constant();
+
+ program.emit_insn(Insn::Copy {
+ src_reg: rowid_reg,
+ dst_reg: turso_cdc_registers + 4,
+ amount: 0,
+ });
+
+ let rowid_reg = program.alloc_register();
+ program.emit_insn(Insn::NewRowid {
+ cursor: cdc_cursor_id,
+ rowid_reg,
+ prev_largest_reg: 0, // todo(sivukhin): properly set value here from sqlite_sequence table when AUTOINCREMENT will be properly implemented in Turso
+ });
+
+ let record_reg = program.alloc_register();
+ program.emit_insn(Insn::MakeRecord {
+ start_reg: turso_cdc_registers,
+ count: 5,
+ dest_reg: record_reg,
+ index_name: None,
+ });
+
+ program.emit_insn(Insn::Insert {
+ cursor: cdc_cursor_id,
+ key_reg: rowid_reg,
+ record_reg,
+ flag: InsertFlags::new(),
+ table_name: "".to_string(),
+ });
+ Ok(())
+}
+
/// Initialize the limit/offset counters and registers.
/// In case of compound SELECTs, the limit counter is initialized only once,
/// hence [LimitCtx::initialize_counter] being false in those cases.
diff --git a/core/translate/expr.rs b/core/translate/expr.rs
index ac6846daf..535bea71a 100644
--- a/core/translate/expr.rs
+++ b/core/translate/expr.rs
@@ -9,7 +9,7 @@ use crate::function::JsonFunc;
use crate::function::{Func, FuncCtx, MathFuncArity, ScalarFunc, VectorFunc};
use crate::functions::datetime;
use crate::schema::{Affinity, Table, Type};
-use crate::util::{exprs_are_equivalent, normalize_ident, parse_numeric_literal};
+use crate::util::{exprs_are_equivalent, parse_numeric_literal};
use crate::vdbe::builder::CursorKey;
use crate::vdbe::{
builder::ProgramBuilder,
@@ -27,7 +27,7 @@ pub struct ConditionMetadata {
pub jump_target_when_false: BranchOffset,
}
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
fn emit_cond_jump(program: &mut ProgramBuilder, cond_meta: ConditionMetadata, reg: usize) {
if cond_meta.jump_if_condition_is_true {
program.emit_insn(Insn::If {
@@ -131,7 +131,7 @@ macro_rules! expect_arguments_even {
}};
}
-#[instrument(skip(program, referenced_tables, expr, resolver), level = Level::TRACE)]
+#[instrument(skip(program, referenced_tables, expr, resolver), level = Level::INFO)]
pub fn translate_condition_expr(
program: &mut ProgramBuilder,
referenced_tables: &TableReferences,
@@ -680,8 +680,7 @@ pub fn translate_expr(
order_by: _,
} => {
let args_count = if let Some(args) = args { args.len() } else { 0 };
- let func_name = normalize_ident(name.0.as_str());
- let func_type = resolver.resolve_function(&func_name, args_count);
+ let func_type = resolver.resolve_function(&name.0, args_count);
if func_type.is_none() {
crate::bail_parse_error!("unknown function {}", name.0);
diff --git a/core/translate/insert.rs b/core/translate/insert.rs
index 872b891c7..319b2e7ba 100644
--- a/core/translate/insert.rs
+++ b/core/translate/insert.rs
@@ -6,6 +6,7 @@ use turso_sqlite3_parser::ast::{
use crate::error::{SQLITE_CONSTRAINT_NOTNULL, SQLITE_CONSTRAINT_PRIMARYKEY};
use crate::schema::{IndexColumn, Table};
+use crate::translate::emitter::{emit_cdc_insns, OperationMode};
use crate::util::normalize_ident;
use crate::vdbe::builder::ProgramBuilderOpts;
use crate::vdbe::insn::{IdxInsertFlags, InsertFlags, RegisterOrLiteral};
@@ -116,6 +117,26 @@ pub fn translate_insert(
let halt_label = program.allocate_label();
let loop_start_label = program.allocate_label();
+ let cdc_table = program.capture_data_changes_mode().table();
+ let cdc_table = if let Some(cdc_table) = cdc_table {
+ if table.get_name() != cdc_table {
+ let Some(turso_cdc_table) = schema.get_table(cdc_table) else {
+ crate::bail_parse_error!("no such table: {}", cdc_table);
+ };
+ let Some(cdc_btree) = turso_cdc_table.btree().clone() else {
+ crate::bail_parse_error!("no such table: {}", cdc_table);
+ };
+ Some((
+ program.alloc_cursor_id(CursorType::BTreeTable(cdc_btree.clone())),
+ cdc_btree,
+ ))
+ } else {
+ None
+ }
+ } else {
+ None
+ };
+
let mut yield_reg_opt = None;
let mut temp_table_ctx = None;
let (num_values, cursor_id) = match body {
@@ -328,6 +349,15 @@ pub fn translate_insert(
&resolver,
)?;
}
+ // Open turso_cdc table btree for writing if necessary
+ if let Some((cdc_cursor_id, cdc_btree)) = &cdc_table {
+ program.emit_insn(Insn::OpenWrite {
+ cursor_id: *cdc_cursor_id,
+ root_page: cdc_btree.root_page.into(),
+ name: cdc_btree.name.clone(),
+ });
+ }
+
// Open all the index btrees for writing
for idx_cursor in idx_cursors.iter() {
program.emit_insn(Insn::OpenWrite {
@@ -414,6 +444,18 @@ pub fn translate_insert(
_ => (),
}
+ // Write record to the turso_cdc table if necessary
+ if let Some((cdc_cursor_id, _)) = &cdc_table {
+ emit_cdc_insns(
+ &mut program,
+ &resolver,
+ OperationMode::INSERT,
+ *cdc_cursor_id,
+ rowid_reg,
+ &table_name.0,
+ )?;
+ }
+
let index_col_mappings = resolve_indicies_for_insert(schema, table.as_ref(), &column_mappings)?;
for index_col_mapping in index_col_mappings {
// find which cursor we opened earlier for this index
diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs
index 88c18c054..252eb193a 100644
--- a/core/translate/main_loop.rs
+++ b/core/translate/main_loop.rs
@@ -117,6 +117,33 @@ pub fn init_loop(
t_ctx.meta_left_joins.len() == tables.joined_tables().len(),
"meta_left_joins length does not match tables length"
);
+
+ let cdc_table = program.capture_data_changes_mode().table();
+ if cdc_table.is_some()
+ && matches!(
+ mode,
+ OperationMode::INSERT | OperationMode::UPDATE | OperationMode::DELETE
+ )
+ {
+ assert!(tables.joined_tables().len() == 1);
+ let cdc_table_name = cdc_table.unwrap();
+ if tables.joined_tables()[0].table.get_name() != cdc_table_name {
+ let Some(cdc_table) = t_ctx.resolver.schema.get_table(cdc_table_name) else {
+ crate::bail_parse_error!("no such table: {}", cdc_table_name);
+ };
+ let Some(cdc_btree) = cdc_table.btree().clone() else {
+ crate::bail_parse_error!("no such table: {}", cdc_table_name);
+ };
+ let cdc_cursor_id = program.alloc_cursor_id(CursorType::BTreeTable(cdc_btree.clone()));
+ program.emit_insn(Insn::OpenWrite {
+ cursor_id: cdc_cursor_id,
+ root_page: cdc_btree.root_page.into(),
+ name: cdc_btree.name.clone(),
+ });
+ t_ctx.cdc_cursor_id = Some(cdc_cursor_id);
+ }
+ }
+
// Initialize ephemeral indexes for distinct aggregates
for (i, agg) in aggregates
.iter_mut()
diff --git a/core/translate/mod.rs b/core/translate/mod.rs
index b7c82d585..e9dd4b395 100644
--- a/core/translate/mod.rs
+++ b/core/translate/mod.rs
@@ -53,7 +53,7 @@ use transaction::{translate_tx_begin, translate_tx_commit};
use turso_sqlite3_parser::ast::{self, Delete, Insert};
use update::translate_update;
-#[instrument(skip_all, level = Level::TRACE)]
+#[instrument(skip_all, level = Level::INFO)]
#[allow(clippy::too_many_arguments)]
pub fn translate(
schema: &Schema,
@@ -75,6 +75,7 @@ pub fn translate(
let mut program = ProgramBuilder::new(
query_mode,
+ connection.get_capture_data_changes().clone(),
// These options will be extended whithin each translate program
ProgramBuilderOpts {
num_cursors: 1,
diff --git a/core/translate/plan.rs b/core/translate/plan.rs
index 462f38442..6347b500f 100644
--- a/core/translate/plan.rs
+++ b/core/translate/plan.rs
@@ -324,6 +324,8 @@ pub enum QueryDestination {
cursor_id: CursorID,
/// The index that will be used to store the results.
index: Arc,
+ /// Whether this is a delete operation that will remove the index entries
+ is_delete: bool,
},
/// The results of the query are stored in an ephemeral table,
/// later used by the parent query.
diff --git a/core/translate/planner.rs b/core/translate/planner.rs
index 56f51bc5c..00ebaf897 100644
--- a/core/translate/planner.rs
+++ b/core/translate/planner.rs
@@ -51,8 +51,7 @@ pub fn resolve_aggregates(
} else {
0
};
- match Func::resolve_function(normalize_ident(name.0.as_str()).as_str(), args_count)
- {
+ match Func::resolve_function(&name.0, args_count) {
Ok(Func::Agg(f)) => {
let distinctness = Distinctness::from_ast(distinctness.as_ref());
if !schema.indexes_enabled() && distinctness.is_distinct() {
@@ -84,9 +83,7 @@ pub fn resolve_aggregates(
}
}
Expr::FunctionCallStar { name, .. } => {
- if let Ok(Func::Agg(f)) =
- Func::resolve_function(normalize_ident(name.0.as_str()).as_str(), 0)
- {
+ if let Ok(Func::Agg(f)) = Func::resolve_function(&name.0, 0) {
aggs.push(Aggregate {
func: f,
args: vec![],
diff --git a/core/translate/pragma.rs b/core/translate/pragma.rs
index 93883a14b..f8912d3c5 100644
--- a/core/translate/pragma.rs
+++ b/core/translate/pragma.rs
@@ -3,17 +3,19 @@
use std::rc::Rc;
use std::sync::Arc;
-use turso_sqlite3_parser::ast::PragmaName;
-use turso_sqlite3_parser::ast::{self, Expr};
+use turso_sqlite3_parser::ast::{self, ColumnDefinition, Expr};
+use turso_sqlite3_parser::ast::{PragmaName, QualifiedName};
+use crate::pragma::pragma_for;
use crate::schema::Schema;
use crate::storage::pager::AutoVacuumMode;
use crate::storage::sqlite3_ondisk::MIN_PAGE_CACHE_SIZE;
use crate::storage::wal::CheckpointMode;
-use crate::util::{normalize_ident, parse_signed_number};
+use crate::translate::schema::translate_create_table;
+use crate::util::{normalize_ident, parse_signed_number, parse_string};
use crate::vdbe::builder::{ProgramBuilder, ProgramBuilderOpts};
use crate::vdbe::insn::{Cookie, Insn};
-use crate::{bail_parse_error, storage, LimboError, Value};
+use crate::{bail_parse_error, storage, CaptureDataChangesMode, LimboError, Value};
use std::str::FromStr;
use strum::IntoEnumIterator;
@@ -57,17 +59,15 @@ pub fn translate_pragma(
Err(_) => bail_parse_error!("Not a valid pragma name"),
};
- match body {
- None => {
- query_pragma(pragma, schema, None, pager, connection, &mut program)?;
- }
+ let mut program = match body {
+ None => query_pragma(pragma, schema, None, pager, connection, program)?,
Some(ast::PragmaBody::Equals(value) | ast::PragmaBody::Call(value)) => match pragma {
PragmaName::TableInfo => {
- query_pragma(pragma, schema, Some(value), pager, connection, &mut program)?;
+ query_pragma(pragma, schema, Some(value), pager, connection, program)?
}
_ => {
write = true;
- update_pragma(pragma, schema, value, pager, connection, &mut program)?;
+ update_pragma(pragma, schema, value, pager, connection, program)?
}
},
};
@@ -85,8 +85,8 @@ fn update_pragma(
value: ast::Expr,
pager: Rc,
connection: Arc,
- program: &mut ProgramBuilder,
-) -> crate::Result<()> {
+ mut program: ProgramBuilder,
+) -> crate::Result {
match pragma {
PragmaName::CacheSize => {
let cache_size = match parse_signed_number(&value)? {
@@ -95,42 +95,33 @@ fn update_pragma(
_ => bail_parse_error!("Invalid value for cache size pragma"),
};
update_cache_size(cache_size, pager, connection)?;
- Ok(())
- }
- PragmaName::JournalMode => {
- query_pragma(
- PragmaName::JournalMode,
- schema,
- None,
- pager,
- connection,
- program,
- )?;
- Ok(())
- }
- PragmaName::LegacyFileFormat => Ok(()),
- PragmaName::WalCheckpoint => {
- query_pragma(
- PragmaName::WalCheckpoint,
- schema,
- Some(value),
- pager,
- connection,
- program,
- )?;
- Ok(())
- }
- PragmaName::PageCount => {
- query_pragma(
- PragmaName::PageCount,
- schema,
- None,
- pager,
- connection,
- program,
- )?;
- Ok(())
+ Ok(program)
}
+ PragmaName::JournalMode => query_pragma(
+ PragmaName::JournalMode,
+ schema,
+ None,
+ pager,
+ connection,
+ program,
+ ),
+ PragmaName::LegacyFileFormat => Ok(program),
+ PragmaName::WalCheckpoint => query_pragma(
+ PragmaName::WalCheckpoint,
+ schema,
+ Some(value),
+ pager,
+ connection,
+ program,
+ ),
+ PragmaName::PageCount => query_pragma(
+ PragmaName::PageCount,
+ schema,
+ None,
+ pager,
+ connection,
+ program,
+ ),
PragmaName::UserVersion => {
let data = parse_signed_number(&value)?;
let version_value = match data {
@@ -145,7 +136,7 @@ fn update_pragma(
value: version_value,
p5: 1,
});
- Ok(())
+ Ok(program)
}
PragmaName::SchemaVersion => {
// TODO: Implement updating schema_version
@@ -214,9 +205,33 @@ fn update_pragma(
value: auto_vacuum_mode - 1,
p5: 0,
});
- Ok(())
+ Ok(program)
}
PragmaName::IntegrityCheck => unreachable!("integrity_check cannot be set"),
+ PragmaName::UnstableCaptureDataChangesConn => {
+ let value = parse_string(&value)?;
+ // todo(sivukhin): ideally, we should consistently update capture_data_changes connection flag only after successfull execution of schema change statement
+ // but for now, let's keep it as is...
+ let opts = CaptureDataChangesMode::parse(&value)?;
+ if let Some(table) = &opts.table() {
+ // make sure that we have table created
+ program = translate_create_table(
+ QualifiedName::single(ast::Name(table.to_string())),
+ false,
+ ast::CreateTableBody::columns_and_constraints_from_definition(
+ turso_cdc_table_columns(),
+ None,
+ ast::TableOptions::NONE,
+ )
+ .unwrap(),
+ true,
+ schema,
+ program,
+ )?;
+ }
+ connection.set_capture_data_changes(opts);
+ Ok(program)
+ }
}
}
@@ -226,8 +241,8 @@ fn query_pragma(
value: Option,
pager: Rc,
connection: Arc,
- program: &mut ProgramBuilder,
-) -> crate::Result<()> {
+ mut program: ProgramBuilder,
+) -> crate::Result {
let register = program.alloc_register();
match pragma {
PragmaName::CacheSize => {
@@ -365,11 +380,25 @@ fn query_pragma(
program.emit_result_row(register, 1);
}
PragmaName::IntegrityCheck => {
- translate_integrity_check(schema, program)?;
+ translate_integrity_check(schema, &mut program)?;
+ }
+ PragmaName::UnstableCaptureDataChangesConn => {
+ let pragma = pragma_for(pragma);
+ let second_column = program.alloc_register();
+ let opts = connection.get_capture_data_changes();
+ program.emit_string8(opts.mode_name().to_string(), register);
+ if let Some(table) = &opts.table() {
+ program.emit_string8(table.to_string(), second_column);
+ } else {
+ program.emit_null(second_column, None);
+ }
+ program.emit_result_row(register, 2);
+ program.add_pragma_result_column(pragma.columns[0].to_string());
+ program.add_pragma_result_column(pragma.columns[1].to_string());
}
}
- Ok(())
+ Ok(program)
}
fn update_auto_vacuum_mode(
@@ -435,3 +464,53 @@ fn update_cache_size(
Ok(())
}
+
+pub const TURSO_CDC_DEFAULT_TABLE_NAME: &str = "turso_cdc";
+fn turso_cdc_table_columns() -> Vec {
+ vec![
+ ast::ColumnDefinition {
+ col_name: ast::Name("operation_id".to_string()),
+ col_type: Some(ast::Type {
+ name: "INTEGER".to_string(),
+ size: None,
+ }),
+ constraints: vec![ast::NamedColumnConstraint {
+ name: None,
+ constraint: ast::ColumnConstraint::PrimaryKey {
+ order: None,
+ conflict_clause: None,
+ auto_increment: true,
+ },
+ }],
+ },
+ ast::ColumnDefinition {
+ col_name: ast::Name("operation_time".to_string()),
+ col_type: Some(ast::Type {
+ name: "INTEGER".to_string(),
+ size: None,
+ }),
+ constraints: vec![],
+ },
+ ast::ColumnDefinition {
+ col_name: ast::Name("operation_type".to_string()),
+ col_type: Some(ast::Type {
+ name: "INTEGER".to_string(),
+ size: None,
+ }),
+ constraints: vec![],
+ },
+ ast::ColumnDefinition {
+ col_name: ast::Name("table_name".to_string()),
+ col_type: Some(ast::Type {
+ name: "TEXT".to_string(),
+ size: None,
+ }),
+ constraints: vec![],
+ },
+ ast::ColumnDefinition {
+ col_name: ast::Name("id".to_string()),
+ col_type: None,
+ constraints: vec![],
+ },
+ ]
+}
diff --git a/core/translate/result_row.rs b/core/translate/result_row.rs
index 74b071b44..62c6bc96a 100644
--- a/core/translate/result_row.rs
+++ b/core/translate/result_row.rs
@@ -85,21 +85,31 @@ pub fn emit_result_row_and_limit(
QueryDestination::EphemeralIndex {
cursor_id: index_cursor_id,
index: dedupe_index,
+ is_delete,
} => {
- let record_reg = program.alloc_register();
- program.emit_insn(Insn::MakeRecord {
- start_reg: result_columns_start_reg,
- count: plan.result_columns.len(),
- dest_reg: record_reg,
- index_name: Some(dedupe_index.name.clone()),
- });
- program.emit_insn(Insn::IdxInsert {
- cursor_id: *index_cursor_id,
- record_reg,
- unpacked_start: None,
- unpacked_count: None,
- flags: IdxInsertFlags::new(),
- });
+ if *is_delete {
+ program.emit_insn(Insn::IdxDelete {
+ start_reg: result_columns_start_reg,
+ num_regs: plan.result_columns.len(),
+ cursor_id: *index_cursor_id,
+ raise_error_if_no_matching_entry: false,
+ });
+ } else {
+ let record_reg = program.alloc_register();
+ program.emit_insn(Insn::MakeRecord {
+ start_reg: result_columns_start_reg,
+ count: plan.result_columns.len(),
+ dest_reg: record_reg,
+ index_name: Some(dedupe_index.name.clone()),
+ });
+ program.emit_insn(Insn::IdxInsert {
+ cursor_id: *index_cursor_id,
+ record_reg,
+ unpacked_start: None,
+ unpacked_count: None,
+ flags: IdxInsertFlags::new().no_op_duplicate(),
+ });
+ }
}
QueryDestination::EphemeralTable {
cursor_id: table_cursor_id,
diff --git a/core/translate/select.rs b/core/translate/select.rs
index a6013a562..50dd6ddf9 100644
--- a/core/translate/select.rs
+++ b/core/translate/select.rs
@@ -124,15 +124,6 @@ pub fn prepare_select_plan(
let mut left = Vec::with_capacity(compounds.len());
for CompoundSelect { select, operator } in compounds {
- // TODO: add support for EXCEPT
- if operator != ast::CompoundOperator::UnionAll
- && operator != ast::CompoundOperator::Union
- && operator != ast::CompoundOperator::Intersect
- {
- crate::bail_parse_error!(
- "only UNION ALL, UNION and INTERSECT are supported for compound SELECTs"
- );
- }
left.push((last, operator));
last = prepare_one_select_plan(
schema,
@@ -215,6 +206,14 @@ fn prepare_one_select_plan(
let mut table_references = TableReferences::new(vec![], outer_query_refs.to_vec());
+ if from.is_none() {
+ for column in &columns {
+ if matches!(column, ResultColumn::Star) {
+ crate::bail_parse_error!("no tables specified");
+ }
+ }
+ }
+
// Parse the FROM clause into a vec of TableReferences. Fold all the join conditions expressions into the WHERE clause.
parse_from(
schema,
@@ -349,10 +348,7 @@ fn prepare_one_select_plan(
if distinctness.is_distinct() && args_count != 1 {
crate::bail_parse_error!("DISTINCT aggregate functions must have exactly one argument");
}
- match Func::resolve_function(
- normalize_ident(name.0.as_str()).as_str(),
- args_count,
- ) {
+ match Func::resolve_function(&name.0, args_count) {
Ok(Func::Agg(f)) => {
let agg_args = match (args, &f) {
(None, crate::function::AggFunc::Count0) => {
@@ -451,11 +447,8 @@ fn prepare_one_select_plan(
ast::Expr::FunctionCallStar {
name,
filter_over: _,
- } => {
- if let Ok(Func::Agg(f)) = Func::resolve_function(
- normalize_ident(name.0.as_str()).as_str(),
- 0,
- ) {
+ } => match Func::resolve_function(&name.0, 0) {
+ Ok(Func::Agg(f)) => {
let agg = Aggregate {
func: f,
args: vec![ast::Expr::Literal(ast::Literal::Numeric(
@@ -473,13 +466,25 @@ fn prepare_one_select_plan(
expr: expr.clone(),
contains_aggregates: true,
});
- } else {
+ }
+ Ok(_) => {
crate::bail_parse_error!(
"Invalid aggregate function: {}",
name.0
);
}
- }
+ Err(e) => match e {
+ crate::LimboError::ParseError(e) => {
+ crate::bail_parse_error!("{}", e);
+ }
+ _ => {
+ crate::bail_parse_error!(
+ "Invalid aggregate function: {}",
+ name.0
+ );
+ }
+ },
+ },
expr => {
let contains_aggregates =
resolve_aggregates(schema, expr, &mut aggregate_expressions)?;
diff --git a/core/translate/subquery.rs b/core/translate/subquery.rs
index 4004a7cd1..645c95f6e 100644
--- a/core/translate/subquery.rs
+++ b/core/translate/subquery.rs
@@ -82,6 +82,7 @@ pub fn emit_subquery(
reg_limit_offset_sum: None,
resolver: Resolver::new(t_ctx.resolver.schema, t_ctx.resolver.symbol_table),
non_aggregate_expressions: Vec::new(),
+ cdc_cursor_id: None,
};
let subquery_body_end_label = program.allocate_label();
program.emit_insn(Insn::InitCoroutine {
diff --git a/core/types.rs b/core/types.rs
index e8dd9e588..f04253e6d 100644
--- a/core/types.rs
+++ b/core/types.rs
@@ -141,7 +141,13 @@ where
D: serde::Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
- s.parse().map_err(serde::de::Error::custom)
+ match crate::numeric::str_to_f64(s) {
+ Some(result) => Ok(match result {
+ crate::numeric::StrToF64::Fractional(non_nan) => non_nan.into(),
+ crate::numeric::StrToF64::Decimal(non_nan) => non_nan.into(),
+ }),
+ None => Err(serde::de::Error::custom("")),
+ }
}
#[derive(Debug, Clone)]
@@ -231,6 +237,20 @@ impl Value {
}
}
+ pub fn as_blob(&self) -> &Vec {
+ match self {
+ Value::Blob(b) => b,
+ _ => panic!("as_blob must be called only for Value::Blob"),
+ }
+ }
+
+ pub fn as_blob_mut(&mut self) -> &mut Vec {
+ match self {
+ Value::Blob(b) => b,
+ _ => panic!("as_blob must be called only for Value::Blob"),
+ }
+ }
+
pub fn from_text(text: &str) -> Self {
Value::Text(Text::new(text))
}
@@ -738,7 +758,9 @@ pub struct ImmutableRecord {
// We have to be super careful with this buffer since we make values point to the payload we need to take care reallocations
// happen in a controlled manner. If we realocate with values that should be correct, they will now point to undefined data.
// We don't use pin here because it would make it imposible to reuse the buffer if we need to push a new record in the same struct.
- payload: Vec,
+ //
+ // payload is the Vec but in order to use Register which holds ImmutableRecord as a Value - we store Vec as Value::Blob
+ payload: Value,
pub values: Vec,
recreating: bool,
}
@@ -828,7 +850,7 @@ impl<'a> AppendWriter<'a> {
impl ImmutableRecord {
pub fn new(payload_capacity: usize, value_capacity: usize) -> Self {
Self {
- payload: Vec::with_capacity(payload_capacity),
+ payload: Value::Blob(Vec::with_capacity(payload_capacity)),
values: Vec::with_capacity(value_capacity),
recreating: false,
}
@@ -977,7 +999,7 @@ impl ImmutableRecord {
writer.assert_finish_capacity();
Self {
- payload: buf,
+ payload: Value::Blob(buf),
values,
recreating: false,
}
@@ -985,7 +1007,7 @@ impl ImmutableRecord {
pub fn start_serialization(&mut self, payload: &[u8]) {
self.recreating = true;
- self.payload.extend_from_slice(payload);
+ self.payload.as_blob_mut().extend_from_slice(payload);
}
pub fn end_serialization(&mut self) {
assert!(self.recreating);
@@ -998,15 +1020,19 @@ impl ImmutableRecord {
}
pub fn invalidate(&mut self) {
- self.payload.clear();
+ self.payload.as_blob_mut().clear();
self.values.clear();
}
pub fn is_invalidated(&self) -> bool {
- self.payload.is_empty()
+ self.payload.as_blob().is_empty()
}
pub fn get_payload(&self) -> &[u8] {
+ self.payload.as_blob()
+ }
+
+ pub fn as_blob_value(&self) -> &Value {
&self.payload
}
}
@@ -1042,20 +1068,20 @@ impl Clone for ImmutableRecord {
RefValue::Float(f) => RefValue::Float(*f),
RefValue::Text(text_ref) => {
// let's update pointer
- let ptr_start = self.payload.as_ptr() as usize;
+ let ptr_start = self.payload.as_blob().as_ptr() as usize;
let ptr_end = text_ref.value.data as usize;
let len = ptr_end - ptr_start;
- let new_ptr = unsafe { new_payload.as_ptr().add(len) };
+ let new_ptr = unsafe { new_payload.as_blob().as_ptr().add(len) };
RefValue::Text(TextRef {
value: RawSlice::new(new_ptr, text_ref.value.len),
subtype: text_ref.subtype.clone(),
})
}
RefValue::Blob(raw_slice) => {
- let ptr_start = self.payload.as_ptr() as usize;
+ let ptr_start = self.payload.as_blob().as_ptr() as usize;
let ptr_end = raw_slice.data as usize;
let len = ptr_end - ptr_start;
- let new_ptr = unsafe { new_payload.as_ptr().add(len) };
+ let new_ptr = unsafe { new_payload.as_blob().as_ptr().add(len) };
RefValue::Blob(RawSlice::new(new_ptr, raw_slice.len))
}
};
diff --git a/core/util.rs b/core/util.rs
index 1415bcfe7..a89620950 100644
--- a/core/util.rs
+++ b/core/util.rs
@@ -3,7 +3,7 @@ use crate::{
schema::{self, Column, Schema, Type},
translate::{collate::CollationSeq, expr::walk_expr, plan::JoinOrderMember},
types::{Value, ValueType},
- LimboError, OpenFlags, Result, Statement, StepResult, SymbolTable, IO,
+ LimboError, OpenFlags, Result, Statement, StepResult, SymbolTable,
};
use std::{rc::Rc, sync::Arc};
use turso_sqlite3_parser::ast::{
@@ -51,7 +51,6 @@ struct UnparsedFromSqlIndex {
pub fn parse_schema_rows(
rows: Option,
schema: &mut Schema,
- io: Arc,
syms: &SymbolTable,
mv_tx_id: Option,
) -> Result<()> {
@@ -130,7 +129,7 @@ pub fn parse_schema_rows(
StepResult::IO => {
// TODO: How do we ensure that the I/O we submitted to
// read the schema is actually complete?
- io.run_once()?;
+ rows.run_once()?;
}
StepResult::Interrupt => break,
StepResult::Done => break,
@@ -1044,6 +1043,41 @@ pub fn parse_signed_number(expr: &Expr) -> Result {
}
}
+pub fn parse_string(expr: &Expr) -> Result {
+ match expr {
+ Expr::Name(ast::Name(s)) if s.len() >= 2 && s.starts_with("'") && s.ends_with("'") => {
+ Ok(s[1..s.len() - 1].to_string())
+ }
+ _ => Err(LimboError::InvalidArgument(format!(
+ "string parameter expected, got {:?} instead",
+ expr
+ ))),
+ }
+}
+
+#[allow(unused)]
+pub fn parse_pragma_bool(expr: &Expr) -> Result {
+ const TRUE_VALUES: &[&str] = &["yes", "true", "on"];
+ const FALSE_VALUES: &[&str] = &["no", "false", "off"];
+ if let Ok(number) = parse_signed_number(expr) {
+ if let Value::Integer(x @ (0 | 1)) = number {
+ return Ok(x != 0);
+ }
+ } else if let Expr::Name(name) = expr {
+ let ident = normalize_ident(&name.0);
+ if TRUE_VALUES.contains(&ident.as_str()) {
+ return Ok(true);
+ }
+ if FALSE_VALUES.contains(&ident.as_str()) {
+ return Ok(false);
+ }
+ }
+ Err(LimboError::InvalidArgument(
+ "boolean pragma value must be either 0|1 integer or yes|true|on|no|false|off token"
+ .to_string(),
+ ))
+}
+
// for TVF's we need these at planning time so we cannot emit translate_expr
pub fn vtable_args(args: &[ast::Expr]) -> Vec {
let mut vtable_args = Vec::new();
@@ -1076,7 +1110,7 @@ pub fn vtable_args(args: &[ast::Expr]) -> Vec {
#[cfg(test)]
pub mod tests {
use super::*;
- use turso_sqlite3_parser::ast::{self, Expr, Id, Literal, Operator::*, Type};
+ use turso_sqlite3_parser::ast::{self, Expr, Id, Literal, Name, Operator::*, Type};
#[test]
fn test_normalize_ident() {
@@ -2031,4 +2065,21 @@ pub mod tests {
Value::Float(-9.223_372_036_854_776e18)
);
}
+
+ #[test]
+ fn test_parse_pragma_bool() {
+ assert!(parse_pragma_bool(&Expr::Literal(Literal::Numeric("1".into()))).unwrap(),);
+ assert!(parse_pragma_bool(&Expr::Name(Name("true".into()))).unwrap(),);
+ assert!(parse_pragma_bool(&Expr::Name(Name("on".into()))).unwrap(),);
+ assert!(parse_pragma_bool(&Expr::Name(Name("yes".into()))).unwrap(),);
+
+ assert!(!parse_pragma_bool(&Expr::Literal(Literal::Numeric("0".into()))).unwrap(),);
+ assert!(!parse_pragma_bool(&Expr::Name(Name("false".into()))).unwrap(),);
+ assert!(!parse_pragma_bool(&Expr::Name(Name("off".into()))).unwrap(),);
+ assert!(!parse_pragma_bool(&Expr::Name(Name("no".into()))).unwrap(),);
+
+ assert!(parse_pragma_bool(&Expr::Name(Name("nono".into()))).is_err());
+ assert!(parse_pragma_bool(&Expr::Name(Name("10".into()))).is_err());
+ assert!(parse_pragma_bool(&Expr::Name(Name("-1".into()))).is_err());
+ }
}
diff --git a/core/vdbe/builder.rs b/core/vdbe/builder.rs
index 7d45c5f4b..ebad2fd24 100644
--- a/core/vdbe/builder.rs
+++ b/core/vdbe/builder.rs
@@ -12,7 +12,7 @@ use crate::{
emitter::TransactionMode,
plan::{ResultSetColumn, TableReferences},
},
- Connection, Value, VirtualTable,
+ CaptureDataChangesMode, Connection, Value, VirtualTable,
};
#[derive(Default)]
@@ -110,6 +110,7 @@ pub struct ProgramBuilder {
nested_level: usize,
init_label: BranchOffset,
start_offset: BranchOffset,
+ capture_data_changes_mode: CaptureDataChangesMode,
}
#[derive(Debug, Clone)]
@@ -149,7 +150,11 @@ pub struct ProgramBuilderOpts {
}
impl ProgramBuilder {
- pub fn new(query_mode: QueryMode, opts: ProgramBuilderOpts) -> Self {
+ pub fn new(
+ query_mode: QueryMode,
+ capture_data_changes_mode: CaptureDataChangesMode,
+ opts: ProgramBuilderOpts,
+ ) -> Self {
Self {
table_reference_counter: TableRefIdCounter::new(),
next_free_register: 1,
@@ -172,9 +177,14 @@ impl ProgramBuilder {
// These labels will be filled when `prologue()` is called
init_label: BranchOffset::Placeholder,
start_offset: BranchOffset::Placeholder,
+ capture_data_changes_mode,
}
}
+ pub fn capture_data_changes_mode(&self) -> &CaptureDataChangesMode {
+ &self.capture_data_changes_mode
+ }
+
pub fn extend(&mut self, opts: &ProgramBuilderOpts) {
self.insns.reserve(opts.approx_num_insns);
self.cursor_ref.reserve(opts.num_cursors);
@@ -291,7 +301,7 @@ impl ProgramBuilder {
});
}
- #[instrument(skip(self), level = Level::TRACE)]
+ #[instrument(skip(self), level = Level::INFO)]
pub fn emit_insn(&mut self, insn: Insn) {
let function = insn.to_function();
// This seemingly empty trace here is needed so that a function span is emmited with it
diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs
index 98c30c312..0549c52cf 100644
--- a/core/vdbe/execute.rs
+++ b/core/vdbe/execute.rs
@@ -1699,22 +1699,22 @@ pub fn op_transaction(
} else {
let current_state = conn.transaction_state.get();
let (new_transaction_state, updated) = match (current_state, write) {
- (TransactionState::Write { change_schema }, true) => {
- (TransactionState::Write { change_schema }, false)
+ (TransactionState::Write { schema_did_change }, true) => {
+ (TransactionState::Write { schema_did_change }, false)
}
- (TransactionState::Write { change_schema }, false) => {
- (TransactionState::Write { change_schema }, false)
+ (TransactionState::Write { schema_did_change }, false) => {
+ (TransactionState::Write { schema_did_change }, false)
}
(TransactionState::Read, true) => (
TransactionState::Write {
- change_schema: false,
+ schema_did_change: false,
},
true,
),
(TransactionState::Read, false) => (TransactionState::Read, false),
(TransactionState::None, true) => (
TransactionState::Write {
- change_schema: false,
+ schema_did_change: false,
},
true,
),
@@ -1766,9 +1766,9 @@ pub fn op_auto_commit(
super::StepResult::Busy => Ok(InsnFunctionStepResult::Busy),
};
}
- let change_schema =
- if let TransactionState::Write { change_schema } = conn.transaction_state.get() {
- change_schema
+ let schema_did_change =
+ if let TransactionState::Write { schema_did_change } = conn.transaction_state.get() {
+ schema_did_change
} else {
false
};
@@ -1776,7 +1776,7 @@ pub fn op_auto_commit(
if *auto_commit != conn.auto_commit.get() {
if *rollback {
// TODO(pere): add rollback I/O logic once we implement rollback journal
- pager.rollback(change_schema, &conn)?;
+ pager.rollback(schema_did_change, &conn)?;
conn.auto_commit.replace(true);
} else {
conn.auto_commit.replace(*auto_commit);
@@ -4254,6 +4254,14 @@ pub fn op_yield(
Ok(InsnFunctionStepResult::Step)
}
+#[derive(Debug, PartialEq, Copy, Clone)]
+pub enum OpInsertState {
+ Insert,
+ /// Updating last_insert_rowid may return IO, so we need a separate state for it so that we don't
+ /// start inserting the same row multiple times.
+ UpdateLastRowid,
+}
+
pub fn op_insert(
program: &Program,
state: &mut ProgramState,
@@ -4262,7 +4270,7 @@ pub fn op_insert(
mv_store: Option<&Rc>,
) -> Result {
let Insn::Insert {
- cursor,
+ cursor: cursor_id,
key_reg,
record_reg,
flag,
@@ -4271,9 +4279,27 @@ pub fn op_insert(
else {
unreachable!("unexpected Insn {:?}", insn)
};
+
+ if state.op_insert_state == OpInsertState::UpdateLastRowid {
+ let maybe_rowid = {
+ let mut cursor = state.get_cursor(*cursor_id);
+ let cursor = cursor.as_btree_mut();
+ return_if_io!(cursor.rowid())
+ };
+ if let Some(rowid) = maybe_rowid {
+ program.connection.update_last_rowid(rowid);
+
+ let prev_changes = program.n_change.get();
+ program.n_change.set(prev_changes + 1);
+ }
+ state.op_insert_state = OpInsertState::Insert;
+ state.pc += 1;
+ return Ok(InsnFunctionStepResult::Step);
+ }
+
{
- let mut cursor = state.get_cursor(*cursor);
- let cursor = cursor.as_btree_mut();
+ let mut cursor_ref = state.get_cursor(*cursor_id);
+ let cursor = cursor_ref.as_btree_mut();
let key = match &state.registers[*key_reg].get_owned_value() {
Value::Integer(i) => *i,
@@ -4293,18 +4319,19 @@ pub fn op_insert(
};
return_if_io!(cursor.insert(&BTreeKey::new_table_rowid(key, Some(record.as_ref())), true));
- // Only update last_insert_rowid for regular table inserts, not schema modifications
- if cursor.root_page() != 1 {
- if let Some(rowid) = return_if_io!(cursor.rowid()) {
- program.connection.update_last_rowid(rowid);
-
- let prev_changes = program.n_change.get();
- program.n_change.set(prev_changes + 1);
- }
- }
}
- state.pc += 1;
+ // Only update last_insert_rowid for regular table inserts, not schema modifications
+ let root_page = {
+ let mut cursor = state.get_cursor(*cursor_id);
+ let cursor = cursor.as_btree_mut();
+ cursor.root_page()
+ };
+ if root_page != 1 {
+ state.op_insert_state = OpInsertState::UpdateLastRowid;
+ } else {
+ state.pc += 1;
+ }
Ok(InsnFunctionStepResult::Step)
}
@@ -4367,6 +4394,7 @@ pub fn op_idx_delete(
cursor_id,
start_reg,
num_regs,
+ raise_error_if_no_matching_entry,
} = insn
else {
unreachable!("unexpected Insn {:?}", insn)
@@ -4382,7 +4410,7 @@ pub fn op_idx_delete(
);
match &state.op_idx_delete_state {
Some(OpIdxDeleteState::Seeking(record)) => {
- {
+ let found = {
let mut cursor = state.get_cursor(*cursor_id);
let cursor = cursor.as_btree_mut();
let found = return_if_io!(
@@ -4394,6 +4422,21 @@ pub fn op_idx_delete(
cursor.root_page(),
record
);
+ found
+ };
+
+ if !found {
+ // If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error if no matching index entry is found
+ // Also, do not raise this (self-correcting and non-critical) error if in writable_schema mode.
+ if *raise_error_if_no_matching_entry {
+ return Err(LimboError::Corrupt(format!(
+ "IdxDelete: no matching index entry found for record {:?}",
+ record
+ )));
+ }
+ state.pc += 1;
+ state.op_idx_delete_state = None;
+ return Ok(InsnFunctionStepResult::Step);
}
state.op_idx_delete_state = Some(OpIdxDeleteState::Verifying);
}
@@ -4404,12 +4447,7 @@ pub fn op_idx_delete(
return_if_io!(cursor.rowid())
};
- if rowid.is_none() {
- // If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error if no matching
- // index entry is found. This happens when running an UPDATE or DELETE statement and the
- // index entry to be updated or deleted is not found. For some uses of IdxDelete
- // (example: the EXCEPT operator) it does not matter that no matching entry is found.
- // For those cases, P5 is zero. Also, do not raise this (self-correcting and non-critical) error if in writable_schema mode.
+ if rowid.is_none() && *raise_error_if_no_matching_entry {
return Err(LimboError::Corrupt(format!(
"IdxDelete: no matching index entry found for record {:?}",
make_record(&state.registers, start_reg, num_regs)
@@ -4437,6 +4475,17 @@ pub fn op_idx_delete(
}
}
+#[derive(Debug, PartialEq, Copy, Clone)]
+pub enum OpIdxInsertState {
+ /// Optional seek step done before an unique constraint check.
+ SeekIfUnique,
+ /// Optional unique constraint check done before an insert.
+ UniqueConstraintCheck,
+ /// Main insert step. This is always performed. Usually the state machine just
+ /// skips to this step unless the insertion is made into a unique index.
+ Insert { moved_before: bool },
+}
+
pub fn op_idx_insert(
program: &Program,
state: &mut ProgramState,
@@ -4444,69 +4493,118 @@ pub fn op_idx_insert(
pager: &Rc,
mv_store: Option<&Rc>,
) -> Result {
- if let Insn::IdxInsert {
+ let Insn::IdxInsert {
cursor_id,
record_reg,
flags,
..
} = *insn
- {
- let (_, cursor_type) = program.cursor_ref.get(cursor_id).unwrap();
- let CursorType::BTreeIndex(index_meta) = cursor_type else {
- panic!("IdxInsert: not a BTree index cursor");
- };
- {
- let mut cursor = state.get_cursor(cursor_id);
- let cursor = cursor.as_btree_mut();
- let record = match &state.registers[record_reg] {
- Register::Record(ref r) => r,
- o => {
- return Err(LimboError::InternalError(format!(
- "expected record, got {:?}",
- o
- )));
- }
- };
- // To make this reentrant in case of `moved_before` = false, we need to check if the previous cursor.insert started
- // a write/balancing operation. If it did, it means we already moved to the place we wanted.
- let moved_before = if cursor.is_write_in_progress() {
- true
- } else if index_meta.unique {
- // check for uniqueness violation
- match cursor.key_exists_in_index(record)? {
- CursorResult::Ok(true) => {
- return Err(LimboError::Constraint(
- "UNIQUE constraint failed: duplicate key".into(),
- ))
- }
- CursorResult::IO => return Ok(InsnFunctionStepResult::IO),
- CursorResult::Ok(false) => {}
- };
- // uniqueness check already moved us to the correct place in the index.
- // the uniqueness check uses SeekOp::GE, which means a non-matching entry
- // will now be positioned at the insertion point where there currently is
- // a) nothing, or
- // b) the first entry greater than the key we are inserting.
- // In both cases, we can insert the new entry without moving again.
- //
- // This is re-entrant, because once we call cursor.insert() with moved_before=true,
- // we will immediately set BTreeCursor::state to CursorState::Write(WriteInfo::new()),
- // in BTreeCursor::insert_into_page; thus, if this function is called again,
- // moved_before will again be true due to cursor.is_write_in_progress() returning true.
- true
- } else {
- flags.has(IdxInsertFlags::USE_SEEK)
- };
+ else {
+ unreachable!("unexpected Insn {:?}", insn)
+ };
- // Start insertion of row. This might trigger a balance procedure which will take care of moving to different pages,
- // therefore, we don't want to seek again if that happens, meaning we don't want to return on io without moving to the following opcode
- // because it could trigger a movement to child page after a balance root which will leave the current page as the root page.
- return_if_io!(cursor.insert(&BTreeKey::new_index_key(record), moved_before));
+ let record_to_insert = match &state.registers[record_reg] {
+ Register::Record(ref r) => r,
+ o => {
+ return Err(LimboError::InternalError(format!(
+ "expected record, got {:?}",
+ o
+ )));
+ }
+ };
+
+ match state.op_idx_insert_state {
+ OpIdxInsertState::SeekIfUnique => {
+ let (_, cursor_type) = program.cursor_ref.get(cursor_id).unwrap();
+ let CursorType::BTreeIndex(index_meta) = cursor_type else {
+ panic!("IdxInsert: not a BTreeIndex cursor");
+ };
+ if !index_meta.unique {
+ state.op_idx_insert_state = OpIdxInsertState::Insert {
+ moved_before: false,
+ };
+ return Ok(InsnFunctionStepResult::Step);
+ }
+ {
+ let mut cursor = state.get_cursor(cursor_id);
+ let cursor = cursor.as_btree_mut();
+
+ return_if_io!(cursor.seek(
+ SeekKey::IndexKey(record_to_insert),
+ SeekOp::GE { eq_only: true }
+ ));
+ }
+ state.op_idx_insert_state = OpIdxInsertState::UniqueConstraintCheck;
+ Ok(InsnFunctionStepResult::Step)
+ }
+ OpIdxInsertState::UniqueConstraintCheck => {
+ let ignore_conflict = 'i: {
+ let mut cursor = state.get_cursor(cursor_id);
+ let cursor = cursor.as_btree_mut();
+ let record_opt = return_if_io!(cursor.record());
+ let Some(record) = record_opt.as_ref() else {
+ // Cursor not pointing at a record — table is empty or past last
+ break 'i false;
+ };
+ // Cursor is pointing at a record; if the index has a rowid, exclude it from the comparison since it's a pointer to the table row;
+ // UNIQUE indexes disallow duplicates like (a=1,b=2,rowid=1) and (a=1,b=2,rowid=2).
+ let existing_key = if cursor.has_rowid() {
+ &record.get_values()[..record.count().saturating_sub(1)]
+ } else {
+ record.get_values()
+ };
+ let inserted_key_vals = &record_to_insert.get_values();
+ if existing_key.len() != inserted_key_vals.len() {
+ break 'i false;
+ }
+
+ let conflict = compare_immutable(
+ existing_key,
+ inserted_key_vals,
+ cursor.key_sort_order(),
+ &cursor.collations,
+ ) == std::cmp::Ordering::Equal;
+ if conflict {
+ if flags.has(IdxInsertFlags::NO_OP_DUPLICATE) {
+ break 'i true;
+ }
+ return Err(LimboError::Constraint(
+ "UNIQUE constraint failed: duplicate key".into(),
+ ));
+ }
+
+ false
+ };
+ state.op_idx_insert_state = if ignore_conflict {
+ state.pc += 1;
+ OpIdxInsertState::SeekIfUnique
+ } else {
+ OpIdxInsertState::Insert { moved_before: true }
+ };
+ Ok(InsnFunctionStepResult::Step)
+ }
+ OpIdxInsertState::Insert { moved_before } => {
+ {
+ let mut cursor = state.get_cursor(cursor_id);
+ let cursor = cursor.as_btree_mut();
+ // To make this reentrant in case of `moved_before` = false, we need to check if the previous cursor.insert started
+ // a write/balancing operation. If it did, it means we already moved to the place we wanted.
+ let moved_before = moved_before
+ || cursor.is_write_in_progress()
+ || flags.has(IdxInsertFlags::USE_SEEK);
+ // Start insertion of row. This might trigger a balance procedure which will take care of moving to different pages,
+ // therefore, we don't want to seek again if that happens, meaning we don't want to return on io without moving to the following opcode
+ // because it could trigger a movement to child page after a balance root which will leave the current page as the root page.
+ return_if_io!(
+ cursor.insert(&BTreeKey::new_index_key(record_to_insert), moved_before)
+ );
+ }
+ state.op_idx_insert_state = OpIdxInsertState::SeekIfUnique;
+ state.pc += 1;
+ // TODO: flag optimizations, update n_change if OPFLAG_NCHANGE
+ Ok(InsnFunctionStepResult::Step)
}
- // TODO: flag optimizations, update n_change if OPFLAG_NCHANGE
- state.pc += 1;
}
- Ok(InsnFunctionStepResult::Step)
}
pub fn op_new_rowid(
@@ -4983,7 +5081,6 @@ pub fn op_parse_schema(
parse_schema_rows(
Some(stmt),
&mut new_schema,
- conn.pager.io.clone(),
&conn.syms.borrow(),
state.mv_tx_id,
)?;
@@ -4998,7 +5095,6 @@ pub fn op_parse_schema(
parse_schema_rows(
Some(stmt),
&mut new_schema,
- conn.pager.io.clone(),
&conn.syms.borrow(),
state.mv_tx_id,
)?;
@@ -5070,8 +5166,8 @@ pub fn op_set_cookie(
Cookie::SchemaVersion => {
// we update transaction state to indicate that the schema has changed
match program.connection.transaction_state.get() {
- TransactionState::Write { change_schema } => {
- program.connection.transaction_state.set(TransactionState::Write { change_schema: true });
+ TransactionState::Write { schema_did_change } => {
+ program.connection.transaction_state.set(TransactionState::Write { schema_did_change: true });
},
TransactionState::Read => unreachable!("invalid transaction state for SetCookie: TransactionState::Read, should be write"),
TransactionState::None => unreachable!("invalid transaction state for SetCookie: TransactionState::None, should be write"),
diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs
index 933b82ee2..13efe1322 100644
--- a/core/vdbe/explain.rs
+++ b/core/vdbe/explain.rs
@@ -1120,13 +1120,14 @@ pub fn insn_to_str(
cursor_id,
start_reg,
num_regs,
+ raise_error_if_no_matching_entry,
} => (
"IdxDelete",
*cursor_id as i32,
*start_reg as i32,
*num_regs as i32,
Value::build_text(""),
- 0,
+ *raise_error_if_no_matching_entry as u16,
"".to_string(),
),
Insn::NewRowid {
diff --git a/core/vdbe/insn.rs b/core/vdbe/insn.rs
index 2927f5009..f094f3db0 100644
--- a/core/vdbe/insn.rs
+++ b/core/vdbe/insn.rs
@@ -63,6 +63,7 @@ impl IdxInsertFlags {
pub const APPEND: u8 = 0x01; // Hint: insert likely at the end
pub const NCHANGE: u8 = 0x02; // Increment the change counter
pub const USE_SEEK: u8 = 0x04; // Skip seek if last one was same key
+ pub const NO_OP_DUPLICATE: u8 = 0x08; // Do not error on duplicate key
pub fn new() -> Self {
IdxInsertFlags(0)
}
@@ -93,6 +94,14 @@ impl IdxInsertFlags {
}
self
}
+ /// If this is set, we will not error on duplicate key.
+ /// This is a bit of a hack we use to make ephemeral indexes for UNION work --
+ /// instead we should allow overwriting index interior cells, which we currently don't;
+ /// this should (and will) be fixed in a future PR.
+ pub fn no_op_duplicate(mut self) -> Self {
+ self.0 |= IdxInsertFlags::NO_OP_DUPLICATE;
+ self
+ }
}
#[derive(Clone, Copy, Debug, Default)]
@@ -733,10 +742,15 @@ pub enum Insn {
cursor_id: CursorID,
},
+ /// If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error if no matching index entry
+ /// is found. This happens when running an UPDATE or DELETE statement and the index entry to
+ /// be updated or deleted is not found. For some uses of IdxDelete (example: the EXCEPT operator)
+ /// it does not matter that no matching entry is found. For those cases, P5 is zero.
IdxDelete {
start_reg: usize,
num_regs: usize,
cursor_id: CursorID,
+ raise_error_if_no_matching_entry: bool, // P5
},
NewRowid {
diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs
index 3663afa6f..a9a813388 100644
--- a/core/vdbe/mod.rs
+++ b/core/vdbe/mod.rs
@@ -29,6 +29,8 @@ use crate::{
function::{AggFunc, FuncCtx},
storage::{pager::PagerCacheflushStatus, sqlite3_ondisk::SmallVec},
translate::plan::TableReferences,
+ vdbe::execute::OpIdxInsertState,
+ vdbe::execute::OpInsertState,
};
use crate::{
@@ -250,6 +252,8 @@ pub struct ProgramState {
op_idx_delete_state: Option,
op_integrity_check_state: OpIntegrityCheckState,
op_open_ephemeral_state: OpOpenEphemeralState,
+ op_idx_insert_state: OpIdxInsertState,
+ op_insert_state: OpInsertState,
}
impl ProgramState {
@@ -276,6 +280,8 @@ impl ProgramState {
op_idx_delete_state: None,
op_integrity_check_state: OpIntegrityCheckState::Start,
op_open_ephemeral_state: OpOpenEphemeralState::Start,
+ op_idx_insert_state: OpIdxInsertState::SeekIfUnique,
+ op_insert_state: OpInsertState::Insert,
}
}
@@ -334,7 +340,11 @@ impl Register {
pub fn get_owned_value(&self) -> &Value {
match self {
Register::Value(v) => v,
- _ => unreachable!(),
+ Register::Record(r) => {
+ assert!(!r.is_invalidated());
+ r.as_blob_value()
+ }
+ _ => panic!("register holds unexpected value: {:?}", self),
}
}
}
@@ -368,6 +378,7 @@ pub struct Program {
}
impl Program {
+ #[instrument(skip_all, level = Level::INFO)]
pub fn step(
&self,
state: &mut ProgramState,
@@ -375,6 +386,14 @@ impl Program {
pager: Rc,
) -> Result {
loop {
+ if self.connection.closed.get() {
+ // Connection is closed for whatever reason, rollback the transaction.
+ let state = self.connection.transaction_state.get();
+ if let TransactionState::Write { schema_did_change } = state {
+ pager.rollback(schema_did_change, &self.connection)?
+ }
+ return Err(LimboError::InternalError("Connection closed".to_string()));
+ }
if state.is_interrupted() {
return Ok(StepResult::Interrupt);
}
@@ -382,8 +401,14 @@ impl Program {
let _ = state.result_row.take();
let (insn, insn_function) = &self.insns[state.pc as usize];
trace_insn(self, state.pc as InsnReference, insn);
- let res = insn_function(self, state, insn, &pager, mv_store.as_ref())?;
- match res {
+ let res = insn_function(self, state, insn, &pager, mv_store.as_ref());
+ if res.is_err() {
+ let state = self.connection.transaction_state.get();
+ if let TransactionState::Write { schema_did_change } = state {
+ pager.rollback(schema_did_change, &self.connection)?
+ }
+ }
+ match res? {
InsnFunctionStepResult::Step => {}
InsnFunctionStepResult::Done => return Ok(StepResult::Done),
InsnFunctionStepResult::IO => return Ok(StepResult::IO),
@@ -394,7 +419,7 @@ impl Program {
}
}
- #[instrument(skip_all, level = Level::TRACE)]
+ #[instrument(skip_all, level = Level::INFO)]
pub fn commit_txn(
&self,
pager: Rc,
@@ -422,7 +447,8 @@ impl Program {
program_state.commit_state
);
if program_state.commit_state == CommitState::Committing {
- let TransactionState::Write { change_schema } = connection.transaction_state.get()
+ let TransactionState::Write { schema_did_change } =
+ connection.transaction_state.get()
else {
unreachable!("invalid state for write commit step")
};
@@ -431,18 +457,18 @@ impl Program {
&mut program_state.commit_state,
&connection,
rollback,
- change_schema,
+ schema_did_change,
)
} else if auto_commit {
let current_state = connection.transaction_state.get();
tracing::trace!("Auto-commit state: {:?}", current_state);
match current_state {
- TransactionState::Write { change_schema } => self.step_end_write_txn(
+ TransactionState::Write { schema_did_change } => self.step_end_write_txn(
&pager,
&mut program_state.commit_state,
&connection,
rollback,
- change_schema,
+ schema_did_change,
),
TransactionState::Read => {
connection.transaction_state.replace(TransactionState::None);
@@ -460,26 +486,32 @@ impl Program {
}
}
- #[instrument(skip(self, pager, connection), level = Level::TRACE)]
+ #[instrument(skip(self, pager, connection), level = Level::INFO)]
fn step_end_write_txn(
&self,
pager: &Rc,
commit_state: &mut CommitState,
connection: &Connection,
rollback: bool,
- change_schema: bool,
+ schema_did_change: bool,
) -> Result {
let cacheflush_status = pager.end_tx(
rollback,
- change_schema,
+ schema_did_change,
connection,
connection.wal_checkpoint_disabled.get(),
)?;
match cacheflush_status {
- PagerCacheflushStatus::Done(_) => {
+ PagerCacheflushStatus::Done(status) => {
if self.change_cnt_on {
self.connection.set_changes(self.n_change.get());
}
+ if matches!(
+ status,
+ crate::storage::pager::PagerCacheflushResult::Rollback
+ ) {
+ pager.rollback(schema_did_change, connection)?;
+ }
connection.transaction_state.replace(TransactionState::None);
*commit_state = CommitState::Ready;
}
@@ -553,7 +585,7 @@ fn make_record(registers: &[Register], start_reg: &usize, count: &usize) -> Immu
ImmutableRecord::from_registers(regs, regs.len())
}
-#[instrument(skip(program), level = Level::TRACE)]
+#[instrument(skip(program), level = Level::INFO)]
fn trace_insn(program: &Program, addr: InsnReference, insn: &Insn) {
if !tracing::enabled!(tracing::Level::TRACE) {
return;
diff --git a/fuzz/fuzz_targets/expression.rs b/fuzz/fuzz_targets/expression.rs
index cf56eebb4..d7fd62d92 100644
--- a/fuzz/fuzz_targets/expression.rs
+++ b/fuzz/fuzz_targets/expression.rs
@@ -194,7 +194,7 @@ fn do_fuzz(expr: Expr) -> Result> {
loop {
use turso_core::StepResult;
match stmt.step()? {
- StepResult::IO => io.run_once()?,
+ StepResult::IO => stmt.run_once()?,
StepResult::Row => {
let row = stmt.row().unwrap();
assert_eq!(row.len(), 1, "expr: {:?}", expr);
diff --git a/perf/connection/gen-database.py b/perf/connection/gen-database.py
index 821e17747..b0f919487 100755
--- a/perf/connection/gen-database.py
+++ b/perf/connection/gen-database.py
@@ -45,4 +45,3 @@ def main() -> None:
if __name__ == "__main__":
main()
-
diff --git a/perf/connection/plot.py b/perf/connection/plot.py
index e39360569..28bc25db4 100755
--- a/perf/connection/plot.py
+++ b/perf/connection/plot.py
@@ -52,4 +52,3 @@ def main() -> None:
if __name__ == "__main__":
main()
-
diff --git a/scripts/gen-changelog.py b/scripts/gen-changelog.py
new file mode 100755
index 000000000..3044c2384
--- /dev/null
+++ b/scripts/gen-changelog.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+import re
+import subprocess
+import sys
+from collections import defaultdict
+
+
+def get_git_merges(prev_version):
+ """Get merge commits since the previous version tag."""
+ try:
+ command = f"git log {prev_version}..HEAD | grep 'Merge '"
+ result = subprocess.run(command, shell=True, check=True, text=True, capture_output=True)
+
+ merge_lines = []
+ for line in result.stdout.strip().split("\n"):
+ if not line.strip() or "Merge:" in line:
+ continue
+
+ # Extract the commit message and author
+ match = re.search(r"Merge '([^']+)' from ([^(]+)", line)
+ if match:
+ message = match.group(1).strip()
+ author = match.group(2).strip()
+ merge_lines.append((message, author))
+
+ return merge_lines
+ except subprocess.CalledProcessError as e:
+ print(f"Error: Failed to get git merge logs: {e}")
+ return []
+
+
+def categorize_commits(merge_lines):
+ """Categorize commits into Added, Updated, Fixed."""
+ categories = defaultdict(list)
+
+ for message, author in merge_lines:
+ # Format the line for our output
+ formatted_line = f"* {message} ({author})"
+
+ # Categorize based on keywords in the commit message
+ message_lower = message.lower()
+ if re.search(r"add|new|implement|support|initial|introduce", message_lower):
+ categories["Added"].append(formatted_line)
+ elif re.search(r"fix|bug|issue|error|crash|resolve|typo", message_lower):
+ categories["Fixed"].append(formatted_line)
+ else:
+ categories["Updated"].append(formatted_line)
+
+ return categories
+
+
+def format_changelog(categories):
+ """Format the categorized commits into a changelog."""
+ changelog = "## Unreleased\n"
+
+ for category in ["Added", "Updated", "Fixed"]:
+ changelog += f"### {category}\n"
+
+ if not categories[category]:
+ changelog += "\n"
+ continue
+
+ for commit_message in categories[category]:
+ changelog += f"{commit_message}\n"
+
+ changelog += "\n"
+
+ return changelog
+
+
+def main():
+ if len(sys.argv) != 2:
+ print("Usage: python changelog_generator.py ")
+ print("Example: python changelog_generator.py v0.0.17")
+ sys.exit(1)
+
+ prev_version = sys.argv[1]
+
+ # Get merge commits since previous version
+ merge_lines = get_git_merges(prev_version)
+
+ if not merge_lines:
+ print(f"No merge commits found since {prev_version}")
+ return
+
+ # Categorize commits
+ categories = categorize_commits(merge_lines)
+
+ # Format changelog
+ changelog = format_changelog(categories)
+
+ # Output changelog
+ print(changelog)
+
+ # Optionally write to file
+ write_to_file = input("Write to CHANGELOG.md? (y/n): ")
+ if write_to_file.lower() == "y":
+ try:
+ with open("CHANGELOG.md", "r") as f:
+ content = f.read()
+ with open("CHANGELOG.md", "w") as f:
+ f.write(changelog + content)
+ print("Changelog written to CHANGELOG.md")
+ except FileNotFoundError:
+ with open("CHANGELOG.md", "w") as f:
+ f.write(changelog)
+ print("Created new CHANGELOG.md file")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/limbo-sqlite3 b/scripts/limbo-sqlite3
index a9d0e08f7..1e8e63290 100755
--- a/scripts/limbo-sqlite3
+++ b/scripts/limbo-sqlite3
@@ -1,8 +1,17 @@
#!/bin/bash
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Go to the project root (one level up from scripts/)
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+TURSODB="$PROJECT_ROOT/target/debug/tursodb"
+
+# Add experimental features for testing
+EXPERIMENTAL_FLAGS=""
+
# if RUST_LOG is non-empty, enable tracing output
if [ -n "$RUST_LOG" ]; then
- target/debug/tursodb -m list -t testing/test.log "$@"
+ "$TURSODB" -m list -q $EXPERIMENTAL_FLAGS -t testing/test.log "$@"
else
- target/debug/tursodb -m list "$@"
+ "$TURSODB" -m list -q $EXPERIMENTAL_FLAGS "$@"
fi
diff --git a/scripts/limbo-sqlite3-index-experimental b/scripts/limbo-sqlite3-index-experimental
index d8250469d..a0ec545aa 100755
--- a/scripts/limbo-sqlite3-index-experimental
+++ b/scripts/limbo-sqlite3-index-experimental
@@ -1,8 +1,17 @@
#!/bin/bash
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Go to the project root (one level up from scripts/)
+PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
+TURSODB="$PROJECT_ROOT/target/debug/tursodb"
+
+# Add experimental features for testing
+EXPERIMENTAL_FLAGS="--experimental-indexes"
+
# if RUST_LOG is non-empty, enable tracing output
if [ -n "$RUST_LOG" ]; then
- target/debug/tursodb --experimental-indexes -m list -t testing/test.log "$@"
+ "$TURSODB" -m list -q $EXPERIMENTAL_FLAGS -t testing/test.log "$@"
else
- target/debug/tursodb --experimental-indexes -m list "$@"
+ "$TURSODB" -m list -q $EXPERIMENTAL_FLAGS "$@"
fi
diff --git a/scripts/run-sim b/scripts/run-sim
index 9985a70f3..661062c54 100755
--- a/scripts/run-sim
+++ b/scripts/run-sim
@@ -2,28 +2,8 @@
set -e
-iterations=""
-while [[ $# -gt 0 ]]; do
- case $1 in
- --iterations)
- iterations="$2"
- shift 2
- ;;
- *)
- echo "Unknown option: $1"
- echo "Usage: $0 [--max-iterations N]"
- exit 1
- ;;
- esac
-done
-
-if [[ -n "$iterations" ]]; then
- echo "Running limbo_sim for $iterations iterations..."
- for ((i=1; i<=iterations; i++)); do
- echo "Iteration $i of $iterations"
- cargo run -p limbo_sim
- done
- echo "Completed $iterations iterations"
+if [[ -n "$@" ]]; then
+ cargo run -p limbo_sim -- "$@"
else
echo "Running limbo_sim in infinite loop..."
while true; do
diff --git a/simulator-docker-runner/docker-entrypoint.simulator.ts b/simulator-docker-runner/docker-entrypoint.simulator.ts
index e5a2c133b..6c977c221 100644
--- a/simulator-docker-runner/docker-entrypoint.simulator.ts
+++ b/simulator-docker-runner/docker-entrypoint.simulator.ts
@@ -2,6 +2,7 @@
import { spawn } from "bun";
import { GithubClient } from "./github";
+import { SlackClient } from "./slack";
import { extractFailureInfo } from "./logParse";
import { randomSeed } from "./random";
@@ -12,12 +13,14 @@ const PER_RUN_TIMEOUT_SECONDS = Number.isInteger(Number(process.env.PER_RUN_TIME
const LOG_TO_STDOUT = process.env.LOG_TO_STDOUT === "true";
const github = new GithubClient();
+const slack = new SlackClient();
process.env.RUST_BACKTRACE = "1";
console.log("Starting limbo_sim in a loop...");
console.log(`Git hash: ${github.GIT_HASH}`);
console.log(`GitHub issues enabled: ${github.mode === 'real'}`);
+console.log(`Slack notifications enabled: ${slack.mode === 'real'}`);
console.log(`Time limit: ${TIME_LIMIT_MINUTES} minutes`);
console.log(`Log simulator output to stdout: ${LOG_TO_STDOUT}`);
console.log(`Sleep between runs: ${SLEEP_BETWEEN_RUNS_SECONDS} seconds`);
@@ -69,7 +72,7 @@ const timeouter = (seconds: number, runNumber: number) => {
return timeouterPromise;
}
-const run = async (seed: string, bin: string, args: string[]) => {
+const run = async (seed: string, bin: string, args: string[]): Promise => {
const proc = spawn([`/app/${bin}`, ...args], {
stdout: LOG_TO_STDOUT ? "inherit" : "pipe",
stderr: LOG_TO_STDOUT ? "inherit" : "pipe",
@@ -77,6 +80,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
});
const timeout = timeouter(PER_RUN_TIMEOUT_SECONDS, runNumber);
+ let issuePosted = false;
try {
const exitCode = await Promise.race([proc.exited, timeout]);
@@ -102,6 +106,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
command: args.join(" "),
stackTrace: failureInfo,
});
+ issuePosted = true;
} else {
await github.postGitHubIssue({
type: "assertion",
@@ -109,6 +114,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
command: args.join(" "),
failureInfo,
});
+ issuePosted = true;
}
} catch (err2) {
console.error(`Error extracting simulator seed and stack trace: ${err2}`);
@@ -134,6 +140,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
command: args.join(" "),
output: lastLines,
});
+ issuePosted = true;
} else {
throw err;
}
@@ -141,12 +148,16 @@ const run = async (seed: string, bin: string, args: string[]) => {
// @ts-ignore
timeout.clear();
}
+
+ return issuePosted;
}
// Main execution loop
const startTime = new Date();
const limboSimArgs = process.argv.slice(2);
let runNumber = 0;
+let totalIssuesPosted = 0;
+
while (new Date().getTime() - startTime.getTime() < TIME_LIMIT_MINUTES * 60 * 1000) {
const timestamp = new Date().toISOString();
const args = [...limboSimArgs];
@@ -160,13 +171,29 @@ while (new Date().getTime() - startTime.getTime() < TIME_LIMIT_MINUTES * 60 * 10
args.push(...loop);
console.log(`[${timestamp}]: Running "limbo_sim ${args.join(" ")}" - (seed ${seed}, run number ${runNumber})`);
- await run(seed, "limbo_sim", args);
+ const issuePosted = await run(seed, "limbo_sim", args);
+
+ if (issuePosted) {
+ totalIssuesPosted++;
+ }
runNumber++;
SLEEP_BETWEEN_RUNS_SECONDS > 0 && (await sleep(SLEEP_BETWEEN_RUNS_SECONDS));
}
+// Post summary to Slack after the run completes
+const endTime = new Date();
+const timeElapsed = Math.floor((endTime.getTime() - startTime.getTime()) / 1000);
+console.log(`\nRun completed! Total runs: ${runNumber}, Issues posted: ${totalIssuesPosted}, Time elapsed: ${timeElapsed}s`);
+
+await slack.postRunSummary({
+ totalRuns: runNumber,
+ issuesPosted: totalIssuesPosted,
+ timeElapsed,
+ gitHash: github.GIT_HASH,
+});
+
async function sleep(sec: number) {
return new Promise(resolve => setTimeout(resolve, sec * 1000));
}
diff --git a/simulator-docker-runner/slack.ts b/simulator-docker-runner/slack.ts
new file mode 100644
index 000000000..2e3356d28
--- /dev/null
+++ b/simulator-docker-runner/slack.ts
@@ -0,0 +1,154 @@
+export class SlackClient {
+ private botToken: string;
+ private channel: string;
+ mode: 'real' | 'dry-run';
+
+ constructor() {
+ this.botToken = process.env.SLACK_BOT_TOKEN || "";
+ this.channel = process.env.SLACK_CHANNEL || "#simulator-results-fake";
+ this.mode = this.botToken ? 'real' : 'dry-run';
+
+ if (this.mode === 'real') {
+ if (this.channel === "#simulator-results-fake") {
+ throw new Error("SLACK_CHANNEL must be set to a real channel when running in real mode");
+ }
+ } else {
+ if (this.channel !== "#simulator-results-fake") {
+ throw new Error("SLACK_CHANNEL must be set to #simulator-results-fake when running in dry-run mode");
+ }
+ }
+ }
+
+ async postRunSummary(stats: {
+ totalRuns: number;
+ issuesPosted: number;
+ timeElapsed: number;
+ gitHash: string;
+ }): Promise {
+ const blocks = this.createSummaryBlocks(stats);
+ const fallbackText = this.createFallbackText(stats);
+
+ if (this.mode === 'dry-run') {
+ console.log(`Dry-run mode: Would post to Slack channel ${this.channel}`);
+ console.log(`Fallback text: ${fallbackText}`);
+ console.log(`Blocks: ${JSON.stringify(blocks, null, 2)}`);
+ return;
+ }
+
+ try {
+ const response = await fetch('https://slack.com/api/chat.postMessage', {
+ method: 'POST',
+ headers: {
+ 'Authorization': `Bearer ${this.botToken}`,
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify({
+ channel: this.channel,
+ text: fallbackText,
+ blocks: blocks,
+ }),
+ });
+
+ const result = await response.json();
+
+ if (!result.ok) {
+ console.error(`Failed to post to Slack: ${result.error}`);
+ return;
+ }
+
+ console.log(`Successfully posted summary to Slack channel ${this.channel}`);
+ } catch (error) {
+ console.error(`Error posting to Slack: ${error}`);
+ }
+ }
+
+ private createFallbackText(stats: {
+ totalRuns: number;
+ issuesPosted: number;
+ timeElapsed: number;
+ gitHash: string;
+ }): string {
+ const { totalRuns, issuesPosted, timeElapsed, gitHash } = stats;
+ const hours = Math.floor(timeElapsed / 3600);
+ const minutes = Math.floor((timeElapsed % 3600) / 60);
+ const seconds = Math.floor(timeElapsed % 60);
+ const timeString = `${hours}h ${minutes}m ${seconds}s`;
+ const gitShortHash = gitHash.substring(0, 7);
+
+ return `🤖 Turso Simulator Run Complete - ${totalRuns} runs, ${issuesPosted} issues posted, ${timeString} elapsed (${gitShortHash})`;
+ }
+
+ private createSummaryBlocks(stats: {
+ totalRuns: number;
+ issuesPosted: number;
+ timeElapsed: number;
+ gitHash: string;
+ }): any[] {
+ const { totalRuns, issuesPosted, timeElapsed, gitHash } = stats;
+ const hours = Math.floor(timeElapsed / 3600);
+ const minutes = Math.floor((timeElapsed % 3600) / 60);
+ const seconds = Math.floor(timeElapsed % 60);
+ const timeString = `${hours}h ${minutes}m ${seconds}s`;
+
+ const statusEmoji = issuesPosted > 0 ? "🔴" : "✅";
+ const statusText = issuesPosted > 0 ? `${issuesPosted} issues found` : "No issues found";
+ const gitShortHash = gitHash.substring(0, 7);
+
+ return [
+ {
+ "type": "header",
+ "text": {
+ "type": "plain_text",
+ "text": "🤖 Turso Simulator Run Complete"
+ }
+ },
+ {
+ "type": "section",
+ "text": {
+ "type": "mrkdwn",
+ "text": `${statusEmoji} *${statusText}*`
+ }
+ },
+ {
+ "type": "divider"
+ },
+ {
+ "type": "section",
+ "fields": [
+ {
+ "type": "mrkdwn",
+ "text": `*Total runs:*\n${totalRuns}`
+ },
+ {
+ "type": "mrkdwn",
+ "text": `*Issues posted:*\n${issuesPosted}`
+ },
+ {
+ "type": "mrkdwn",
+ "text": `*Time elapsed:*\n${timeString}`
+ },
+ {
+ "type": "mrkdwn",
+ "text": `*Git hash:*\n\`${gitShortHash}\``
+ },
+ {
+ "type": "mrkdwn",
+ "text": `*See open issues:*\n`
+ }
+ ]
+ },
+ {
+ "type": "divider"
+ },
+ {
+ "type": "context",
+ "elements": [
+ {
+ "type": "mrkdwn",
+ "text": `Full git hash: \`${gitHash}\` | Timestamp: ${new Date().toISOString()}`
+ }
+ ]
+ }
+ ];
+ }
+}
\ No newline at end of file
diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs
index 8fa2dcf02..fa5ba86c0 100644
--- a/simulator/generation/plan.rs
+++ b/simulator/generation/plan.rs
@@ -7,14 +7,14 @@ use std::{
};
use serde::{Deserialize, Serialize};
-use turso_core::{Connection, Result, StepResult, IO};
+use turso_core::{Connection, Result, StepResult};
use crate::{
model::{
query::{update::Update, Create, CreateIndex, Delete, Drop, Insert, Query, Select},
table::SimValue,
},
- runner::{env::SimConnection, io::SimulatorIO},
+ runner::env::SimConnection,
SimulatorEnv,
};
@@ -266,7 +266,7 @@ impl Display for Interaction {
}
}
-type AssertionFunc = dyn Fn(&Vec