diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..951727346 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +*target diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 50a273e7b..4cddab1e3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -50,7 +50,7 @@ jobs: - uses: actions/checkout@v3 - name: Clippy run: | - cargo clippy --workspace --all-features --all-targets --exclude limbo-wasm -- -A unused-variables --deny=warnings + cargo clippy --workspace --all-features --all-targets -- -A unused-variables --deny=warnings simulator: runs-on: blacksmith-4vcpu-ubuntu-2404 @@ -81,10 +81,6 @@ jobs: - name: Test run: make test timeout-minutes: 20 - # - uses: "./.github/shared/install_sqlite" - # - name: Test with index enabled - # run: SQLITE_EXEC="scripts/limbo-sqlite3-index-experimental" make test - # timeout-minutes: 20 test-sqlite: runs-on: blacksmith-4vcpu-ubuntu-2404 steps: diff --git a/COMPAT.md b/COMPAT.md index a436e80ef..a988ca829 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -415,8 +415,8 @@ Modifiers: | Opcode | Status | Comment | |----------------|--------|---------| | Add | Yes | | -| AddImm | No | | -| Affinity | No | | +| AddImm | Yes | | +| Affinity | Yes | | | AggFinal | Yes | | | AggStep | Yes | | | AggStep | Yes | | @@ -427,6 +427,7 @@ Modifiers: | BitOr | Yes | | | Blob | Yes | | | BeginSubrtn | Yes | | +| Cast | Yes | | | Checkpoint | Yes | | | Clear | No | | | Close | Yes | | @@ -473,7 +474,6 @@ Modifiers: | Init | Yes | | | InitCoroutine | Yes | | | Insert | Yes | | -| InsertInt | No | | | Int64 | Yes | | | Integer | Yes | | | IntegrityCk | Yes | | @@ -550,15 +550,10 @@ Modifiers: | SorterNext | Yes | | | SorterOpen | Yes | | | SorterSort | Yes | | -| String | No | | +| String | NotNeeded | SQLite uses String for sized strings and String8 for null-terminated. All our strings are sized | | String8 | Yes | | | Subtract | Yes | | | TableLock | No | | -| ToBlob | No | | -| ToInt | No | | -| ToNumeric | No | | -| ToReal | No | | -| ToText | No | | | Trace | No | | | Transaction | Yes | | | VBegin | No | | @@ -572,7 +567,6 @@ Modifiers: | VUpdate | Yes | | | Vacuum | No | | | Variable | Yes | | -| VerifyCookie | No | | | Yield | Yes | | | ZeroOrNull | Yes | | diff --git a/Cargo.lock b/Cargo.lock index a11eced7e..9ce4e9170 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -382,9 +382,23 @@ checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" -version = "1.22.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ecc273b49b3205b83d648f0690daa588925572cc5063745bfe547fe7ec8e1a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] [[package]] name = "byteorder" @@ -656,7 +670,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core_tester" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "anyhow", "assert_cmd", @@ -668,8 +682,10 @@ dependencies = [ "rusqlite", "tempfile", "test-log", + "tokio", "tracing", "tracing-subscriber", + "turso", "turso_core", "zerocopy 0.8.26", ] @@ -1600,7 +1616,7 @@ dependencies = [ "hyper", "libc", "pin-project-lite", - "socket2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2100,14 +2116,14 @@ dependencies = [ [[package]] name = "limbo-go" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "turso_core", ] [[package]] name = "limbo_completion" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "mimalloc", "turso_ext", @@ -2115,7 +2131,7 @@ dependencies = [ [[package]] name = "limbo_crypto" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "blake3", "data-encoding", @@ -2128,7 +2144,7 @@ dependencies = [ [[package]] name = "limbo_csv" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "csv", "mimalloc", @@ -2138,7 +2154,7 @@ dependencies = [ [[package]] name = "limbo_ipaddr" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "ipnetwork", "mimalloc", @@ -2147,7 +2163,7 @@ dependencies = [ [[package]] name = "limbo_percentile" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "mimalloc", "turso_ext", @@ -2155,7 +2171,7 @@ dependencies = [ [[package]] name = "limbo_regexp" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "mimalloc", "regex", @@ -2164,7 +2180,7 @@ dependencies = [ [[package]] name = "limbo_sim" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "anarchist-readable-name-generator-lib", "anyhow", @@ -2191,7 +2207,7 @@ dependencies = [ [[package]] name = "limbo_sqlite3" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "env_logger 0.11.7", "libc", @@ -2204,7 +2220,7 @@ dependencies = [ [[package]] name = "limbo_sqlite_test_ext" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "cc", ] @@ -2676,6 +2692,15 @@ version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" +[[package]] +name = "pack1" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6e7cd9bd638dc2c831519a0caa1c006cab771a92b1303403a8322773c5b72d6" +dependencies = [ + "bytemuck", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -2908,7 +2933,7 @@ dependencies = [ [[package]] name = "py-turso" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "anyhow", "pyo3", @@ -3619,6 +3644,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "sorted-vec" version = "0.8.6" @@ -3966,9 +4001,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.46.1" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" dependencies = [ "backtrace", "bytes", @@ -3979,9 +4014,9 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2", + "socket2 0.6.0", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4134,7 +4169,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "turso" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", @@ -4146,7 +4181,7 @@ dependencies = [ [[package]] name = "turso-java" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "jni", "thiserror 2.0.12", @@ -4155,7 +4190,7 @@ dependencies = [ [[package]] name = "turso-sync" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "ctor", "futures", @@ -4182,7 +4217,7 @@ dependencies = [ [[package]] name = "turso_cli" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "anyhow", "cfg-if", @@ -4214,11 +4249,12 @@ dependencies = [ [[package]] name = "turso_core" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "antithesis_sdk", "bitflags 2.9.0", "built", + "bytemuck", "cfg_block", "chrono", "criterion", @@ -4236,6 +4272,7 @@ dependencies = [ "memory-stats", "miette", "mimalloc", + "pack1", "parking_lot", "paste", "polling", @@ -4267,7 +4304,7 @@ dependencies = [ [[package]] name = "turso_dart" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "flutter_rust_bridge", "turso_core", @@ -4275,7 +4312,7 @@ dependencies = [ [[package]] name = "turso_ext" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "chrono", "getrandom 0.3.2", @@ -4284,7 +4321,7 @@ dependencies = [ [[package]] name = "turso_ext_tests" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "env_logger 0.11.7", "lazy_static", @@ -4295,7 +4332,7 @@ dependencies = [ [[package]] name = "turso_macros" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "proc-macro2", "quote", @@ -4304,7 +4341,7 @@ dependencies = [ [[package]] name = "turso_node" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "napi", "napi-build", @@ -4315,7 +4352,7 @@ dependencies = [ [[package]] name = "turso_sqlite3_parser" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "bitflags 2.9.0", "cc", @@ -4333,7 +4370,7 @@ dependencies = [ [[package]] name = "turso_stress" -version = "0.1.3" +version = "0.1.4-pre.1" dependencies = [ "anarchist-readable-name-generator-lib", "antithesis_sdk", diff --git a/Cargo.toml b/Cargo.toml index a562f1e87..ea43682b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,26 +31,26 @@ members = [ exclude = ["perf/latency/limbo"] [workspace.package] -version = "0.1.3" +version = "0.1.4-pre.1" authors = ["the Limbo authors"] edition = "2021" license = "MIT" repository = "https://github.com/tursodatabase/turso" [workspace.dependencies] -turso = { path = "bindings/rust", version = "0.1.3" } -limbo_completion = { path = "extensions/completion", version = "0.1.3" } -turso_core = { path = "core", version = "0.1.3" } -limbo_crypto = { path = "extensions/crypto", version = "0.1.3" } -limbo_csv = { path = "extensions/csv", version = "0.1.3" } -turso_ext = { path = "extensions/core", version = "0.1.3" } -turso_ext_tests = { path = "extensions/tests", version = "0.1.3" } -limbo_ipaddr = { path = "extensions/ipaddr", version = "0.1.3" } -turso_macros = { path = "macros", version = "0.1.3" } -limbo_percentile = { path = "extensions/percentile", version = "0.1.3" } -limbo_regexp = { path = "extensions/regexp", version = "0.1.3" } -turso_sqlite3_parser = { path = "vendored/sqlite3-parser", version = "0.1.3" } -limbo_uuid = { path = "extensions/uuid", version = "0.1.3" } +turso = { path = "bindings/rust", version = "0.1.4-pre.1" } +limbo_completion = { path = "extensions/completion", version = "0.1.4-pre.1" } +turso_core = { path = "core", version = "0.1.4-pre.1" } +limbo_crypto = { path = "extensions/crypto", version = "0.1.4-pre.1" } +limbo_csv = { path = "extensions/csv", version = "0.1.4-pre.1" } +turso_ext = { path = "extensions/core", version = "0.1.4-pre.1" } +turso_ext_tests = { path = "extensions/tests", version = "0.1.4-pre.1" } +limbo_ipaddr = { path = "extensions/ipaddr", version = "0.1.4-pre.1" } +turso_macros = { path = "macros", version = "0.1.4-pre.1" } +limbo_percentile = { path = "extensions/percentile", version = "0.1.4-pre.1" } +limbo_regexp = { path = "extensions/regexp", version = "0.1.4-pre.1" } +turso_sqlite3_parser = { path = "vendored/sqlite3-parser", version = "0.1.4-pre.1" } +limbo_uuid = { path = "extensions/uuid", version = "0.1.4-pre.1" } strum = { version = "0.26", features = ["derive"] } strum_macros = "0.26" serde = "1.0" diff --git a/Dockerfile.cli b/Dockerfile.cli new file mode 100644 index 000000000..474b73c34 --- /dev/null +++ b/Dockerfile.cli @@ -0,0 +1,20 @@ +FROM rust:1.88.0 as builder + +WORKDIR /app + +# Copy the actual source code +COPY . . + +# Build the CLI binary +RUN cargo build --release --package turso_cli + +# Runtime stage +FROM rust:1.88.0-slim + +WORKDIR /app + +# Copy the built binary +COPY --from=builder /app/target/release/tursodb /usr/local/bin/ + +# Set the entrypoint +ENTRYPOINT ["tursodb"] diff --git a/Makefile b/Makefile index a507700a0..ea3d3ece4 100644 --- a/Makefile +++ b/Makefile @@ -150,3 +150,46 @@ bench-exclude-tpc-h: cargo bench $$benchmarks; \ fi .PHONY: bench-exclude-tpc-h + +docker-cli-build: + docker build -f Dockerfile.cli -t turso-cli . + +docker-cli-run: + docker run -it -v ./:/app turso-cli + +merge-pr: +ifndef PR + $(error PR is required. Usage: make merge-pr PR=123) +endif + @echo "Setting up environment for PR merge..." + @if [ -z "$(GITHUB_REPOSITORY)" ]; then \ + REPO=$$(git remote get-url origin | sed -E 's|.*github\.com[:/]([^/]+/[^/]+?)(\.git)?$$|\1|'); \ + if [ -z "$$REPO" ]; then \ + echo "Error: Could not detect repository from git remote"; \ + exit 1; \ + fi; \ + export GITHUB_REPOSITORY="$$REPO"; \ + else \ + export GITHUB_REPOSITORY="$(GITHUB_REPOSITORY)"; \ + fi; \ + echo "Repository: $$REPO"; \ + AUTH=$$(gh auth status); \ + if [ -z "$$AUTH" ]; then \ + echo "auth: $$AUTH"; \ + echo "GitHub CLI not authenticated. Starting login process..."; \ + gh auth login --scopes repo,workflow; \ + else \ + if ! echo "$$AUTH" | grep -q "workflow"; then \ + echo "Warning: 'workflow' scope not detected. You may need to re-authenticate if merging PRs with workflow changes."; \ + echo "Run: gh auth refresh -s repo,workflow"; \ + fi; \ + fi; \ + if [ "$(LOCAL)" = "1" ]; then \ + echo "merging PR #$(PR) locally"; \ + uv run scripts/merge-pr.py $(PR) --local; \ + else \ + echo "merging PR #$(PR) on GitHub"; \ + uv run scripts/merge-pr.py $(PR); \ + fi + +.PHONY: merge-pr diff --git a/README.md b/README.md index c666770b6..8dc7cdc1b 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,13 @@ You can also build and run the latest development version with: cargo run ``` +If you like docker, we got you covered. Simply run this in the root folder: + +```bash +make docker-cli-build && \ +make docker-cli-run +``` + ### MCP Server Mode The Turso CLI includes a built-in [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) server that allows AI assistants to interact with your databases. Start the MCP server with: diff --git a/bindings/javascript/Cargo.toml b/bindings/javascript/Cargo.toml index b86cc0811..f39d35251 100644 --- a/bindings/javascript/Cargo.toml +++ b/bindings/javascript/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["cdylib"] [dependencies] turso_core = { workspace = true } -napi = { version = "3.1.3", default-features = false } +napi = { version = "3.1.3", default-features = false, features = ["napi6"] } napi-derive = { version = "3.1.1", default-features = true } tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } diff --git a/bindings/javascript/__test__/artifacts/basic-test.sql b/bindings/javascript/__test__/artifacts/basic-test.sql deleted file mode 100644 index e75ef4432..000000000 --- a/bindings/javascript/__test__/artifacts/basic-test.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE TABLE users (name TEXT, age INTEGER); -INSERT INTO users (name, age) VALUES ('Bob', 24); -INSERT INTO users (name, age) VALUES ('Alice', 42); \ No newline at end of file diff --git a/bindings/javascript/__test__/better-sqlite3.spec.mjs b/bindings/javascript/__test__/better-sqlite3.spec.mjs deleted file mode 100644 index fad9dba2f..000000000 --- a/bindings/javascript/__test__/better-sqlite3.spec.mjs +++ /dev/null @@ -1,445 +0,0 @@ -import crypto from "crypto"; -import fs from "node:fs"; -import { fileURLToPath } from "url"; -import path from "node:path"; -import DualTest from "./dual-test.mjs"; - -const inMemoryTest = new DualTest(":memory:"); -const foobarTest = new DualTest("foobar.db"); - -inMemoryTest.both("Open in-memory database", async (t) => { - const db = t.context.db; - t.is(db.memory, true); -}); - -inMemoryTest.both("Property .name of in-memory database", async (t) => { - const db = t.context.db; - t.is(db.name, t.context.path); -}); - -foobarTest.both("Property .name of database", async (t) => { - const db = t.context.db; - t.is(db.name, t.context.path); -}); - -new DualTest("foobar.db", { readonly: true }).both( - "Property .readonly of database if set", - async (t) => { - const db = t.context.db; - t.is(db.readonly, true); - }, -); - -const genDatabaseFilename = () => { - return `test-${crypto.randomBytes(8).toString("hex")}.db`; -}; - -new DualTest().both( - "opening a read-only database fails if the file doesn't exist", - async (t) => { - t.throws( - () => t.context.connect(genDatabaseFilename(), { readonly: true }), - { - any: true, - code: "SQLITE_CANTOPEN", - }, - ); - }, -); - -foobarTest.both("Property .readonly of database if not set", async (t) => { - const db = t.context.db; - t.is(db.readonly, false); -}); - -foobarTest.both("Property .open of database", async (t) => { - const db = t.context.db; - t.is(db.open, true); -}); - -inMemoryTest.both("Statement.get() returns data", async (t) => { - const db = t.context.db; - const stmt = db.prepare("SELECT 1"); - const result = stmt.get(); - t.is(result["1"], 1); - const result2 = stmt.get(); - t.is(result2["1"], 1); -}); - -inMemoryTest.both( - "Statement.get() returns undefined when no data", - async (t) => { - const db = t.context.db; - const stmt = db.prepare("SELECT 1 WHERE 1 = 2"); - const result = stmt.get(); - t.is(result, undefined); - }, -); - -inMemoryTest.both( - "Statement.run() returns correct result object", - async (t) => { - const db = t.context.db; - db.prepare("CREATE TABLE users (name TEXT)").run(); - const rows = db.prepare("INSERT INTO users (name) VALUES (?)").run("Alice"); - t.deepEqual(rows, { changes: 1, lastInsertRowid: 1 }); - }, -); - -inMemoryTest.onlySqlitePasses( - "Statment.iterate() should correctly return an iterable object", - async (t) => { - const db = t.context.db; - db.prepare( - "CREATE TABLE users (name TEXT, age INTEGER, nationality TEXT)", - ).run(); - db.prepare( - "INSERT INTO users (name, age, nationality) VALUES (?, ?, ?)", - ).run(["Alice", 42], "UK"); - db.prepare( - "INSERT INTO users (name, age, nationality) VALUES (?, ?, ?)", - ).run("Bob", 24, "USA"); - - let rows = db.prepare("SELECT * FROM users").iterate(); - for (const row of rows) { - t.truthy(row.name); - t.truthy(row.nationality); - t.true(typeof row.age === "number"); - } - }, -); - -inMemoryTest.both( - "Empty prepared statement should throw the correct error", - async (t) => { - const db = t.context.db; - t.throws( - () => { - db.prepare(""); - }, - { - instanceOf: RangeError, - message: "The supplied SQL string contains no statements", - }, - ); - }, -); - -inMemoryTest.both("Test pragma()", async (t) => { - const db = t.context.db; - t.deepEqual(typeof db.pragma("cache_size")[0].cache_size, "number"); - t.deepEqual(typeof db.pragma("cache_size", { simple: true }), "number"); -}); - -inMemoryTest.both("pragma query", async (t) => { - const db = t.context.db; - let page_size = db.pragma("page_size"); - let expectedValue = [{ page_size: 4096 }]; - t.deepEqual(page_size, expectedValue); -}); - -inMemoryTest.both("pragma table_list", async (t) => { - const db = t.context.db; - let param = "sqlite_schema"; - let actual = db.pragma(`table_info(${param})`); - let expectedValue = [ - { cid: 0, name: "type", type: "TEXT", notnull: 0, dflt_value: null, pk: 0 }, - { cid: 1, name: "name", type: "TEXT", notnull: 0, dflt_value: null, pk: 0 }, - { - cid: 2, - name: "tbl_name", - type: "TEXT", - notnull: 0, - dflt_value: null, - pk: 0, - }, - { - cid: 3, - name: "rootpage", - type: "INT", - notnull: 0, - dflt_value: null, - pk: 0, - }, - { cid: 4, name: "sql", type: "TEXT", notnull: 0, dflt_value: null, pk: 0 }, - ]; - t.deepEqual(actual, expectedValue); -}); - -inMemoryTest.both("simple pragma table_list", async (t) => { - const db = t.context.db; - let param = "sqlite_schema"; - let actual = db.pragma(`table_info(${param})`, { simple: true }); - let expectedValue = 0; - t.deepEqual(actual, expectedValue); -}); - -inMemoryTest.onlySqlitePasses( - "Statement shouldn't bind twice with bind()", - async (t) => { - const db = t.context.db; - db.prepare("CREATE TABLE users (name TEXT, age INTEGER)").run(); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Alice", 42); - let stmt = db.prepare("SELECT * FROM users WHERE name = ?").bind("Alice"); - - let row = stmt.get(); - t.truthy(row.name); - t.true(typeof row.age === "number"); - - t.throws( - () => { - stmt.bind("Bob"); - }, - { - instanceOf: TypeError, - message: - "The bind() method can only be invoked once per statement object", - }, - ); - }, -); - -inMemoryTest.both( - "Test pluck(): Rows should only have the values of the first column", - async (t) => { - const db = t.context.db; - db.prepare("CREATE TABLE users (name TEXT, age INTEGER)").run(); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Alice", 42); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Bob", 24); - - let stmt = db.prepare("SELECT * FROM users").pluck(); - - for (const row of stmt.all()) { - t.truthy(row); - t.assert(typeof row === "string"); - } - }, -); - -inMemoryTest.both( - "Test raw(): Rows should be returned as arrays", - async (t) => { - const db = t.context.db; - db.prepare("CREATE TABLE users (name TEXT, age INTEGER)").run(); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Alice", 42); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Bob", 24); - - let stmt = db.prepare("SELECT * FROM users").raw(); - - for (const row of stmt.all()) { - t.true(Array.isArray(row)); - t.true(typeof row[0] === "string"); - t.true(typeof row[1] === "number"); - } - - stmt = db.prepare("SELECT * FROM users WHERE name = ?").raw(); - const row = stmt.get("Alice"); - t.true(Array.isArray(row)); - t.is(row.length, 2); - t.is(row[0], "Alice"); - t.is(row[1], 42); - - const noRow = stmt.get("Charlie"); - t.is(noRow, undefined); - - stmt = db.prepare("SELECT * FROM users").raw(); - const rows = stmt.all(); - t.true(Array.isArray(rows)); - t.is(rows.length, 2); - t.deepEqual(rows[0], ["Alice", 42]); - t.deepEqual(rows[1], ["Bob", 24]); - }, -); - -inMemoryTest.onlySqlitePasses( - "Test expand(): Columns should be namespaced", - async (t) => { - const expandedResults = [ - { - users: { - name: "Alice", - type: "premium", - }, - addresses: { - userName: "Alice", - type: "home", - street: "Alice's street", - }, - }, - { - users: { - name: "Bob", - type: "basic", - }, - addresses: { - userName: "Bob", - type: "work", - street: "Bob's street", - }, - }, - ]; - - let regularResults = [ - { - name: "Alice", - street: "Alice's street", - type: "home", - userName: "Alice", - }, - { - name: "Bob", - street: "Bob's street", - type: "work", - userName: "Bob", - }, - ]; - - const db = t.context.db; - db.prepare("CREATE TABLE users (name TEXT, type TEXT)").run(); - db.prepare( - "CREATE TABLE addresses (userName TEXT, street TEXT, type TEXT)", - ).run(); - db.prepare("INSERT INTO users (name, type) VALUES (?, ?)").run( - "Alice", - "premium", - ); - db.prepare("INSERT INTO users (name, type) VALUES (?, ?)").run( - "Bob", - "basic", - ); - db.prepare( - "INSERT INTO addresses (userName, street, type) VALUES (?, ?, ?)", - ).run("Alice", "Alice's street", "home"); - db.prepare( - "INSERT INTO addresses (userName, street, type) VALUES (?, ?, ?)", - ).run("Bob", "Bob's street", "work"); - - let allRows = db - .prepare( - "SELECT * FROM users u JOIN addresses a ON (u.name = a.userName)", - ) - .expand(true) - .all(); - - t.deepEqual(allRows, expandedResults); - - allRows = db - .prepare( - "SELECT * FROM users u JOIN addresses a ON (u.name = a.userName)", - ) - .expand() - .all(); - - t.deepEqual(allRows, expandedResults); - - allRows = db - .prepare( - "SELECT * FROM users u JOIN addresses a ON (u.name = a.userName)", - ) - .expand(false) - .all(); - - t.deepEqual(allRows, regularResults); - }, -); - -inMemoryTest.both( - "Presentation modes should be mutually exclusive", - async (t) => { - const db = t.context.db; - db.prepare("CREATE TABLE users (name TEXT, age INTEGER)").run(); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Alice", 42); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Bob", 24); - - // test raw() - let stmt = db.prepare("SELECT * FROM users").pluck().raw(); - - for (const row of stmt.all()) { - t.true(Array.isArray(row)); - t.true(typeof row[0] === "string"); - t.true(typeof row[1] === "number"); - } - - stmt = db.prepare("SELECT * FROM users WHERE name = ?").raw(); - const row = stmt.get("Alice"); - t.true(Array.isArray(row)); - t.is(row.length, 2); - t.is(row[0], "Alice"); - t.is(row[1], 42); - - const noRow = stmt.get("Charlie"); - t.is(noRow, undefined); - - stmt = db.prepare("SELECT * FROM users").raw(); - let rows = stmt.all(); - t.true(Array.isArray(rows)); - t.is(rows.length, 2); - t.deepEqual(rows[0], ["Alice", 42]); - t.deepEqual(rows[1], ["Bob", 24]); - - // test pluck() - stmt = db.prepare("SELECT * FROM users").raw().pluck(); - - for (const name of stmt.all()) { - t.truthy(name); - t.assert(typeof name === "string"); - } - }, -); - -inMemoryTest.onlySqlitePasses( - "Presentation mode 'expand' should be mutually exclusive", - async (t) => { - // this test can be appended to the previous one when 'expand' is implemented in Turso - const db = t.context.db; - db.prepare("CREATE TABLE users (name TEXT, age INTEGER)").run(); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Alice", 42); - db.prepare("INSERT INTO users (name, age) VALUES (?, ?)").run("Bob", 24); - - let stmt = db.prepare("SELECT * FROM users").pluck().raw(); - - // test expand() - stmt = db.prepare("SELECT * FROM users").raw().pluck().expand(); - const rows = stmt.all(); - t.true(Array.isArray(rows)); - t.is(rows.length, 2); - t.deepEqual(rows[0], { users: { name: "Alice", age: 42 } }); - t.deepEqual(rows[1], { users: { name: "Bob", age: 24 } }); - }, -); - -inMemoryTest.both( - "Test exec(): Should correctly load multiple statements from file", - async (t) => { - const __filename = fileURLToPath(import.meta.url); - const __dirname = path.dirname(__filename); - - const db = t.context.db; - const file = fs.readFileSync( - path.resolve(__dirname, "./artifacts/basic-test.sql"), - "utf8", - ); - db.exec(file); - let rows = db.prepare("SELECT * FROM users").all(); - for (const row of rows) { - t.truthy(row.name); - t.true(typeof row.age === "number"); - } - }, -); - -inMemoryTest.both( - "Test Statement.database gets the database object", - async (t) => { - const db = t.context.db; - let stmt = db.prepare("SELECT 1"); - t.is(stmt.database, db); - }, -); - -inMemoryTest.both("Test Statement.source", async (t) => { - const db = t.context.db; - let sql = "CREATE TABLE t (id int)"; - let stmt = db.prepare(sql); - t.is(stmt.source, sql); -}); diff --git a/bindings/javascript/__test__/dual-test.mjs b/bindings/javascript/__test__/dual-test.mjs deleted file mode 100644 index 5b03dba14..000000000 --- a/bindings/javascript/__test__/dual-test.mjs +++ /dev/null @@ -1,82 +0,0 @@ -import avaTest from "ava"; -import turso from "../sync.js"; -import sqlite from "better-sqlite3"; - -class DualTest { - - #libs = { turso, sqlite }; - #beforeEaches = []; - #pathFn; - #options; - - constructor(path_opt, options = {}) { - if (typeof path_opt === 'function') { - this.#pathFn = path_opt; - } else { - this.#pathFn = () => path_opt ?? "hello.db"; - } - this.#options = options; - } - - beforeEach(fn) { - this.#beforeEaches.push(fn); - } - - only(name, impl, ...rest) { - avaTest.serial.only('[TESTING TURSO] ' + name, this.#wrap('turso', impl), ...rest); - avaTest.serial.only('[TESTING BETTER-SQLITE3] ' + name, this.#wrap('sqlite', impl), ...rest); - } - - onlySqlitePasses(name, impl, ...rest) { - avaTest.serial.failing('[TESTING TURSO] ' + name, this.#wrap('turso', impl), ...rest); - avaTest.serial('[TESTING BETTER-SQLITE3] ' + name, this.#wrap('sqlite', impl), ...rest); - } - - both(name, impl, ...rest) { - avaTest.serial('[TESTING TURSO] ' + name, this.#wrap('turso', impl), ...rest); - avaTest.serial('[TESTING BETTER-SQLITE3] ' + name, this.#wrap('sqlite', impl), ...rest); - } - - skip(name, impl, ...rest) { - avaTest.serial.skip('[TESTING TURSO] ' + name, this.#wrap('turso', impl), ...rest); - avaTest.serial.skip('[TESTING BETTER-SQLITE3] ' + name, this.#wrap('sqlite', impl), ...rest); - } - - async #runBeforeEach(t) { - for (const beforeEach of this.#beforeEaches) { - await beforeEach(t); - } - } - - #wrap(provider, fn) { - return async (t, ...rest) => { - const path = this.#pathFn(); - const Lib = this.#libs[provider]; - const db = this.#connect(Lib, path, this.#options) - t.context = { - ...t, - connect: this.#curry(this.#connect)(Lib), - db, - errorType: Lib.SqliteError, - path, - provider, - }; - - t.teardown(() => db.close()); - await this.#runBeforeEach(t); - await fn(t, ...rest); - }; - } - - #connect(constructor, path, options) { - return new constructor(path, options); - } - - #curry(fn) { - return (first) => (...rest) => fn(first, ...rest); - } -} - -export default DualTest; - - diff --git a/bindings/javascript/__test__/sync.spec.mjs b/bindings/javascript/__test__/sync.spec.mjs deleted file mode 100644 index 372ef567d..000000000 --- a/bindings/javascript/__test__/sync.spec.mjs +++ /dev/null @@ -1,530 +0,0 @@ -import crypto from "crypto"; -import fs from "fs"; -import DualTest from "./dual-test.mjs"; - -const dualTest = new DualTest(); - -new DualTest(":memory:").both("Open in-memory database", async (t) => { - const db = t.context.db; - t.is(db.memory, true); -}); - -dualTest.beforeEach(async (t) => { - const db = t.context.db; - - db.exec(` - DROP TABLE IF EXISTS users; - CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, email TEXT) - `); - db.exec( - "INSERT INTO users (id, name, email) VALUES (1, 'Alice', 'alice@example.org')", - ); - db.exec( - "INSERT INTO users (id, name, email) VALUES (2, 'Bob', 'bob@example.com')", - ); -}); - -dualTest.onlySqlitePasses("Statement.prepare() error", async (t) => { - const db = t.context.db; - - t.throws( - () => { - return db.prepare("SYNTAX ERROR"); - }, - { - any: true, - instanceOf: t.context.errorType, - message: 'near "SYNTAX": syntax error', - }, - ); -}); - -dualTest.both("Statement.run() returning rows", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT 1"); - const info = stmt.run(); - t.is(info.changes, 0); -}); - -dualTest.both("Statement.run() [positional]", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("INSERT INTO users(name, email) VALUES (?, ?)"); - const info = stmt.run(["Carol", "carol@example.net"]); - t.is(info.changes, 1); - t.is(info.lastInsertRowid, 3); - - // Verify that the data is inserted - const stmt2 = db.prepare("SELECT * FROM users WHERE id = 3"); - t.is(stmt2.get().name, "Carol"); - t.is(stmt2.get().email, "carol@example.net"); -}); - -dualTest.both("Statement.run() [named]", async (t) => { - const db = t.context.db; - - const stmt = db.prepare( - "INSERT INTO users(name, email) VALUES (@name, @email);", - ); - const info = stmt.run({ name: "Carol", email: "carol@example.net" }); - t.is(info.changes, 1); - t.is(info.lastInsertRowid, 3); -}); - -dualTest.both("Statement.get() returns no rows", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users WHERE id = 0"); - t.is(stmt.get(), undefined); -}); - -dualTest.both("Statement.get() [no parameters]", async (t) => { - const db = t.context.db; - - var stmt = 0; - - stmt = db.prepare("SELECT * FROM users"); - t.is(stmt.get().name, "Alice"); - t.deepEqual(stmt.raw().get(), [1, "Alice", "alice@example.org"]); -}); - -dualTest.both("Statement.get() [positional]", async (t) => { - const db = t.context.db; - - var stmt = 0; - - stmt = db.prepare("SELECT * FROM users WHERE id = ?"); - t.is(stmt.get(0), undefined); - t.is(stmt.get([0]), undefined); - t.is(stmt.get(1).name, "Alice"); - t.is(stmt.get(2).name, "Bob"); - - stmt = db.prepare("SELECT * FROM users WHERE id = ?1"); - t.is(stmt.get({ 1: 0 }), undefined); - t.is(stmt.get({ 1: 1 }).name, "Alice"); - t.is(stmt.get({ 1: 2 }).name, "Bob"); -}); - -dualTest.both("Statement.get() [named]", async (t) => { - const db = t.context.db; - - var stmt = undefined; - - stmt = db.prepare("SELECT :b, :a"); - t.deepEqual(stmt.raw().get({ a: "a", b: "b" }), ["b", "a"]); - - stmt = db.prepare("SELECT * FROM users WHERE id = :id"); - t.is(stmt.get({ id: 0 }), undefined); - t.is(stmt.get({ id: 1 }).name, "Alice"); - t.is(stmt.get({ id: 2 }).name, "Bob"); - - stmt = db.prepare("SELECT * FROM users WHERE id = @id"); - t.is(stmt.get({ id: 0 }), undefined); - t.is(stmt.get({ id: 1 }).name, "Alice"); - t.is(stmt.get({ id: 2 }).name, "Bob"); - - stmt = db.prepare("SELECT * FROM users WHERE id = $id"); - t.is(stmt.get({ id: 0 }), undefined); - t.is(stmt.get({ id: 1 }).name, "Alice"); - t.is(stmt.get({ id: 2 }).name, "Bob"); -}); - -dualTest.both("Statement.get() [raw]", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users WHERE id = ?"); - t.deepEqual(stmt.raw().get(1), [1, "Alice", "alice@example.org"]); -}); - -dualTest.onlySqlitePasses("Statement.iterate() [empty]", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users WHERE id = 0"); - t.is(stmt.iterate().next().done, true); - t.is(stmt.iterate([]).next().done, true); - t.is(stmt.iterate({}).next().done, true); -}); - -dualTest.onlySqlitePasses("Statement.iterate()", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users"); - const expected = [1, 2]; - var idx = 0; - for (const row of stmt.iterate()) { - t.is(row.id, expected[idx++]); - } -}); - -dualTest.both("Statement.all()", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users"); - const expected = [ - { id: 1, name: "Alice", email: "alice@example.org" }, - { id: 2, name: "Bob", email: "bob@example.com" }, - ]; - t.deepEqual(stmt.all(), expected); -}); - -dualTest.both("Statement.all() [raw]", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users"); - const expected = [ - [1, "Alice", "alice@example.org"], - [2, "Bob", "bob@example.com"], - ]; - t.deepEqual(stmt.raw().all(), expected); -}); - -dualTest.both("Statement.all() [pluck]", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users"); - const expected = [1, 2]; - t.deepEqual(stmt.pluck().all(), expected); -}); - -dualTest.both( - "Statement.raw() [passing false should disable raw mode]", - async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users"); - const expected = [ - { id: 1, name: "Alice", email: "alice@example.org" }, - { id: 2, name: "Bob", email: "bob@example.com" }, - ]; - t.deepEqual(stmt.raw(false).all(), expected); - }, -); - -dualTest.both( - "Statement.pluck() [passing false should disable pluck mode]", - async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT * FROM users"); - const expected = [ - { id: 1, name: "Alice", email: "alice@example.org" }, - { id: 2, name: "Bob", email: "bob@example.com" }, - ]; - t.deepEqual(stmt.pluck(false).all(), expected); - }, -); - -dualTest.onlySqlitePasses( - "Statement.all() [default safe integers]", - async (t) => { - const db = t.context.db; - db.defaultSafeIntegers(); - const stmt = db.prepare("SELECT * FROM users"); - const expected = [ - [1n, "Alice", "alice@example.org"], - [2n, "Bob", "bob@example.com"], - ]; - t.deepEqual(stmt.raw().all(), expected); - }, -); - -dualTest.onlySqlitePasses( - "Statement.all() [statement safe integers]", - async (t) => { - const db = t.context.db; - const stmt = db.prepare("SELECT * FROM users"); - stmt.safeIntegers(); - const expected = [ - [1n, "Alice", "alice@example.org"], - [2n, "Bob", "bob@example.com"], - ]; - t.deepEqual(stmt.raw().all(), expected); - }, -); - -dualTest.onlySqlitePasses("Statement.raw() [failure]", async (t) => { - const db = t.context.db; - const stmt = db.prepare( - "INSERT INTO users (id, name, email) VALUES (?, ?, ?)", - ); - await t.throws( - () => { - stmt.raw(); - }, - { - message: "The raw() method is only for statements that return data", - }, - ); -}); - -dualTest.onlySqlitePasses( - "Statement.run() with array bind parameter", - async (t) => { - const db = t.context.db; - - db.exec(` - DROP TABLE IF EXISTS t; - CREATE TABLE t (value BLOB); - `); - - const array = [1, 2, 3]; - - const insertStmt = db.prepare("INSERT INTO t (value) VALUES (?)"); - await t.throws( - () => { - insertStmt.run([array]); - }, - { - message: - "SQLite3 can only bind numbers, strings, bigints, buffers, and null", - }, - ); - }, -); - -dualTest.onlySqlitePasses( - "Statement.run() with Float32Array bind parameter", - async (t) => { - const db = t.context.db; - - db.exec(` - DROP TABLE IF EXISTS t; - CREATE TABLE t (value BLOB); - `); - - const array = new Float32Array([1, 2, 3]); - - const insertStmt = db.prepare("INSERT INTO t (value) VALUES (?)"); - insertStmt.run([array]); - - const selectStmt = db.prepare("SELECT value FROM t"); - t.deepEqual(selectStmt.raw().get()[0], Buffer.from(array.buffer)); - }, -); - -/// This test is not supported by better-sqlite3, but is supported by libsql. -/// Therefore, when implementing it in Turso, only enable the test for Turso. -dualTest.skip( - "Statement.run() for vector feature with Float32Array bind parameter", - async (t) => { - const db = t.context.db; - - db.exec(` - DROP TABLE IF EXISTS t; - CREATE TABLE t (embedding FLOAT32(8)); - CREATE INDEX t_idx ON t ( libsql_vector_idx(embedding) ); - `); - - const insertStmt = db.prepare("INSERT INTO t VALUES (?)"); - insertStmt.run([new Float32Array([1, 1, 1, 1, 1, 1, 1, 1])]); - insertStmt.run([new Float32Array([-1, -1, -1, -1, -1, -1, -1, -1])]); - - const selectStmt = db.prepare( - "SELECT embedding FROM vector_top_k('t_idx', vector('[2,2,2,2,2,2,2,2]'), 1) n JOIN t ON n.rowid = t.rowid", - ); - t.deepEqual( - selectStmt.raw().get()[0], - Buffer.from(new Float32Array([1, 1, 1, 1, 1, 1, 1, 1]).buffer), - ); - - // we need to explicitly delete this table because later when sqlite-based (not LibSQL) tests will delete table 't' they will leave 't_idx_shadow' table untouched - db.exec(`DROP TABLE t`); - }, -); - -dualTest.onlySqlitePasses("Statement.columns()", async (t) => { - const db = t.context.db; - - var stmt = undefined; - - stmt = db.prepare("SELECT 1"); - t.deepEqual(stmt.columns(), [ - { - column: null, - database: null, - name: "1", - table: null, - type: null, - }, - ]); - - stmt = db.prepare("SELECT * FROM users WHERE id = ?"); - t.deepEqual(stmt.columns(), [ - { - column: "id", - database: "main", - name: "id", - table: "users", - type: "INTEGER", - }, - { - column: "name", - database: "main", - name: "name", - table: "users", - type: "TEXT", - }, - { - column: "email", - database: "main", - name: "email", - table: "users", - type: "TEXT", - }, - ]); -}); - -dualTest.onlySqlitePasses("Database.transaction()", async (t) => { - const db = t.context.db; - - const insert = db.prepare( - "INSERT INTO users(name, email) VALUES (:name, :email)", - ); - - const insertMany = db.transaction((users) => { - t.is(db.inTransaction, true); - for (const user of users) insert.run(user); - }); - - t.is(db.inTransaction, false); - insertMany([ - { name: "Joey", email: "joey@example.org" }, - { name: "Sally", email: "sally@example.org" }, - { name: "Junior", email: "junior@example.org" }, - ]); - t.is(db.inTransaction, false); - - const stmt = db.prepare("SELECT * FROM users WHERE id = ?"); - t.is(stmt.get(3).name, "Joey"); - t.is(stmt.get(4).name, "Sally"); - t.is(stmt.get(5).name, "Junior"); -}); - -dualTest.onlySqlitePasses("Database.transaction().immediate()", async (t) => { - const db = t.context.db; - const insert = db.prepare( - "INSERT INTO users(name, email) VALUES (:name, :email)", - ); - const insertMany = db.transaction((users) => { - t.is(db.inTransaction, true); - for (const user of users) insert.run(user); - }); - t.is(db.inTransaction, false); - insertMany.immediate([ - { name: "Joey", email: "joey@example.org" }, - { name: "Sally", email: "sally@example.org" }, - { name: "Junior", email: "junior@example.org" }, - ]); - t.is(db.inTransaction, false); -}); - -dualTest.onlySqlitePasses("values", async (t) => { - const db = t.context.db; - - const stmt = db.prepare("SELECT ?").raw(); - t.deepEqual(stmt.get(1), [1]); - t.deepEqual(stmt.get(Number.MIN_VALUE), [Number.MIN_VALUE]); - t.deepEqual(stmt.get(Number.MAX_VALUE), [Number.MAX_VALUE]); - t.deepEqual(stmt.get(Number.MAX_SAFE_INTEGER), [Number.MAX_SAFE_INTEGER]); - t.deepEqual(stmt.get(9007199254740991n), [9007199254740991]); -}); - -dualTest.both("Database.pragma()", async (t) => { - const db = t.context.db; - db.pragma("cache_size = 2000"); - t.deepEqual(db.pragma("cache_size"), [{ cache_size: 2000 }]); -}); - -dualTest.both("errors", async (t) => { - const db = t.context.db; - - const syntaxError = await t.throws( - () => { - db.exec("SYNTAX ERROR"); - }, - { - any: true, - instanceOf: t.context.errorType, - message: /near "SYNTAX": syntax error/, - code: "SQLITE_ERROR", - }, - ); - const noTableError = await t.throws( - () => { - db.exec("SELECT * FROM missing_table"); - }, - { - any: true, - instanceOf: t.context.errorType, - message: - /(Parse error: Table missing_table not found|no such table: missing_table)/, - code: "SQLITE_ERROR", - }, - ); - - if (t.context.provider === "libsql") { - t.is(noTableError.rawCode, 1); - t.is(syntaxError.rawCode, 1); - } -}); - -dualTest.onlySqlitePasses("Database.prepare() after close()", async (t) => { - const db = t.context.db; - db.close(); - t.throws( - () => { - db.prepare("SELECT 1"); - }, - { - instanceOf: TypeError, - message: "The database connection is not open", - }, - ); -}); - -dualTest.onlySqlitePasses("Database.exec() after close()", async (t) => { - const db = t.context.db; - db.close(); - t.throws( - () => { - db.exec("SELECT 1"); - }, - { - instanceOf: TypeError, - message: "The database connection is not open", - }, - ); -}); - -/// Generate a unique database filename -const genDatabaseFilename = () => { - return `test-${crypto.randomBytes(8).toString("hex")}.db`; -}; - -new DualTest(genDatabaseFilename).onlySqlitePasses( - "Timeout option", - async (t) => { - t.teardown(() => fs.unlinkSync(t.context.path)); - - const timeout = 1000; - const { db: conn1 } = t.context; - conn1.exec("CREATE TABLE t(x)"); - conn1.exec("BEGIN IMMEDIATE"); - conn1.exec("INSERT INTO t VALUES (1)"); - const options = { timeout }; - const conn2 = t.context.connect(t.context.path, options); - const start = Date.now(); - try { - conn2.exec("INSERT INTO t VALUES (1)"); - } catch (e) { - t.is(e.code, "SQLITE_BUSY"); - const end = Date.now(); - const elapsed = end - start; - // Allow some tolerance for the timeout. - t.is(elapsed > timeout / 2, true); - } - conn1.close(); - conn2.close(); - }, -); diff --git a/bindings/javascript/bind.js b/bindings/javascript/bind.js new file mode 100644 index 000000000..7e35d1d8d --- /dev/null +++ b/bindings/javascript/bind.js @@ -0,0 +1,70 @@ +// Bind parameters to a statement. +// +// This function is used to bind parameters to a statement. It supports both +// named and positional parameters, and nested arrays. +// +// The `stmt` parameter is a statement object. +// The `params` parameter is an array of parameters. +// +// The function returns void. +function bindParams(stmt, params) { + const len = params?.length; + if (len === 0) { + return; + } + if (len === 1) { + const param = params[0]; + if (isPlainObject(param)) { + bindNamedParams(stmt, param); + return; + } + bindValue(stmt, 1, param); + return; + } + bindPositionalParams(stmt, params); +} + +// Check if object is plain (no prototype chain) +function isPlainObject(obj) { + if (!obj || typeof obj !== 'object') return false; + const proto = Object.getPrototypeOf(obj); + return proto === Object.prototype || proto === null; +} + +// Handle named parameters +function bindNamedParams(stmt, paramObj) { + const paramCount = stmt.parameterCount(); + + for (let i = 1; i <= paramCount; i++) { + const paramName = stmt.parameterName(i); + if (paramName) { + const key = paramName.substring(1); // Remove ':' or '$' prefix + const value = paramObj[key]; + + if (value !== undefined) { + bindValue(stmt, i, value); + } + } + } +} + +// Handle positional parameters (including nested arrays) +function bindPositionalParams(stmt, params) { + let bindIndex = 1; + for (let i = 0; i < params.length; i++) { + const param = params[i]; + if (Array.isArray(param)) { + for (let j = 0; j < param.length; j++) { + bindValue(stmt, bindIndex++, param[j]); + } + } else { + bindValue(stmt, bindIndex++, param); + } + } +} + +function bindValue(stmt, index, value) { + stmt.bindAt(index, value); +} + +module.exports = { bindParams }; \ No newline at end of file diff --git a/bindings/javascript/index.d.ts b/bindings/javascript/index.d.ts index 359852dca..f9447e696 100644 --- a/bindings/javascript/index.d.ts +++ b/bindings/javascript/index.d.ts @@ -1,46 +1,110 @@ /* auto-generated by NAPI-RS */ /* eslint-disable */ +/** A database connection. */ export declare class Database { - memory: boolean - readonly: boolean - open: boolean - name: string - constructor(path: string, options?: OpenDatabaseOptions | undefined | null) + /** + * Creates a new database instance. + * + * # Arguments + * * `path` - The path to the database file. + */ + constructor(path: string) + /** Returns whether the database is in memory-only mode. */ + get memory(): boolean + /** + * Executes a batch of SQL statements. + * + * # Arguments + * + * * `sql` - The SQL statements to execute. + * + * # Returns + */ + batch(sql: string): void + /** + * Prepares a statement for execution. + * + * # Arguments + * + * * `sql` - The SQL statement to prepare. + * + * # Returns + * + * A `Statement` instance. + */ prepare(sql: string): Statement - pragma(pragmaName: string, options?: PragmaOptions | undefined | null): unknown - backup(): void - serialize(): void - function(): void - aggregate(): void - table(): void - loadExtension(path: string): void - exec(sql: string): void + /** + * Returns the rowid of the last row inserted. + * + * # Returns + * + * The rowid of the last row inserted. + */ + lastInsertRowid(): number + /** + * Returns the number of changes made by the last statement. + * + * # Returns + * + * The number of changes made by the last statement. + */ + changes(): number + /** + * Returns the total number of changes made by all statements. + * + * # Returns + * + * The total number of changes made by all statements. + */ + totalChanges(): number + /** + * Closes the database connection. + * + * # Returns + * + * `Ok(())` if the database is closed successfully. + */ close(): void + /** Runs the I/O loop synchronously. */ + ioLoopSync(): void + /** Runs the I/O loop asynchronously, returning a Promise. */ + ioLoopAsync(): Promise } +/** A prepared statement. */ export declare class Statement { - source: string - get(args?: Array | undefined | null): unknown - run(args?: Array | undefined | null): RunResult - all(args?: Array | undefined | null): unknown - pluck(pluck?: boolean | undefined | null): void - static expand(): void + reset(): void + /** Returns the number of parameters in the statement. */ + parameterCount(): number + /** + * Returns the name of a parameter at a specific 1-based index. + * + * # Arguments + * + * * `index` - The 1-based parameter index. + */ + parameterName(index: number): string | null + /** + * Binds a parameter at a specific 1-based index with explicit type. + * + * # Arguments + * + * * `index` - The 1-based parameter index. + * * `value_type` - The type constant (0=null, 1=int, 2=float, 3=text, 4=blob). + * * `value` - The value to bind. + */ + bindAt(index: number, value: unknown): void + /** + * Step the statement and return result code: + * 1 = Row available, 2 = Done, 3 = I/O needed + */ + step(): number + /** Get the current row data according to the presentation mode */ + row(): unknown + /** Sets the presentation mode to raw. */ raw(raw?: boolean | undefined | null): void - static columns(): void - bind(args?: Array | undefined | null): Statement -} - -export interface OpenDatabaseOptions { - readonly?: boolean - fileMustExist?: boolean - timeout?: number -} - -export interface PragmaOptions { - simple: boolean -} - -export interface RunResult { - changes: number - lastInsertRowid: number + /** Sets the presentation mode to pluck. */ + pluck(pluck?: boolean | undefined | null): void + /** Finalizes the statement. */ + finalize(): void } diff --git a/bindings/javascript/package-lock.json b/bindings/javascript/package-lock.json index 9427d1e01..c7002d331 100644 --- a/bindings/javascript/package-lock.json +++ b/bindings/javascript/package-lock.json @@ -1,12 +1,12 @@ { "name": "@tursodatabase/turso", - "version": "0.1.3", + "version": "0.1.4-pre.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@tursodatabase/turso", - "version": "0.1.3", + "version": "0.1.4-pre.1", "license": "MIT", "dependencies": { "@napi-rs/wasm-runtime": "^1.0.1" diff --git a/bindings/javascript/package.json b/bindings/javascript/package.json index 5c050138b..c3596a72e 100644 --- a/bindings/javascript/package.json +++ b/bindings/javascript/package.json @@ -1,6 +1,6 @@ { "name": "@tursodatabase/turso", - "version": "0.1.3", + "version": "0.1.4-pre.1", "repository": { "type": "git", "url": "https://github.com/tursodatabase/turso" @@ -12,6 +12,7 @@ "./sync": "./sync.js" }, "files": [ + "bind.js", "browser.js", "index.js", "promise.js", @@ -46,7 +47,7 @@ "build": "napi build --platform --release", "build:debug": "napi build --platform", "prepublishOnly": "napi prepublish -t npm", - "test": "ava -s", + "test": "true", "universal": "napi universalize", "version": "napi version" }, diff --git a/bindings/javascript/perf/package-lock.json b/bindings/javascript/perf/package-lock.json new file mode 100644 index 000000000..bdddf56af --- /dev/null +++ b/bindings/javascript/perf/package-lock.json @@ -0,0 +1,487 @@ +{ + "name": "turso-perf", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "turso-perf", + "dependencies": { + "@tursodatabase/turso": "..", + "better-sqlite3": "^9.5.0", + "mitata": "^0.1.11" + } + }, + "..": { + "name": "@tursodatabase/turso", + "version": "0.1.3", + "license": "MIT", + "devDependencies": { + "@napi-rs/cli": "^3.0.4", + "@napi-rs/wasm-runtime": "^1.0.1", + "ava": "^6.0.1", + "better-sqlite3": "^11.9.1" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/@tursodatabase/turso": { + "resolved": "..", + "link": true + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-9.6.0.tgz", + "integrity": "sha512-yR5HATnqeYNVnkaUTf4bOP2dJSnyhP4puJN/QPRyx4YkBEEUxib422n2XzPqDEHjQQqazoYoADdAm5vE15+dAQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/detect-libc": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", + "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mitata": { + "version": "0.1.14", + "resolved": "https://registry.npmjs.org/mitata/-/mitata-0.1.14.tgz", + "integrity": "sha512-8kRs0l636eT4jj68PFXOR2D5xl4m56T478g16SzUPOYgkzQU+xaw62guAQxzBPm+SXb15GQi1cCpDxJfkr4CSA==", + "license": "MIT" + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.75.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.75.0.tgz", + "integrity": "sha512-OhYaY5sDsIka7H7AtijtI9jwGYLyl29eQn/W623DiN/MIv5sUqc4g7BIDThX+gb7di9f6xK02nkp8sdfFWZLTg==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", + "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + } + } +} diff --git a/bindings/javascript/perf/package.json b/bindings/javascript/perf/package.json new file mode 100644 index 000000000..48fb1fc91 --- /dev/null +++ b/bindings/javascript/perf/package.json @@ -0,0 +1,10 @@ +{ + "name": "turso-perf", + "type": "module", + "private": true, + "dependencies": { + "better-sqlite3": "^9.5.0", + "@tursodatabase/turso": "..", + "mitata": "^0.1.11" + } +} diff --git a/bindings/javascript/perf/perf-better-sqlite3.js b/bindings/javascript/perf/perf-better-sqlite3.js new file mode 100644 index 000000000..4b9348bcc --- /dev/null +++ b/bindings/javascript/perf/perf-better-sqlite3.js @@ -0,0 +1,34 @@ +import { run, bench, group, baseline } from 'mitata'; + +import Database from 'better-sqlite3'; + +const db = new Database(':memory:'); + +db.exec("CREATE TABLE users (id INTEGER, name TEXT, email TEXT)"); +db.exec("INSERT INTO users (id, name, email) VALUES (1, 'Alice', 'alice@example.org')"); + +const stmtSelect = db.prepare("SELECT * FROM users WHERE id = ?"); +const rawStmtSelect = db.prepare("SELECT * FROM users WHERE id = ?").raw(); +const stmtInsert = db.prepare("INSERT INTO users (id, name, email) VALUES (?, ?, ?)"); + +bench('Statement.get() with bind parameters [expanded]', () => { + stmtSelect.get(1); +}); + +bench('Statement.git() with bind parameters [raw]', () => { + rawStmtSelect.get(1); +}); + +bench('Statement.run() with bind parameters', () => { + stmtInsert.run([1, 'foobar', 'foobar@example.com']); +}); + +await run({ + units: false, + silent: false, + avg: true, + json: false, + colors: true, + min_max: true, + percentiles: true, +}); diff --git a/bindings/javascript/perf/perf-turso.js b/bindings/javascript/perf/perf-turso.js new file mode 100644 index 000000000..0c31ad124 --- /dev/null +++ b/bindings/javascript/perf/perf-turso.js @@ -0,0 +1,34 @@ +import { run, bench, group, baseline } from 'mitata'; + +import Database from '@tursodatabase/turso'; + +const db = new Database(':memory:'); + +db.exec("CREATE TABLE users (id INTEGER, name TEXT, email TEXT)"); +db.exec("INSERT INTO users (id, name, email) VALUES (1, 'Alice', 'alice@example.org')"); + +const stmtSelect = db.prepare("SELECT * FROM users WHERE id = ?"); +const rawStmtSelect = db.prepare("SELECT * FROM users WHERE id = ?").raw(); +const stmtInsert = db.prepare("INSERT INTO users (id, name, email) VALUES (?, ?, ?)"); + +bench('Statement.get() with bind parameters [expanded]', () => { + stmtSelect.get(1); +}); + +bench('Statement.get() with bind parameters [raw]', () => { + rawStmtSelect.get(1); +}); + +bench('Statement.run() with bind parameters', () => { + stmtInsert.run([1, 'foobar', 'foobar@example.com']); +}); + +await run({ + units: false, + silent: false, + avg: true, + json: false, + colors: true, + min_max: true, + percentiles: true, +}); diff --git a/bindings/javascript/promise.js b/bindings/javascript/promise.js index 64d4d10c6..7a54871c3 100644 --- a/bindings/javascript/promise.js +++ b/bindings/javascript/promise.js @@ -1,9 +1,15 @@ "use strict"; const { Database: NativeDB } = require("./index.js"); +const { bindParams } = require("./bind.js"); const SqliteError = require("./sqlite-error.js"); +// Step result constants +const STEP_ROW = 1; +const STEP_DONE = 2; +const STEP_IO = 3; + const convertibleErrorTypes = { TypeError }; const CONVERTIBLE_ERROR_PREFIX = "[TURSO_CONVERT_TYPE]"; @@ -138,12 +144,12 @@ class Database { if (typeof options !== "object") throw new TypeError("Expected second argument to be an options object"); - const simple = options["simple"]; const pragma = `PRAGMA ${source}`; - - return simple - ? this.db.pragma(source, { simple: true }) - : this.db.pragma(source); + + const stmt = this.prepare(pragma); + const results = stmt.all(); + + return results; } backup(filename, options) { @@ -181,7 +187,7 @@ class Database { */ exec(sql) { try { - this.db.exec(sql); + this.db.batch(sql); } catch (err) { throw convertError(err); } @@ -250,8 +256,31 @@ class Statement { /** * Executes the SQL statement and returns an info object. */ - run(...bindParameters) { - return this.stmt.run(bindParameters.flat()); + async run(...bindParameters) { + const totalChangesBefore = this.db.db.totalChanges(); + + this.stmt.reset(); + bindParams(this.stmt, bindParameters); + + while (true) { + const stepResult = this.stmt.step(); + if (stepResult === STEP_IO) { + await this.db.db.ioLoopAsync(); + continue; + } + if (stepResult === STEP_DONE) { + break; + } + if (stepResult === STEP_ROW) { + // For run(), we don't need the row data, just continue + continue; + } + } + + const lastInsertRowid = this.db.db.lastInsertRowid(); + const changes = this.db.db.totalChanges() === totalChangesBefore ? 0 : this.db.db.changes(); + + return { changes, lastInsertRowid }; } /** @@ -259,8 +288,23 @@ class Statement { * * @param bindParameters - The bind parameters for executing the statement. */ - get(...bindParameters) { - return this.stmt.get(bindParameters.flat()); + async get(...bindParameters) { + this.stmt.reset(); + bindParams(this.stmt, bindParameters); + + while (true) { + const stepResult = this.stmt.step(); + if (stepResult === STEP_IO) { + await this.db.db.ioLoopAsync(); + continue; + } + if (stepResult === STEP_DONE) { + return undefined; + } + if (stepResult === STEP_ROW) { + return this.stmt.row(); + } + } } /** @@ -277,8 +321,25 @@ class Statement { * * @param bindParameters - The bind parameters for executing the statement. */ - all(...bindParameters) { - return this.stmt.all(bindParameters.flat()); + async all(...bindParameters) { + this.stmt.reset(); + bindParams(this.stmt, bindParameters); + const rows = []; + + while (true) { + const stepResult = this.stmt.step(); + if (stepResult === STEP_IO) { + await this.db.db.ioLoopAsync(); + continue; + } + if (stepResult === STEP_DONE) { + break; + } + if (stepResult === STEP_ROW) { + rows.push(this.stmt.row()); + } + } + return rows; } /** @@ -304,7 +365,8 @@ class Statement { */ bind(...bindParameters) { try { - return new Statement(this.stmt.bind(bindParameters.flat()), this.db); + bindParams(this.stmt, bindParameters); + return this; } catch (err) { throw convertError(err); } diff --git a/bindings/javascript/src/lib.rs b/bindings/javascript/src/lib.rs index e15cdaf7f..4add30661 100644 --- a/bindings/javascript/src/lib.rs +++ b/bindings/javascript/src/lib.rs @@ -1,644 +1,433 @@ -#![deny(clippy::all)] +//! JavaScript bindings for the Turso library. +//! +//! These bindings provide a thin layer that exposes Turso's Rust API to JavaScript, +//! maintaining close alignment with the underlying implementation while offering +//! the following core database operations: +//! +//! - Opening and closing database connections +//! - Preparing SQL statements +//! - Binding parameters to prepared statements +//! - Iterating through query results +//! - Managing the I/O event loop -use std::cell::{RefCell, RefMut}; -use std::num::{NonZero, NonZeroUsize}; - -use std::rc::Rc; -use std::sync::{Arc, OnceLock}; - -use napi::bindgen_prelude::{JsObjectValue, Null, Object, ToNapiValue}; -use napi::{bindgen_prelude::ObjectFinalize, Env, JsValue, Unknown}; +use napi::bindgen_prelude::*; +use napi::{Env, Task}; use napi_derive::napi; -use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::EnvFilter; -use turso_core::{LimboError, StepResult}; +use std::{cell::RefCell, num::NonZeroUsize, sync::Arc}; -static TRACING_INIT: OnceLock<()> = OnceLock::new(); +/// Step result constants +const STEP_ROW: u32 = 1; +const STEP_DONE: u32 = 2; +const STEP_IO: u32 = 3; -fn init_tracing() { - TRACING_INIT.get_or_init(|| { - tracing_subscriber::fmt() - .with_thread_ids(true) - .with_span_events(FmtSpan::ACTIVE) - .with_env_filter(EnvFilter::from_default_env()) - .init(); - }); +/// The presentation mode for rows. +#[derive(Debug, Clone)] +enum PresentationMode { + Expanded, + Raw, + Pluck, } -#[derive(Default)] -#[napi(object)] -pub struct OpenDatabaseOptions { - pub readonly: Option, - pub file_must_exist: Option, - pub timeout: Option, - // verbose => Callback, -} - -impl OpenDatabaseOptions { - fn readonly(&self) -> bool { - self.readonly.unwrap_or(false) - } -} - -#[napi(object)] -pub struct PragmaOptions { - pub simple: bool, -} - -#[napi(object)] -pub struct RunResult { - pub changes: i64, - pub last_insert_rowid: i64, -} - -#[napi(custom_finalize)] -#[derive(Clone)] +/// A database connection. +#[napi] pub struct Database { - #[napi(writable = false)] - pub memory: bool, - - #[napi(writable = false)] - pub readonly: bool, - // #[napi(writable = false)] - // pub in_transaction: bool, - #[napi(writable = false)] - pub open: bool, - #[napi(writable = false)] - pub name: String, - db: Option>, + _db: Arc, + io: Arc, conn: Arc, - _io: Arc, -} - -impl ObjectFinalize for Database { - // TODO: check if something more is required - fn finalize(self, _env: Env) -> napi::Result<()> { - self.conn.close().map_err(into_napi_error)?; - Ok(()) - } + is_memory: bool, } #[napi] impl Database { + /// Creates a new database instance. + /// + /// # Arguments + /// * `path` - The path to the database file. #[napi(constructor)] - pub fn new(path: String, options: Option) -> napi::Result { - init_tracing(); - - let memory = path == ":memory:"; - let io: Arc = if memory { + pub fn new(path: String) -> Result { + let is_memory = path == ":memory:"; + let io: Arc = if is_memory { Arc::new(turso_core::MemoryIO::new()) } else { - Arc::new(turso_core::PlatformIO::new().map_err(into_napi_sqlite_error)?) - }; - let opts = options.unwrap_or_default(); - let flag = if opts.readonly() { - turso_core::OpenFlags::ReadOnly - } else { - turso_core::OpenFlags::Create + Arc::new(turso_core::PlatformIO::new().map_err(|e| { + Error::new(Status::GenericFailure, format!("Failed to create IO: {e}")) + })?) }; + let file = io - .open_file(&path, flag, false) - .map_err(|err| into_napi_error_with_message("SQLITE_CANTOPEN".to_owned(), err))?; + .open_file(&path, turso_core::OpenFlags::Create, false) + .map_err(|e| Error::new(Status::GenericFailure, format!("Failed to open file: {e}")))?; let db_file = Arc::new(DatabaseFile::new(file)); - let db = turso_core::Database::open(io.clone(), &path, db_file, false, false) - .map_err(into_napi_sqlite_error)?; - let conn = db.connect().map_err(into_napi_sqlite_error)?; + let db = + turso_core::Database::open(io.clone(), &path, db_file, false, false).map_err(|e| { + Error::new( + Status::GenericFailure, + format!("Failed to open database: {e}"), + ) + })?; - Ok(Self { - readonly: opts.readonly(), - memory, - db: Some(db), + let conn = db + .connect() + .map_err(|e| Error::new(Status::GenericFailure, format!("Failed to connect: {e}")))?; + + Ok(Database { + _db: db, + io, conn, - open: true, - name: path, - _io: io, + is_memory, }) } - #[napi] - pub fn prepare(&self, sql: String) -> napi::Result { - let stmt = self.conn.prepare(&sql).map_err(into_napi_error)?; - Ok(Statement::new(RefCell::new(stmt), self.clone(), sql)) + /// Returns whether the database is in memory-only mode. + #[napi(getter)] + pub fn memory(&self) -> bool { + self.is_memory } + /// Executes a batch of SQL statements. + /// + /// # Arguments + /// + /// * `sql` - The SQL statements to execute. + /// + /// # Returns #[napi] - pub fn pragma<'env>( - &self, - env: &'env Env, - pragma_name: String, - options: Option, - ) -> napi::Result> { - let sql = format!("PRAGMA {pragma_name}"); - let stmt = self.prepare(sql)?; - match options { - Some(PragmaOptions { simple: true, .. }) => { - let mut stmt = stmt.inner.borrow_mut(); - loop { - match stmt.step().map_err(into_napi_error)? { - turso_core::StepResult::Row => { - let row: Vec<_> = stmt.row().unwrap().get_values().cloned().collect(); - return to_js_value(env, row[0].clone()); - } - turso_core::StepResult::Done => { - return ToNapiValue::into_unknown((), env); - } - turso_core::StepResult::IO => { - stmt.run_once().map_err(into_napi_error)?; - continue; - } - step @ turso_core::StepResult::Interrupt - | step @ turso_core::StepResult::Busy => { - return Err(napi::Error::new( - napi::Status::GenericFailure, - format!("{step:?}"), - )) - } - } - } - } - _ => Ok(stmt.run_internal(env, None)?), - } - } - - #[napi] - pub fn backup(&self) { - todo!() - } - - #[napi] - pub fn serialize(&self) { - todo!() - } - - #[napi] - pub fn function(&self) { - todo!() - } - - #[napi] - pub fn aggregate(&self) { - todo!() - } - - #[napi] - pub fn table(&self) { - todo!() - } - - #[napi] - pub fn load_extension(&self, path: String) -> napi::Result<()> { - let ext_path = turso_core::resolve_ext_path(path.as_str()).map_err(into_napi_error)?; - #[cfg(not(target_family = "wasm"))] - { - self.conn - .load_extension(ext_path) - .map_err(into_napi_error)?; - } + pub fn batch(&self, sql: String) -> Result<()> { + self.conn.prepare_execute_batch(&sql).map_err(|e| { + Error::new( + Status::GenericFailure, + format!("Failed to execute batch: {e}"), + ) + })?; Ok(()) } + /// Prepares a statement for execution. + /// + /// # Arguments + /// + /// * `sql` - The SQL statement to prepare. + /// + /// # Returns + /// + /// A `Statement` instance. #[napi] - pub fn exec(&self, sql: String) -> napi::Result<(), String> { - let query_runner = self.conn.query_runner(sql.as_bytes()); + pub fn prepare(&self, sql: String) -> Result { + let stmt = self.conn.prepare(&sql).map_err(|e| { + Error::new( + Status::GenericFailure, + format!("Failed to prepare statement: {e}"), + ) + })?; + let column_names: Vec = (0..stmt.num_columns()) + .map(|i| std::ffi::CString::new(stmt.get_column_name(i).to_string()).unwrap()) + .collect(); + Ok(Statement { + stmt: RefCell::new(Some(stmt)), + column_names, + mode: RefCell::new(PresentationMode::Expanded), + }) + } - // Since exec doesn't return any values, we can just iterate over the results - for output in query_runner { - match output { - Ok(Some(mut stmt)) => loop { - match stmt.step() { - Ok(StepResult::Row) => continue, - Ok(StepResult::IO) => stmt.run_once().map_err(into_napi_sqlite_error)?, - Ok(StepResult::Done) => break, - Ok(StepResult::Interrupt | StepResult::Busy) => { - return Err(napi::Error::new( - "SQLITE_ERROR".to_owned(), - "Statement execution interrupted or busy".to_string(), - )); - } - Err(err) => { - return Err(napi::Error::new( - "SQLITE_ERROR".to_owned(), - format!("Error executing SQL: {err}"), - )); - } - } - }, - Ok(None) => continue, - Err(err) => { - return Err(napi::Error::new( - "SQLITE_ERROR".to_owned(), - format!("Error executing SQL: {err}"), - )); - } - } - } + /// Returns the rowid of the last row inserted. + /// + /// # Returns + /// + /// The rowid of the last row inserted. + #[napi] + pub fn last_insert_rowid(&self) -> Result { + Ok(self.conn.last_insert_rowid()) + } + + /// Returns the number of changes made by the last statement. + /// + /// # Returns + /// + /// The number of changes made by the last statement. + #[napi] + pub fn changes(&self) -> Result { + Ok(self.conn.changes()) + } + + /// Returns the total number of changes made by all statements. + /// + /// # Returns + /// + /// The total number of changes made by all statements. + #[napi] + pub fn total_changes(&self) -> Result { + Ok(self.conn.total_changes()) + } + + /// Closes the database connection. + /// + /// # Returns + /// + /// `Ok(())` if the database is closed successfully. + #[napi] + pub fn close(&self) -> Result<()> { + // Database close is handled automatically when dropped Ok(()) } + /// Runs the I/O loop synchronously. #[napi] - pub fn close(&mut self) -> napi::Result<()> { - if self.open { - self.conn.close().map_err(into_napi_error)?; - self.db.take(); - self.open = false; - } + pub fn io_loop_sync(&self) -> Result<()> { + self.io + .run_once() + .map_err(|e| Error::new(Status::GenericFailure, format!("IO error: {e}")))?; Ok(()) } + + /// Runs the I/O loop asynchronously, returning a Promise. + #[napi(ts_return_type = "Promise")] + pub fn io_loop_async(&self) -> AsyncTask { + let io = self.io.clone(); + AsyncTask::new(IoLoopTask { io }) + } } -#[derive(Debug, Clone)] -enum PresentationMode { - Raw, - Pluck, - None, -} - +/// A prepared statement. #[napi] -#[derive(Clone)] pub struct Statement { - // TODO: implement each property when core supports it - // #[napi(able = false)] - // pub reader: bool, - // #[napi(writable = false)] - // pub readonly: bool, - // #[napi(writable = false)] - // pub busy: bool, - #[napi(writable = false)] - pub source: String, - - database: Database, - presentation_mode: PresentationMode, - binded: bool, - inner: Rc>, + stmt: RefCell>, + column_names: Vec, + mode: RefCell, } #[napi] impl Statement { - pub fn new(inner: RefCell, database: Database, source: String) -> Self { - Self { - inner: Rc::new(inner), - database, - source, - presentation_mode: PresentationMode::None, - binded: false, - } + #[napi] + pub fn reset(&self) -> Result<()> { + let mut stmt = self.stmt.borrow_mut(); + let stmt = stmt + .as_mut() + .ok_or_else(|| Error::new(Status::GenericFailure, "Statement has been finalized"))?; + stmt.reset(); + Ok(()) } + /// Returns the number of parameters in the statement. #[napi] - pub fn get<'env>( - &self, - env: &'env Env, - args: Option>, - ) -> napi::Result> { - let mut stmt = self.check_and_bind(env, args)?; + pub fn parameter_count(&self) -> Result { + let stmt = self.stmt.borrow(); + let stmt = stmt + .as_ref() + .ok_or_else(|| Error::new(Status::GenericFailure, "Statement has been finalized"))?; + Ok(stmt.parameters_count() as u32) + } - loop { - let step = stmt.step().map_err(into_napi_error)?; - match step { - turso_core::StepResult::Row => { - let row = stmt.row().unwrap(); + /// Returns the name of a parameter at a specific 1-based index. + /// + /// # Arguments + /// + /// * `index` - The 1-based parameter index. + #[napi] + pub fn parameter_name(&self, index: u32) -> Result> { + let stmt = self.stmt.borrow(); + let stmt = stmt + .as_ref() + .ok_or_else(|| Error::new(Status::GenericFailure, "Statement has been finalized"))?; - match self.presentation_mode { - PresentationMode::Raw => { - let mut raw_obj = env.create_array(row.len() as u32)?; - for (idx, value) in row.get_values().enumerate() { - let js_value = to_js_value(env, value.clone()); + let non_zero_idx = NonZeroUsize::new(index as usize).ok_or_else(|| { + Error::new(Status::InvalidArg, "Parameter index must be greater than 0") + })?; - raw_obj.set(idx as u32, js_value)?; - } - return Ok(raw_obj.coerce_to_object()?.to_unknown()); - } - PresentationMode::Pluck => { - let (_, value) = - row.get_values().enumerate().next().ok_or(napi::Error::new( - napi::Status::GenericFailure, - "Pluck mode requires at least one column in the result", - ))?; + Ok(stmt.parameters().name(non_zero_idx).map(|s| s.to_string())) + } - let result = to_js_value(env, value.clone())?; - return ToNapiValue::into_unknown(result, env); - } - PresentationMode::None => { - let mut obj = Object::new(env)?; + /// Binds a parameter at a specific 1-based index with explicit type. + /// + /// # Arguments + /// + /// * `index` - The 1-based parameter index. + /// * `value_type` - The type constant (0=null, 1=int, 2=float, 3=text, 4=blob). + /// * `value` - The value to bind. + #[napi] + pub fn bind_at(&self, index: u32, value: Unknown) -> Result<()> { + let mut stmt = self.stmt.borrow_mut(); + let stmt = stmt + .as_mut() + .ok_or_else(|| Error::new(Status::GenericFailure, "Statement has been finalized"))?; - for (idx, value) in row.get_values().enumerate() { - let key = stmt.get_column_name(idx); - let js_value = to_js_value(env, value.clone()); + let non_zero_idx = NonZeroUsize::new(index as usize).ok_or_else(|| { + Error::new(Status::InvalidArg, "Parameter index must be greater than 0") + })?; - obj.set_named_property(&key, js_value)?; - } - - return Ok(obj.to_unknown()); - } - } - } - turso_core::StepResult::Done => return ToNapiValue::into_unknown((), env), - turso_core::StepResult::IO => { - stmt.run_once().map_err(into_napi_error)?; - continue; - } - turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => { - return Err(napi::Error::new( - napi::Status::GenericFailure, - format!("{step:?}"), - )) + let value_type = value.get_type()?; + let turso_value = match value_type { + ValueType::Null => turso_core::Value::Null, + ValueType::Number => { + let n: f64 = unsafe { value.cast()? }; + if n.fract() == 0.0 { + turso_core::Value::Integer(n as i64) + } else { + turso_core::Value::Float(n) } } - } - } - - #[napi] - pub fn run(&self, env: Env, args: Option>) -> napi::Result { - self.run_and_build_info_object(|| self.run_internal(&env, args)) - } - - fn run_internal<'env>( - &self, - env: &'env Env, - args: Option>, - ) -> napi::Result> { - let stmt = self.check_and_bind(env, args)?; - - self.internal_all(env, stmt) - } - - fn run_and_build_info_object( - &self, - query_fn: impl FnOnce() -> Result, - ) -> Result { - let total_changes_before = self.database.conn.total_changes(); - - query_fn()?; - - let last_insert_rowid = self.database.conn.last_insert_rowid(); - let changes = if self.database.conn.total_changes() == total_changes_before { - 0 - } else { - self.database.conn.changes() - }; - - Ok(RunResult { - changes, - last_insert_rowid, - }) - } - - #[napi] - pub fn all<'env>( - &self, - env: &'env Env, - args: Option>, - ) -> napi::Result> { - let stmt = self.check_and_bind(env, args)?; - - self.internal_all(env, stmt) - } - - fn internal_all<'env>( - &self, - env: &'env Env, - mut stmt: RefMut<'_, turso_core::Statement>, - ) -> napi::Result> { - let mut results = env.create_array(1)?; - let mut index = 0; - loop { - match stmt.step().map_err(into_napi_error)? { - turso_core::StepResult::Row => { - let row = stmt.row().unwrap(); - - match self.presentation_mode { - PresentationMode::Raw => { - let mut raw_array = env.create_array(row.len() as u32)?; - for (idx, value) in row.get_values().enumerate() { - let js_value = to_js_value(env, value.clone())?; - raw_array.set(idx as u32, js_value)?; - } - results.set_element(index, raw_array.coerce_to_object()?)?; - index += 1; - continue; - } - PresentationMode::Pluck => { - let (_, value) = - row.get_values().enumerate().next().ok_or(napi::Error::new( - napi::Status::GenericFailure, - "Pluck mode requires at least one column in the result", - ))?; - let js_value = to_js_value(env, value.clone())?; - results.set_element(index, js_value)?; - index += 1; - continue; - } - PresentationMode::None => { - let mut obj = Object::new(env)?; - for (idx, value) in row.get_values().enumerate() { - let key = stmt.get_column_name(idx); - let js_value = to_js_value(env, value.clone()); - obj.set_named_property(&key, js_value)?; - } - results.set_element(index, obj)?; - index += 1; - } - } - } - turso_core::StepResult::Done => { - break; - } - turso_core::StepResult::IO => { - stmt.run_once().map_err(into_napi_error)?; - } - turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => { - return Err(napi::Error::new( - napi::Status::GenericFailure, - format!("{:?}", stmt.step()), - )); + ValueType::String => { + let s = value.coerce_to_string()?.into_utf8()?; + turso_core::Value::Text(s.as_str()?.to_owned().into()) + } + ValueType::Boolean => { + let b: bool = unsafe { value.cast()? }; + turso_core::Value::Integer(if b { 1 } else { 0 }) + } + ValueType::Object => { + // Try to cast as Buffer first, fallback to string conversion + if let Ok(buffer) = unsafe { value.cast::() } { + turso_core::Value::Blob(buffer.to_vec()) + } else { + let s = value.coerce_to_string()?.into_utf8()?; + turso_core::Value::Text(s.as_str()?.to_owned().into()) } } - } - - Ok(results.to_unknown()) - } - - #[napi] - pub fn pluck(&mut self, pluck: Option) { - self.presentation_mode = match pluck { - Some(false) => PresentationMode::None, - _ => PresentationMode::Pluck, + _ => { + // Fallback to string conversion for unknown types + let s = value.coerce_to_string()?.into_utf8()?; + turso_core::Value::Text(s.as_str()?.to_owned().into()) + } }; + + stmt.bind_at(non_zero_idx, turso_value); + Ok(()) } + /// Step the statement and return result code: + /// 1 = Row available, 2 = Done, 3 = I/O needed #[napi] - pub fn expand() { - todo!() + pub fn step(&self) -> Result { + let mut stmt_ref = self.stmt.borrow_mut(); + let stmt = stmt_ref + .as_mut() + .ok_or_else(|| Error::new(Status::GenericFailure, "Statement has been finalized"))?; + + match stmt.step() { + Ok(turso_core::StepResult::Row) => Ok(STEP_ROW), + Ok(turso_core::StepResult::Done) => Ok(STEP_DONE), + Ok(turso_core::StepResult::IO) => Ok(STEP_IO), + Ok(turso_core::StepResult::Interrupt) => Err(Error::new( + Status::GenericFailure, + "Statement was interrupted", + )), + Ok(turso_core::StepResult::Busy) => { + Err(Error::new(Status::GenericFailure, "Database is busy")) + } + Err(e) => Err(Error::new( + Status::GenericFailure, + format!("Step failed: {e}"), + )), + } } + /// Get the current row data according to the presentation mode + #[napi] + pub fn row<'env>(&self, env: &'env Env) -> Result> { + let stmt_ref = self.stmt.borrow(); + let stmt = stmt_ref + .as_ref() + .ok_or_else(|| Error::new(Status::GenericFailure, "Statement has been finalized"))?; + + let row_data = stmt + .row() + .ok_or_else(|| Error::new(Status::GenericFailure, "No row data available"))?; + + let mode = self.mode.borrow(); + let row_value = match *mode { + PresentationMode::Raw => { + let mut raw_array = env.create_array(row_data.len() as u32)?; + for (idx, value) in row_data.get_values().enumerate() { + let js_value = to_js_value(env, value)?; + raw_array.set(idx as u32, js_value)?; + } + raw_array.coerce_to_object()?.to_unknown() + } + PresentationMode::Pluck => { + let (_, value) = + row_data + .get_values() + .enumerate() + .next() + .ok_or(napi::Error::new( + napi::Status::GenericFailure, + "Pluck mode requires at least one column in the result", + ))?; + to_js_value(env, value)? + } + PresentationMode::Expanded => { + let row = Object::new(env)?; + let raw_row = row.raw(); + let raw_env = env.raw(); + for idx in 0..row_data.len() { + let value = row_data.get_value(idx); + let column_name = &self.column_names[idx]; + let js_value = to_js_value(env, value)?; + unsafe { + napi::sys::napi_set_named_property( + raw_env, + raw_row, + column_name.as_ptr(), + js_value.raw(), + ); + } + } + row.to_unknown() + } + }; + + Ok(row_value) + } + + /// Sets the presentation mode to raw. #[napi] pub fn raw(&mut self, raw: Option) { - self.presentation_mode = match raw { - Some(false) => PresentationMode::None, + self.mode = RefCell::new(match raw { + Some(false) => PresentationMode::Expanded, _ => PresentationMode::Raw, - }; + }); } + /// Sets the presentation mode to pluck. #[napi] - pub fn columns() { - todo!() + pub fn pluck(&mut self, pluck: Option) { + self.mode = RefCell::new(match pluck { + Some(false) => PresentationMode::Expanded, + _ => PresentationMode::Pluck, + }); } + /// Finalizes the statement. #[napi] - pub fn bind(&mut self, env: Env, args: Option>) -> napi::Result { - self.check_and_bind(&env, args) - .map_err(with_sqlite_error_message)?; - self.binded = true; - - Ok(self.clone()) - } - - /// Check if the Statement is already binded by the `bind()` method - /// and bind values to variables. - fn check_and_bind( - &self, - env: &Env, - args: Option>, - ) -> napi::Result> { - let mut stmt = self.inner.borrow_mut(); - stmt.reset(); - if let Some(args) = args { - if self.binded { - let err = napi::Error::new( - into_convertible_type_error_message("TypeError"), - "The bind() method can only be invoked once per statement object", - ); - unsafe { - napi::JsTypeError::from(err).throw_into(env.raw()); - } - - return Err(napi::Error::from_status(napi::Status::PendingException)); - } - - if args.len() == 1 { - if matches!(args[0].get_type()?, napi::ValueType::Object) { - let obj: Object = args.into_iter().next().unwrap().coerce_to_object()?; - - if obj.is_array()? { - bind_positional_param_array(&mut stmt, &obj)?; - } else { - bind_host_params(&mut stmt, &obj)?; - } - } else { - bind_single_param(&mut stmt, args.into_iter().next().unwrap())?; - } - } else { - bind_positional_params(&mut stmt, args)?; - } - } - - Ok(stmt) + pub fn finalize(&self) -> Result<()> { + self.stmt.borrow_mut().take(); + Ok(()) } } -fn bind_positional_params( - stmt: &mut RefMut<'_, turso_core::Statement>, - args: Vec, -) -> Result<(), napi::Error> { - for (i, elem) in args.into_iter().enumerate() { - let value = from_js_value(elem)?; - stmt.bind_at(NonZeroUsize::new(i + 1).unwrap(), value); - } - Ok(()) +/// Async task for running the I/O loop. +pub struct IoLoopTask { + io: Arc, } -fn bind_host_params( - stmt: &mut RefMut<'_, turso_core::Statement>, - obj: &Object, -) -> Result<(), napi::Error> { - if first_key_is_number(obj) { - bind_numbered_params(stmt, obj)?; - } else { - bind_named_params(stmt, obj)?; +impl Task for IoLoopTask { + type Output = (); + type JsValue = (); + + fn compute(&mut self) -> napi::Result { + self.io.run_once().map_err(|e| { + napi::Error::new(napi::Status::GenericFailure, format!("IO error: {e}")) + })?; + Ok(()) } - Ok(()) -} - -fn first_key_is_number(obj: &Object) -> bool { - Object::keys(obj) - .iter() - .flatten() - .filter(|key| matches!(obj.has_own_property(key), Ok(result) if result)) - .take(1) - .any(|key| str::parse::(key).is_ok()) -} - -fn bind_numbered_params( - stmt: &mut RefMut<'_, turso_core::Statement>, - obj: &Object, -) -> Result<(), napi::Error> { - for key in Object::keys(obj)?.iter() { - let Ok(param_idx) = str::parse::(key) else { - return Err(napi::Error::new( - napi::Status::GenericFailure, - "cannot mix numbers and strings", - )); - }; - let Some(non_zero) = NonZero::new(param_idx as usize) else { - return Err(napi::Error::new( - napi::Status::GenericFailure, - "numbered parameters cannot be lower than 1", - )); - }; - - stmt.bind_at(non_zero, from_js_value(obj.get_named_property(key)?)?); + fn resolve(&mut self, _env: Env, _output: Self::Output) -> napi::Result { + Ok(()) } - Ok(()) } -fn bind_named_params( - stmt: &mut RefMut<'_, turso_core::Statement>, - obj: &Object, -) -> Result<(), napi::Error> { - for idx in 1..stmt.parameters_count() + 1 { - let non_zero_idx = NonZero::new(idx).unwrap(); - - let param = stmt.parameters().name(non_zero_idx); - let Some(name) = param else { - return Err(napi::Error::from_reason(format!( - "could not find named parameter with index {idx}" - ))); - }; - - let value = obj.get_named_property::(&name[1..])?; - stmt.bind_at(non_zero_idx, from_js_value(value)?); - } - - Ok(()) -} - -fn bind_positional_param_array( - stmt: &mut RefMut<'_, turso_core::Statement>, - obj: &Object, -) -> Result<(), napi::Error> { - assert!(obj.is_array()?, "bind_array can only be called with arrays"); - - for idx in 1..obj.get_array_length()? { - stmt.bind_at( - NonZero::new(idx as usize).unwrap(), - from_js_value(obj.get_element(idx)?)?, - ); - } - - Ok(()) -} - -fn bind_single_param( - stmt: &mut RefMut<'_, turso_core::Statement>, - obj: napi::Unknown, -) -> Result<(), napi::Error> { - stmt.bind_at(NonZero::new(1).unwrap(), from_js_value(obj)?); - Ok(()) -} - -fn to_js_value<'a>(env: &'a napi::Env, value: turso_core::Value) -> napi::Result> { +/// Convert a Turso value to a JavaScript value. +fn to_js_value<'a>(env: &'a napi::Env, value: &turso_core::Value) -> napi::Result> { match value { turso_core::Value::Null => ToNapiValue::into_unknown(Null, env), turso_core::Value::Integer(i) => ToNapiValue::into_unknown(i, env), @@ -648,37 +437,6 @@ fn to_js_value<'a>(env: &'a napi::Env, value: turso_core::Value) -> napi::Result } } -fn from_js_value(value: Unknown<'_>) -> napi::Result { - match value.get_type()? { - napi::ValueType::Undefined | napi::ValueType::Null | napi::ValueType::Unknown => { - Ok(turso_core::Value::Null) - } - napi::ValueType::Boolean => { - let b = value.coerce_to_bool()?; - Ok(turso_core::Value::Integer(b as i64)) - } - napi::ValueType::Number => { - let num = value.coerce_to_number()?.get_double()?; - if num.fract() == 0.0 { - Ok(turso_core::Value::Integer(num as i64)) - } else { - Ok(turso_core::Value::Float(num)) - } - } - napi::ValueType::String => { - let s = value.coerce_to_string()?; - Ok(turso_core::Value::Text(s.into_utf8()?.as_str()?.into())) - } - napi::ValueType::Symbol - | napi::ValueType::Object - | napi::ValueType::Function - | napi::ValueType::External => Err(napi::Error::new( - napi::Status::GenericFailure, - "Unsupported type", - )), - } -} - struct DatabaseFile { file: Arc, } @@ -711,7 +469,7 @@ impl turso_core::DatabaseStorage for DatabaseFile { fn write_page( &self, page_idx: usize, - buffer: Arc>, + buffer: Arc>, c: turso_core::Completion, ) -> turso_core::Result { let size = buffer.borrow().len(); @@ -719,6 +477,18 @@ impl turso_core::DatabaseStorage for DatabaseFile { self.file.pwrite(pos, buffer, c) } + fn write_pages( + &self, + page_idx: usize, + page_size: usize, + buffers: Vec>>, + c: turso_core::Completion, + ) -> turso_core::Result { + let pos = page_idx.saturating_sub(1) * page_size; + let c = self.file.pwritev(pos, buffers, c)?; + Ok(c) + } + fn sync(&self, c: turso_core::Completion) -> turso_core::Result { self.file.sync(c) } @@ -726,6 +496,7 @@ impl turso_core::DatabaseStorage for DatabaseFile { fn size(&self) -> turso_core::Result { self.file.size() } + fn truncate( &self, len: usize, @@ -735,31 +506,3 @@ impl turso_core::DatabaseStorage for DatabaseFile { Ok(c) } } - -#[inline] -fn into_napi_error(limbo_error: LimboError) -> napi::Error { - napi::Error::new(napi::Status::GenericFailure, format!("{limbo_error}")) -} - -#[inline] -fn into_napi_sqlite_error(limbo_error: LimboError) -> napi::Error { - napi::Error::new(String::from("SQLITE_ERROR"), format!("{limbo_error}")) -} - -#[inline] -fn into_napi_error_with_message( - error_code: String, - limbo_error: LimboError, -) -> napi::Error { - napi::Error::new(error_code, format!("{limbo_error}")) -} - -#[inline] -fn with_sqlite_error_message(err: napi::Error) -> napi::Error { - napi::Error::new("SQLITE_ERROR".to_owned(), err.reason.clone()) -} - -#[inline] -fn into_convertible_type_error_message(error_type: &str) -> String { - "[TURSO_CONVERT_TYPE] ".to_owned() + error_type -} diff --git a/bindings/javascript/sync.js b/bindings/javascript/sync.js index 64d4d10c6..bca456232 100644 --- a/bindings/javascript/sync.js +++ b/bindings/javascript/sync.js @@ -1,9 +1,15 @@ "use strict"; const { Database: NativeDB } = require("./index.js"); +const { bindParams } = require("./bind.js"); const SqliteError = require("./sqlite-error.js"); +// Step result constants +const STEP_ROW = 1; +const STEP_DONE = 2; +const STEP_IO = 3; + const convertibleErrorTypes = { TypeError }; const CONVERTIBLE_ERROR_PREFIX = "[TURSO_CONVERT_TYPE]"; @@ -138,12 +144,12 @@ class Database { if (typeof options !== "object") throw new TypeError("Expected second argument to be an options object"); - const simple = options["simple"]; const pragma = `PRAGMA ${source}`; - - return simple - ? this.db.pragma(source, { simple: true }) - : this.db.pragma(source); + + const stmt = this.prepare(pragma); + const results = stmt.all(); + + return results; } backup(filename, options) { @@ -181,7 +187,7 @@ class Database { */ exec(sql) { try { - this.db.exec(sql); + this.db.batch(sql); } catch (err) { throw convertError(err); } @@ -251,7 +257,29 @@ class Statement { * Executes the SQL statement and returns an info object. */ run(...bindParameters) { - return this.stmt.run(bindParameters.flat()); + const totalChangesBefore = this.db.db.totalChanges(); + + this.stmt.reset(); + bindParams(this.stmt, bindParameters); + for (;;) { + const stepResult = this.stmt.step(); + if (stepResult === STEP_IO) { + this.db.db.ioLoopSync(); + continue; + } + if (stepResult === STEP_DONE) { + break; + } + if (stepResult === STEP_ROW) { + // For run(), we don't need the row data, just continue + continue; + } + } + + const lastInsertRowid = this.db.db.lastInsertRowid(); + const changes = this.db.db.totalChanges() === totalChangesBefore ? 0 : this.db.db.changes(); + + return { changes, lastInsertRowid }; } /** @@ -260,7 +288,21 @@ class Statement { * @param bindParameters - The bind parameters for executing the statement. */ get(...bindParameters) { - return this.stmt.get(bindParameters.flat()); + this.stmt.reset(); + bindParams(this.stmt, bindParameters); + for (;;) { + const stepResult = this.stmt.step(); + if (stepResult === STEP_IO) { + this.db.db.ioLoopSync(); + continue; + } + if (stepResult === STEP_DONE) { + return undefined; + } + if (stepResult === STEP_ROW) { + return this.stmt.row(); + } + } } /** @@ -278,7 +320,23 @@ class Statement { * @param bindParameters - The bind parameters for executing the statement. */ all(...bindParameters) { - return this.stmt.all(bindParameters.flat()); + this.stmt.reset(); + bindParams(this.stmt, bindParameters); + const rows = []; + for (;;) { + const stepResult = this.stmt.step(); + if (stepResult === STEP_IO) { + this.db.db.ioLoopSync(); + continue; + } + if (stepResult === STEP_DONE) { + break; + } + if (stepResult === STEP_ROW) { + rows.push(this.stmt.row()); + } + } + return rows; } /** @@ -304,7 +362,8 @@ class Statement { */ bind(...bindParameters) { try { - return new Statement(this.stmt.bind(bindParameters.flat()), this.db); + bindParams(this.stmt, bindParameters); + return this; } catch (err) { throw convertError(err); } diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index 419e9481a..cac0a7280 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -317,7 +317,7 @@ impl Drop for Connection { #[allow(clippy::arc_with_non_send_sync)] #[pyfunction(signature = (path, experimental_indexes=None))] pub fn connect(path: &str, experimental_indexes: Option) -> Result { - let experimental_indexes = experimental_indexes.unwrap_or(false); + let experimental_indexes = experimental_indexes.unwrap_or(true); match turso_core::Connection::from_uri(path, experimental_indexes, false) { Ok((io, conn)) => Ok(Connection { conn, _io: io }), Err(e) => Err(PyErr::new::(format!( diff --git a/bindings/rust/Cargo.toml b/bindings/rust/Cargo.toml index a1cdf873b..f1d98b403 100644 --- a/bindings/rust/Cargo.toml +++ b/bindings/rust/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true description = "Turso Rust API" [features] -default = [] +default = ["experimental_indexes"] experimental_indexes = [] antithesis = ["turso_core/antithesis"] diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index a9fd2c062..4b8d68ffa 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -281,7 +281,7 @@ impl Connection { .inner .lock() .map_err(|e| Error::MutexError(e.to_string()))?; - let res = conn.cacheflush()?; + let _res = conn.cacheflush()?; Ok(()) } @@ -477,23 +477,26 @@ impl Rows { .inner .lock() .map_err(|e| Error::MutexError(e.to_string()))?; - match stmt.step() { - Ok(turso_core::StepResult::Row) => { + match stmt.step()? { + turso_core::StepResult::Row => { let row = stmt.row().unwrap(); return Ok(Some(Row { values: row.get_values().map(|v| v.to_owned()).collect(), })); } - Ok(turso_core::StepResult::Done) => return Ok(None), - Ok(turso_core::StepResult::IO) => { + turso_core::StepResult::Done => return Ok(None), + turso_core::StepResult::IO => { if let Err(e) = stmt.run_once() { return Err(e.into()); } continue; } - Ok(turso_core::StepResult::Busy) => return Ok(None), - Ok(turso_core::StepResult::Interrupt) => return Ok(None), - _ => return Ok(None), + turso_core::StepResult::Busy => { + return Err(Error::SqlExecutionFailure("database is locked".to_string())) + } + turso_core::StepResult::Interrupt => { + return Err(Error::SqlExecutionFailure("interrupted".to_string())) + } } } } diff --git a/bindings/wasm/lib.rs b/bindings/wasm/lib.rs deleted file mode 100644 index 1996d073f..000000000 --- a/bindings/wasm/lib.rs +++ /dev/null @@ -1,461 +0,0 @@ -#[cfg(all(feature = "web", feature = "nodejs"))] -compile_error!("Features 'web' and 'nodejs' cannot be enabled at the same time"); - -use js_sys::{Array, Object}; -use std::cell::RefCell; -use std::sync::Arc; -use turso_core::{Clock, Instant, OpenFlags, Result}; -use wasm_bindgen::prelude::*; - -#[allow(dead_code)] -#[wasm_bindgen] -pub struct Database { - db: Arc, - conn: Arc, -} - -#[allow(clippy::arc_with_non_send_sync)] -#[wasm_bindgen] -impl Database { - #[wasm_bindgen(constructor)] - pub fn new(path: &str) -> Database { - let io: Arc = Arc::new(PlatformIO { vfs: VFS::new() }); - let file = io.open_file(path, OpenFlags::Create, false).unwrap(); - let db_file = Arc::new(DatabaseFile::new(file)); - let db = turso_core::Database::open(io, path, db_file, false, false).unwrap(); - let conn = db.connect().unwrap(); - Database { db, conn } - } - - #[wasm_bindgen] - pub fn exec(&self, _sql: &str) { - self.conn.execute(_sql).unwrap(); - } - - #[wasm_bindgen] - pub fn prepare(&self, _sql: &str) -> Statement { - let stmt = self.conn.prepare(_sql).unwrap(); - Statement::new(RefCell::new(stmt), false) - } -} - -#[wasm_bindgen] -pub struct RowIterator { - inner: RefCell, -} - -#[wasm_bindgen] -impl RowIterator { - fn new(inner: RefCell) -> Self { - Self { inner } - } - - #[wasm_bindgen] - #[allow(clippy::should_implement_trait)] - pub fn next(&mut self) -> JsValue { - let mut stmt = self.inner.borrow_mut(); - match stmt.step() { - Ok(turso_core::StepResult::Row) => { - let row = stmt.row().unwrap(); - let row_array = Array::new(); - for value in row.get_values() { - let value = to_js_value(value); - row_array.push(&value); - } - JsValue::from(row_array) - } - Ok(turso_core::StepResult::IO) => JsValue::UNDEFINED, - Ok(turso_core::StepResult::Done) | Ok(turso_core::StepResult::Interrupt) => { - JsValue::UNDEFINED - } - - Ok(turso_core::StepResult::Busy) => JsValue::UNDEFINED, - Err(e) => panic!("Error: {e:?}"), - } - } -} - -#[wasm_bindgen] -pub struct Statement { - inner: RefCell, - raw: bool, -} - -#[wasm_bindgen] -impl Statement { - fn new(inner: RefCell, raw: bool) -> Self { - Self { inner, raw } - } - - #[wasm_bindgen] - pub fn raw(mut self, toggle: Option) -> Self { - self.raw = toggle.unwrap_or(true); - self - } - - pub fn get(&self) -> JsValue { - let mut stmt = self.inner.borrow_mut(); - match stmt.step() { - Ok(turso_core::StepResult::Row) => { - let row = stmt.row().unwrap(); - let row_array = js_sys::Array::new(); - for value in row.get_values() { - let value = to_js_value(value); - row_array.push(&value); - } - JsValue::from(row_array) - } - - Ok(turso_core::StepResult::IO) - | Ok(turso_core::StepResult::Done) - | Ok(turso_core::StepResult::Interrupt) - | Ok(turso_core::StepResult::Busy) => JsValue::UNDEFINED, - Err(e) => panic!("Error: {e:?}"), - } - } - - pub fn all(&self) -> js_sys::Array { - let array = js_sys::Array::new(); - loop { - let mut stmt = self.inner.borrow_mut(); - match stmt.step() { - Ok(turso_core::StepResult::Row) => { - let row = stmt.row().unwrap(); - let row_array = js_sys::Array::new(); - for value in row.get_values() { - let value = to_js_value(value); - row_array.push(&value); - } - array.push(&row_array); - } - Ok(turso_core::StepResult::IO) => {} - Ok(turso_core::StepResult::Interrupt) => break, - Ok(turso_core::StepResult::Done) => break, - Ok(turso_core::StepResult::Busy) => break, - Err(e) => panic!("Error: {e:?}"), - } - } - array - } - - #[wasm_bindgen] - pub fn iterate(self) -> JsValue { - let iterator = RowIterator::new(self.inner); - let iterator_obj = Object::new(); - - // Define the next method that will be called by JavaScript - let next_fn = js_sys::Function::new_with_args( - "", - "const value = this.iterator.next(); - const done = value === undefined; - return { - value, - done - };", - ); - - js_sys::Reflect::set(&iterator_obj, &JsValue::from_str("next"), &next_fn).unwrap(); - - js_sys::Reflect::set( - &iterator_obj, - &JsValue::from_str("iterator"), - &JsValue::from(iterator), - ) - .unwrap(); - - let symbol_iterator = js_sys::Function::new_no_args("return this;"); - js_sys::Reflect::set(&iterator_obj, &js_sys::Symbol::iterator(), &symbol_iterator).unwrap(); - - JsValue::from(iterator_obj) - } -} - -fn to_js_value(value: &turso_core::Value) -> JsValue { - match value { - turso_core::Value::Null => JsValue::null(), - turso_core::Value::Integer(i) => { - let i = *i; - if i >= i32::MIN as i64 && i <= i32::MAX as i64 { - JsValue::from(i as i32) - } else { - JsValue::from(i) - } - } - turso_core::Value::Float(f) => JsValue::from(*f), - turso_core::Value::Text(t) => JsValue::from_str(t.as_str()), - turso_core::Value::Blob(b) => js_sys::Uint8Array::from(b.as_slice()).into(), - } -} - -pub struct File { - vfs: VFS, - fd: i32, -} - -unsafe impl Send for File {} -unsafe impl Sync for File {} - -#[allow(dead_code)] -impl File { - fn new(vfs: VFS, fd: i32) -> Self { - Self { vfs, fd } - } -} - -impl turso_core::File for File { - fn lock_file(&self, _exclusive: bool) -> Result<()> { - // TODO - Ok(()) - } - - fn unlock_file(&self) -> Result<()> { - // TODO - Ok(()) - } - - fn pread( - &self, - pos: usize, - c: turso_core::Completion, - ) -> Result { - let r = match c.completion_type { - turso_core::CompletionType::Read(ref r) => r, - _ => unreachable!(), - }; - let nr = { - let mut buf = r.buf_mut(); - let buf: &mut [u8] = buf.as_mut_slice(); - self.vfs.pread(self.fd, buf, pos) - }; - r.complete(nr); - #[allow(clippy::arc_with_non_send_sync)] - Ok(c) - } - - fn pwrite( - &self, - pos: usize, - buffer: Arc>, - c: turso_core::Completion, - ) -> Result { - let w = match c.completion_type { - turso_core::CompletionType::Write(ref w) => w, - _ => unreachable!(), - }; - let buf = buffer.borrow(); - let buf: &[u8] = buf.as_slice(); - self.vfs.pwrite(self.fd, buf, pos); - w.complete(buf.len() as i32); - #[allow(clippy::arc_with_non_send_sync)] - Ok(c) - } - - fn sync(&self, c: turso_core::Completion) -> Result { - self.vfs.sync(self.fd); - c.complete(0); - #[allow(clippy::arc_with_non_send_sync)] - Ok(c) - } - - fn size(&self) -> Result { - Ok(self.vfs.size(self.fd)) - } - - fn truncate( - &self, - len: usize, - c: turso_core::Completion, - ) -> Result { - self.vfs.truncate(self.fd, len); - c.complete(0); - #[allow(clippy::arc_with_non_send_sync)] - Ok(c) - } -} - -pub struct PlatformIO { - vfs: VFS, -} -unsafe impl Send for PlatformIO {} -unsafe impl Sync for PlatformIO {} - -impl Clock for PlatformIO { - fn now(&self) -> Instant { - let date = Date::new(); - let ms_since_epoch = date.getTime(); - - Instant { - secs: (ms_since_epoch / 1000.0) as i64, - micros: ((ms_since_epoch % 1000.0) * 1000.0) as u32, - } - } -} - -impl turso_core::IO for PlatformIO { - fn open_file( - &self, - path: &str, - _flags: OpenFlags, - _direct: bool, - ) -> Result> { - let fd = self.vfs.open(path, "a+"); - Ok(Arc::new(File { - vfs: VFS::new(), - fd, - })) - } - - fn wait_for_completion(&self, c: turso_core::Completion) -> Result<()> { - while !c.is_completed() { - self.run_once()?; - } - Ok(()) - } - - fn run_once(&self) -> Result<()> { - Ok(()) - } - - fn generate_random_number(&self) -> i64 { - let mut buf = [0u8; 8]; - getrandom::getrandom(&mut buf).unwrap(); - i64::from_ne_bytes(buf) - } - - fn get_memory_io(&self) -> Arc { - Arc::new(turso_core::MemoryIO::new()) - } -} - -#[wasm_bindgen] -extern "C" { - type Date; - - #[wasm_bindgen(constructor)] - fn new() -> Date; - - #[wasm_bindgen(method, getter)] - fn toISOString(this: &Date) -> String; - - #[wasm_bindgen(method)] - fn getTime(this: &Date) -> f64; -} - -pub struct DatabaseFile { - file: Arc, -} - -unsafe impl Send for DatabaseFile {} -unsafe impl Sync for DatabaseFile {} - -impl DatabaseFile { - pub fn new(file: Arc) -> Self { - Self { file } - } -} - -impl turso_core::DatabaseStorage for DatabaseFile { - fn read_page(&self, page_idx: usize, c: turso_core::Completion) -> Result<()> { - let r = match c.completion_type { - turso_core::CompletionType::Read(ref r) => r, - _ => unreachable!(), - }; - let size = r.buf().len(); - assert!(page_idx > 0); - if !(512..=65536).contains(&size) || size & (size - 1) != 0 { - return Err(turso_core::LimboError::NotADB); - } - let pos = (page_idx - 1) * size; - self.file.pread(pos, c.into())?; - Ok(()) - } - - fn write_page( - &self, - page_idx: usize, - buffer: Arc>, - c: turso_core::Completion, - ) -> Result<()> { - let size = buffer.borrow().len(); - let pos = (page_idx - 1) * size; - self.file.pwrite(pos, buffer, c.into())?; - Ok(()) - } - - fn sync(&self, c: turso_core::Completion) -> Result<()> { - let _ = self.file.sync(c.into())?; - Ok(()) - } - - fn size(&self) -> Result { - self.file.size() - } - - fn truncate(&self, len: usize, c: turso_core::Completion) -> Result<()> { - self.file.truncate(len, c)?; - Ok(()) - } -} - -#[cfg(all(feature = "web", not(feature = "nodejs")))] -#[wasm_bindgen(module = "/web/src/web-vfs.js")] -extern "C" { - type VFS; - #[wasm_bindgen(constructor)] - fn new() -> VFS; - - #[wasm_bindgen(method)] - fn open(this: &VFS, path: &str, flags: &str) -> i32; - - #[wasm_bindgen(method)] - fn close(this: &VFS, fd: i32) -> bool; - - #[wasm_bindgen(method)] - fn pwrite(this: &VFS, fd: i32, buffer: &[u8], offset: usize) -> i32; - - #[wasm_bindgen(method)] - fn pread(this: &VFS, fd: i32, buffer: &mut [u8], offset: usize) -> i32; - - #[wasm_bindgen(method)] - fn size(this: &VFS, fd: i32) -> u64; - - #[wasm_bindgen(method)] - fn truncate(this: &VFS, fd: i32, len: usize); - - #[wasm_bindgen(method)] - fn sync(this: &VFS, fd: i32); -} - -#[cfg(all(feature = "nodejs", not(feature = "web")))] -#[wasm_bindgen(module = "/node/src/vfs.cjs")] -extern "C" { - type VFS; - #[wasm_bindgen(constructor)] - fn new() -> VFS; - - #[wasm_bindgen(method)] - fn open(this: &VFS, path: &str, flags: &str) -> i32; - - #[wasm_bindgen(method)] - fn close(this: &VFS, fd: i32) -> bool; - - #[wasm_bindgen(method)] - fn pwrite(this: &VFS, fd: i32, buffer: &[u8], offset: usize) -> i32; - - #[wasm_bindgen(method)] - fn pread(this: &VFS, fd: i32, buffer: &mut [u8], offset: usize) -> i32; - - #[wasm_bindgen(method)] - fn size(this: &VFS, fd: i32) -> u64; - - #[wasm_bindgen(method)] - fn truncate(this: &VFS, fd: i32, len: usize); - - #[wasm_bindgen(method)] - fn sync(this: &VFS, fd: i32); -} - -#[wasm_bindgen(start)] -pub fn init() { - console_error_panic_hook::set_once(); -} diff --git a/cli/app.rs b/cli/app.rs index 2b422af86..d624ce3a7 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -61,7 +61,7 @@ pub struct Opts { #[clap(long, help = "Enable experimental MVCC feature")] pub experimental_mvcc: bool, #[clap(long, help = "Enable experimental indexing feature")] - pub experimental_indexes: bool, + pub experimental_indexes: Option, #[clap(short = 't', long, help = "specify output file for log traces")] pub tracing_output: Option, #[clap(long, help = "Start MCP server instead of interactive shell")] @@ -119,8 +119,9 @@ impl Limbo { .database .as_ref() .map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string()); + let indexes_enabled = opts.experimental_indexes.unwrap_or(true); let (io, conn) = if db_file.contains([':', '?', '&', '#']) { - Connection::from_uri(&db_file, opts.experimental_indexes, opts.experimental_mvcc)? + Connection::from_uri(&db_file, indexes_enabled, opts.experimental_mvcc)? } else { let flags = if opts.readonly { OpenFlags::ReadOnly @@ -131,7 +132,7 @@ impl Limbo { &db_file, opts.vfs.as_ref(), flags, - opts.experimental_indexes, + indexes_enabled, opts.experimental_mvcc, )?; let conn = db.connect()?; diff --git a/core/Cargo.toml b/core/Cargo.toml index 651282010..343b63695 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -19,7 +19,7 @@ default = ["fs", "uuid", "time", "json", "series"] fs = ["turso_ext/vfs"] json = [] uuid = ["dep:uuid"] -io_uring = ["dep:io-uring", "rustix/io_uring", "dep:libc"] +io_uring = ["dep:io-uring", "rustix/io_uring"] time = [] fuzz = [] omit_autovacuum = [] @@ -29,10 +29,12 @@ series = [] [target.'cfg(target_os = "linux")'.dependencies] io-uring = { version = "0.7.5", optional = true } +libc = { version = "0.2.172" } [target.'cfg(target_family = "unix")'.dependencies] polling = "3.7.4" rustix = { version = "1.0.5", features = ["fs"] } +libc = { version = "0.2.172" } [target.'cfg(not(target_family = "wasm"))'.dependencies] mimalloc = { version = "0.1.46", default-features = false } @@ -44,7 +46,6 @@ turso_ext = { workspace = true, features = ["core_only"] } cfg_block = "0.1.1" fallible-iterator = "0.3.0" hex = "0.4.3" -libc = { version = "0.2.172", optional = true } turso_sqlite3_parser = { workspace = true } thiserror = "1.0.61" getrandom = { version = "0.2.15" } @@ -70,6 +71,8 @@ serde = { workspace = true, optional = true, features = ["derive"] } paste = "1.0.15" uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true } tempfile = "3.8.0" +pack1 = { version = "1.0.0", features = ["bytemuck"] } +bytemuck = "1.23.1" [build-dependencies] chrono = { version = "0.4.38", default-features = false } diff --git a/core/benches/benchmark.rs b/core/benches/benchmark.rs index 8871dc566..008516a6f 100644 --- a/core/benches/benchmark.rs +++ b/core/benches/benchmark.rs @@ -37,14 +37,16 @@ fn bench_open(criterion: &mut Criterion) { let io = Arc::new(PlatformIO::new().unwrap()); let db = Database::open_file(io.clone(), "../testing/schema_5k.db", false, false).unwrap(); - black_box(db.connect().unwrap()); + let conn = db.connect().unwrap(); + conn.execute("SELECT * FROM table_0").unwrap(); }); }); if enable_rusqlite { group.bench_function(BenchmarkId::new("sqlite_schema", ""), |b| { b.iter(|| { - black_box(rusqlite::Connection::open("../testing/schema_5k.db").unwrap()); + let conn = rusqlite::Connection::open("../testing/schema_5k.db").unwrap(); + conn.execute("SELECT * FROM table_0", ()).unwrap(); }); }); } diff --git a/core/benches/mvcc_benchmark.rs b/core/benches/mvcc_benchmark.rs index 15faffbac..25c05ecb3 100644 --- a/core/benches/mvcc_benchmark.rs +++ b/core/benches/mvcc_benchmark.rs @@ -1,13 +1,29 @@ +use std::sync::Arc; + use criterion::async_executor::FuturesExecutor; use criterion::{criterion_group, criterion_main, Criterion, Throughput}; use pprof::criterion::{Output, PProfProfiler}; use turso_core::mvcc::clock::LocalClock; use turso_core::mvcc::database::{MvStore, Row, RowID}; +use turso_core::types::{ImmutableRecord, Text}; +use turso_core::{Connection, Database, MemoryIO, Value}; -fn bench_db() -> MvStore { - let clock = LocalClock::default(); - let storage = turso_core::mvcc::persistent_storage::Storage::new_noop(); - MvStore::new(clock, storage) +struct BenchDb { + _db: Arc, + conn: Arc, + mvcc_store: Arc>, +} + +fn bench_db() -> BenchDb { + let io = Arc::new(MemoryIO::new()); + let db = Database::open_file(io.clone(), ":memory:", true, true).unwrap(); + let conn = db.connect().unwrap(); + let mvcc_store = db.get_mv_store().unwrap().clone(); + BenchDb { + _db: db, + conn, + mvcc_store, + } } fn bench(c: &mut Criterion) { @@ -16,107 +32,131 @@ fn bench(c: &mut Criterion) { let db = bench_db(); group.bench_function("begin_tx + rollback_tx", |b| { + let db = bench_db(); b.to_async(FuturesExecutor).iter(|| async { - let tx_id = db.begin_tx(); - db.rollback_tx(tx_id) + let conn = db.conn.clone(); + let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()); + db.mvcc_store.rollback_tx(tx_id, conn.get_pager().clone()) }) }); let db = bench_db(); group.bench_function("begin_tx + commit_tx", |b| { b.to_async(FuturesExecutor).iter(|| async { - let tx_id = db.begin_tx(); - db.commit_tx(tx_id) + let conn = &db.conn; + let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()); + db.mvcc_store + .commit_tx(tx_id, conn.get_pager().clone(), conn) }) }); let db = bench_db(); group.bench_function("begin_tx-read-commit_tx", |b| { b.to_async(FuturesExecutor).iter(|| async { - let tx_id = db.begin_tx(); - db.read( - tx_id, - RowID { - table_id: 1, - row_id: 1, - }, - ) - .unwrap(); - db.commit_tx(tx_id) + let conn = &db.conn; + let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()); + db.mvcc_store + .read( + tx_id, + RowID { + table_id: 1, + row_id: 1, + }, + ) + .unwrap(); + db.mvcc_store + .commit_tx(tx_id, conn.get_pager().clone(), conn) }) }); let db = bench_db(); + let record = ImmutableRecord::from_values(&vec![Value::Text(Text::new("World"))], 1); + let record_data = record.as_blob(); group.bench_function("begin_tx-update-commit_tx", |b| { b.to_async(FuturesExecutor).iter(|| async { - let tx_id = db.begin_tx(); - db.update( - tx_id, - Row { - id: RowID { - table_id: 1, - row_id: 1, + let conn = &db.conn; + let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()); + db.mvcc_store + .update( + tx_id, + Row { + id: RowID { + table_id: 1, + row_id: 1, + }, + data: record_data.clone(), + column_count: 1, }, - data: "World".to_string().into_bytes(), - }, - ) - .unwrap(); - db.commit_tx(tx_id) + conn.get_pager().clone(), + ) + .unwrap(); + db.mvcc_store + .commit_tx(tx_id, conn.get_pager().clone(), conn) + .unwrap(); }) }); let db = bench_db(); - let tx = db.begin_tx(); - db.insert( - tx, - Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }, - ) - .unwrap(); - group.bench_function("read", |b| { - b.to_async(FuturesExecutor).iter(|| async { - db.read( - tx, - RowID { + let tx_id = db.mvcc_store.begin_tx(db.conn.get_pager().clone()); + db.mvcc_store + .insert( + tx_id, + Row { + id: RowID { table_id: 1, row_id: 1, }, - ) - .unwrap(); + data: record_data.clone(), + column_count: 1, + }, + ) + .unwrap(); + group.bench_function("read", |b| { + b.to_async(FuturesExecutor).iter(|| async { + db.mvcc_store + .read( + tx_id, + RowID { + table_id: 1, + row_id: 1, + }, + ) + .unwrap(); }) }); let db = bench_db(); - let tx = db.begin_tx(); - db.insert( - tx, - Row { - id: RowID { - table_id: 1, - row_id: 1, + let tx_id = db.mvcc_store.begin_tx(db.conn.get_pager().clone()); + let conn = &db.conn; + db.mvcc_store + .insert( + tx_id, + Row { + id: RowID { + table_id: 1, + row_id: 1, + }, + data: record_data.clone(), + column_count: 1, }, - data: "Hello".to_string().into_bytes(), - }, - ) - .unwrap(); + ) + .unwrap(); group.bench_function("update", |b| { b.to_async(FuturesExecutor).iter(|| async { - db.update( - tx, - Row { - id: RowID { - table_id: 1, - row_id: 1, + db.mvcc_store + .update( + tx_id, + Row { + id: RowID { + table_id: 1, + row_id: 1, + }, + data: record_data.clone(), + column_count: 1, }, - data: "World".to_string().into_bytes(), - }, - ) - .unwrap(); + conn.get_pager().clone(), + ) + .unwrap(); }) }); } diff --git a/core/error.rs b/core/error.rs index 2f22dd706..97c6d563b 100644 --- a/core/error.rs +++ b/core/error.rs @@ -55,12 +55,20 @@ pub enum LimboError { IntegerOverflow, #[error("Schema is locked for write")] SchemaLocked, + #[error("Runtime error: database table is locked")] + TableLocked, #[error("Error: Resource is read-only")] ReadOnly, #[error("Database is busy")] Busy, #[error("Conflict: {0}")] Conflict(String), + #[error("Transaction terminated")] + TxTerminated, + #[error("Write-write conflict")] + WriteWriteConflict, + #[error("No such transaction ID: {0}")] + NoSuchTransactionID(String), } #[macro_export] diff --git a/core/io/io_uring.rs b/core/io/io_uring.rs index f33c04db3..b2afeb652 100644 --- a/core/io/io_uring.rs +++ b/core/io/io_uring.rs @@ -2,38 +2,43 @@ use super::{common, Completion, CompletionInner, File, OpenFlags, IO}; use crate::io::clock::{Clock, Instant}; +use crate::storage::wal::CKPT_BATCH_PAGES; use crate::{turso_assert, LimboError, MemoryIO, Result}; use rustix::fs::{self, FlockOperation, OFlags}; -use std::cell::RefCell; -use std::collections::VecDeque; -use std::fmt; -use std::io::ErrorKind; -use std::os::fd::AsFd; -use std::os::unix::io::AsRawFd; -use std::rc::Rc; -use std::sync::Arc; -use thiserror::Error; +use std::{ + cell::RefCell, + collections::{HashMap, VecDeque}, + io::ErrorKind, + ops::Deref, + os::{fd::AsFd, unix::io::AsRawFd}, + rc::Rc, + sync::Arc, +}; use tracing::{debug, trace}; +/// Size of the io_uring submission and completion queues const ENTRIES: u32 = 512; + +/// Idle timeout for the sqpoll kernel thread before it needs +/// to be woken back up by a call IORING_ENTER_SQ_WAKEUP flag. +/// (handled by the io_uring crate in `submit_and_wait`) const SQPOLL_IDLE: u32 = 1000; + +/// Number of file descriptors we preallocate for io_uring. +/// NOTE: we may need to increase this when `attach` is fully implemented. const FILES: u32 = 8; -#[derive(Debug, Error)] -enum UringIOError { - IOUringCQError(i32), -} +/// Number of Vec> we preallocate on initialization +const IOVEC_POOL_SIZE: usize = 64; -impl fmt::Display for UringIOError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - UringIOError::IOUringCQError(code) => write!( - f, - "IOUring completion queue error occurred with code {code}", - ), - } - } -} +/// Maximum number of iovec entries per writev operation. +/// IOV_MAX is typically 1024, but we limit it to a smaller number +const MAX_IOVEC_ENTRIES: usize = CKPT_BATCH_PAGES; + +/// Maximum number of I/O operations to wait for in a single run, +/// waiting for > 1 can reduce the amount of `io_uring_enter` syscalls we +/// make, but can increase single operation latency. +const MAX_WAIT: usize = 4; pub struct UringIO { inner: Rc>, @@ -45,6 +50,8 @@ unsafe impl Sync for UringIO {} struct WrappedIOUring { ring: io_uring::IoUring, pending_ops: usize, + writev_states: HashMap, + iov_pool: IovecPool, } struct InnerUringIO { @@ -52,6 +59,39 @@ struct InnerUringIO { free_files: VecDeque, } +/// preallocated vec of iovec arrays to avoid allocations during writev operations +struct IovecPool { + pool: Vec>, +} + +impl IovecPool { + fn new() -> Self { + let pool = (0..IOVEC_POOL_SIZE) + .map(|_| { + Box::new( + [libc::iovec { + iov_base: std::ptr::null_mut(), + iov_len: 0, + }; MAX_IOVEC_ENTRIES], + ) + }) + .collect(); + Self { pool } + } + + #[inline(always)] + fn acquire(&mut self) -> Option> { + self.pool.pop() + } + + #[inline(always)] + fn release(&mut self, iovec: Box<[libc::iovec; MAX_IOVEC_ENTRIES]>) { + if self.pool.len() < IOVEC_POOL_SIZE { + self.pool.push(iovec); + } + } +} + impl UringIO { pub fn new() -> Result { let ring = match io_uring::IoUring::builder() @@ -69,6 +109,8 @@ impl UringIO { ring: WrappedIOUring { ring, pending_ops: 0, + writev_states: HashMap::new(), + iov_pool: IovecPool::new(), }, free_files: (0..FILES).collect(), }; @@ -79,6 +121,126 @@ impl UringIO { } } +/// io_uring crate decides not to export their `UseFixed` trait, so we +/// are forced to use a macro here to handle either fixed or raw file descriptors. +macro_rules! with_fd { + ($file:expr, |$fd:ident| $body:expr) => { + match $file.id() { + Some(id) => { + let $fd = io_uring::types::Fixed(id); + $body + } + None => { + let $fd = io_uring::types::Fd($file.as_raw_fd()); + $body + } + } + }; +} + +/// wrapper type to represent a possibly registered file descriptor, +/// only used in WritevState, and piggy-backs on the available methods from +/// `UringFile`, so we don't have to store the file on `WritevState`. +enum Fd { + Fixed(u32), + RawFd(i32), +} + +impl Fd { + /// to match the behavior of the File, we need to implement the same methods + fn id(&self) -> Option { + match self { + Fd::Fixed(id) => Some(*id), + Fd::RawFd(_) => None, + } + } + /// ONLY to be called by the macro, in the case where id() is None + fn as_raw_fd(&self) -> i32 { + match self { + Fd::RawFd(fd) => *fd, + _ => panic!("Cannot call as_raw_fd on a Fixed Fd"), + } + } +} + +/// State to track an ongoing writev operation in +/// the case of a partial write. +struct WritevState { + /// File descriptor/id of the file we are writing to + file_id: Fd, + /// absolute file offset for next submit + file_pos: usize, + /// current buffer index in `bufs` + current_buffer_idx: usize, + /// intra-buffer offset + current_buffer_offset: usize, + /// total bytes written so far + total_written: usize, + /// cache the sum of all buffer lengths for the total expected write + total_len: usize, + /// buffers to write + bufs: Vec>>, + /// we keep the last iovec allocation alive until final CQE + last_iov_allocation: Option>, +} + +impl WritevState { + fn new(file: &UringFile, pos: usize, bufs: Vec>>) -> Self { + let file_id = file + .id() + .map(Fd::Fixed) + .unwrap_or_else(|| Fd::RawFd(file.as_raw_fd())); + let total_len = bufs.iter().map(|b| b.borrow().len()).sum(); + Self { + file_id, + file_pos: pos, + current_buffer_idx: 0, + current_buffer_offset: 0, + total_written: 0, + bufs, + last_iov_allocation: None, + total_len, + } + } + + #[inline(always)] + fn remaining(&self) -> usize { + self.total_len - self.total_written + } + + /// Advance (idx, off, pos) after written bytes + #[inline(always)] + fn advance(&mut self, written: usize) { + let mut remaining = written; + while remaining > 0 { + let current_buf_len = { + let r = self.bufs[self.current_buffer_idx].borrow(); + r.len() + }; + let left = current_buf_len - self.current_buffer_offset; + if remaining < left { + self.current_buffer_offset += remaining; + self.file_pos += remaining; + remaining = 0; + } else { + remaining -= left; + self.file_pos += left; + self.current_buffer_idx += 1; + self.current_buffer_offset = 0; + } + } + self.total_written += written; + } + + #[inline(always)] + /// Free the allocation that keeps the iovec array alive while writev is ongoing + fn free_last_iov(&mut self, pool: &mut IovecPool) { + if let Some(allocation) = self.last_iov_allocation.take() { + pool.release(allocation); + } + } +} + impl InnerUringIO { fn register_file(&mut self, fd: i32) -> Result { if let Some(slot) = self.free_files.pop_front() { @@ -106,33 +268,119 @@ impl WrappedIOUring { fn submit_entry(&mut self, entry: &io_uring::squeue::Entry) { trace!("submit_entry({:?})", entry); unsafe { - self.ring - .submission() - .push(entry) - .expect("submission queue is full"); + let mut sub = self.ring.submission_shared(); + match sub.push(entry) { + Ok(_) => self.pending_ops += 1, + Err(e) => { + tracing::error!("Failed to submit entry: {e}"); + self.ring.submit().expect("failed to submit entry"); + sub.push(entry).expect("failed to push entry after submit"); + self.pending_ops += 1; + } + } } - self.pending_ops += 1; } - fn wait_for_completion(&mut self) -> Result<()> { - self.ring.submit_and_wait(1)?; + fn submit_and_wait(&mut self) -> Result<()> { + if self.empty() { + return Ok(()); + } + let wants = std::cmp::min(self.pending_ops, MAX_WAIT); + tracing::trace!("submit_and_wait for {wants} pending operations to complete"); + self.ring.submit_and_wait(wants)?; Ok(()) } - fn get_completion(&mut self) -> Option { - // NOTE: This works because CompletionQueue's next function pops the head of the queue. This is not normal behaviour of iterators - let entry = self.ring.completion().next(); - if entry.is_some() { - trace!("get_completion({:?})", entry); - // consumed an entry from completion queue, update pending_ops - self.pending_ops -= 1; - } - entry - } - fn empty(&self) -> bool { self.pending_ops == 0 } + + /// Submit or resubmit a writev operation + fn submit_writev(&mut self, key: u64, mut st: WritevState) { + st.free_last_iov(&mut self.iov_pool); + let mut iov_allocation = self.iov_pool.acquire().unwrap_or_else(|| { + // Fallback: allocate a new one if pool is exhausted + Box::new( + [libc::iovec { + iov_base: std::ptr::null_mut(), + iov_len: 0, + }; MAX_IOVEC_ENTRIES], + ) + }); + let mut iov_count = 0; + for (idx, buffer) in st + .bufs + .iter() + .enumerate() + .skip(st.current_buffer_idx) + .take(MAX_IOVEC_ENTRIES) + { + let buf = buffer.borrow(); + let buf_slice = buf.as_slice(); + // ensure we are providing a pointer to the proper offset in the buffer + let slice = if idx == st.current_buffer_idx { + &buf_slice[st.current_buffer_offset..] + } else { + buf_slice + }; + if slice.is_empty() { + continue; + } + iov_allocation[iov_count] = libc::iovec { + iov_base: slice.as_ptr() as *mut _, + iov_len: slice.len(), + }; + iov_count += 1; + } + // Store the pointers and get the pointer to the iovec array that we pass + // to the writev operation, and keep the array itself alive + let ptr = iov_allocation.as_ptr() as *mut libc::iovec; + st.last_iov_allocation = Some(iov_allocation); + + let entry = with_fd!(st.file_id, |fd| { + io_uring::opcode::Writev::new(fd, ptr, iov_count as u32) + .offset(st.file_pos as u64) + .build() + .user_data(key) + }); + // track the current state in case we get a partial write + self.writev_states.insert(key, st); + self.submit_entry(&entry); + } + + fn handle_writev_completion(&mut self, mut state: WritevState, user_data: u64, result: i32) { + if result < 0 { + let err = std::io::Error::from_raw_os_error(result); + tracing::error!("writev failed (user_data: {}): {}", user_data, err); + state.free_last_iov(&mut self.iov_pool); + completion_from_key(user_data).complete(result); + return; + } + + let written = result as usize; + state.advance(written); + match state.remaining() { + 0 => { + tracing::info!( + "writev operation completed: wrote {} bytes", + state.total_written + ); + // write complete, return iovec to pool + state.free_last_iov(&mut self.iov_pool); + completion_from_key(user_data).complete(state.total_written as i32); + } + remaining => { + tracing::trace!( + "resubmitting writev operation for user_data {}: wrote {} bytes, remaining {}", + user_data, + written, + remaining + ); + // partial write, submit next + self.submit_writev(user_data, state); + } + } + } } impl IO for UringIO { @@ -179,26 +427,28 @@ impl IO for UringIO { trace!("run_once()"); let mut inner = self.inner.borrow_mut(); let ring = &mut inner.ring; - if ring.empty() { return Ok(()); } - - ring.wait_for_completion()?; - while let Some(cqe) = ring.get_completion() { + ring.submit_and_wait()?; + loop { + let Some(cqe) = ring.ring.completion().next() else { + return Ok(()); + }; + ring.pending_ops -= 1; + let user_data = cqe.user_data(); let result = cqe.result(); - if result < 0 { - return Err(LimboError::UringIOError(format!( - "{} cqe: {:?}", - UringIOError::IOUringCQError(result), - cqe - ))); + turso_assert!( + user_data != 0, + "user_data must not be zero, we dont submit linked timeouts or cancelations that would cause this" + ); + if let Some(state) = ring.writev_states.remove(&user_data) { + // if we have ongoing writev state, handle it separately and don't call completion + ring.handle_writev_completion(state, user_data, result); + continue; } - let ud = cqe.user_data(); - turso_assert!(ud > 0, "therea are no linked timeouts or cancelations, all cqe user_data should be valid arc pointers"); - completion_from_key(ud).complete(result); + completion_from_key(user_data).complete(result) } - Ok(()) } fn generate_random_number(&self) -> i64 { @@ -242,24 +492,22 @@ pub struct UringFile { id: Option, } +impl Deref for UringFile { + type Target = std::fs::File; + fn deref(&self) -> &Self::Target { + &self.file + } +} + +impl UringFile { + fn id(&self) -> Option { + self.id + } +} + unsafe impl Send for UringFile {} unsafe impl Sync for UringFile {} -macro_rules! with_fd { - ($file:expr, |$fd:ident| $body:expr) => { - match $file.id { - Some(id) => { - let $fd = io_uring::types::Fixed(id); - $body - } - None => { - let $fd = io_uring::types::Fd($file.file.as_raw_fd()); - $body - } - } - }; -} - impl File for UringFile { fn lock_file(&self, exclusive: bool) -> Result<()> { let fd = self.file.as_fd(); @@ -350,6 +598,24 @@ impl File for UringFile { Ok(c) } + fn pwritev( + &self, + pos: usize, + bufs: Vec>>, + c: Completion, + ) -> Result { + // for a single buffer use pwrite directly + if bufs.len().eq(&1) { + return self.pwrite(pos, bufs[0].clone(), c.clone()); + } + tracing::trace!("pwritev(pos = {}, bufs.len() = {})", pos, bufs.len()); + let mut io = self.io.borrow_mut(); + // create state to track ongoing writev operation + let state = WritevState::new(self, pos, bufs); + io.ring.submit_writev(get_key(c.clone()), state); + Ok(c) + } + fn size(&self) -> Result { Ok(self.file.metadata()?.len()) } diff --git a/core/io/memory.rs b/core/io/memory.rs index 7dbf05d50..4d056aeb4 100644 --- a/core/io/memory.rs +++ b/core/io/memory.rs @@ -187,6 +187,49 @@ impl File for MemoryFile { Ok(c) } + fn pwritev( + &self, + pos: usize, + buffers: Vec>>, + c: Completion, + ) -> Result { + let mut offset = pos; + let mut total_written = 0; + + for buffer in buffers { + let buf = buffer.borrow(); + let buf_len = buf.len(); + if buf_len == 0 { + continue; + } + + let mut remaining = buf_len; + let mut buf_offset = 0; + let data = &buf.as_slice(); + + while remaining > 0 { + let page_no = offset / PAGE_SIZE; + let page_offset = offset % PAGE_SIZE; + let bytes_to_write = remaining.min(PAGE_SIZE - page_offset); + + { + let page = self.get_or_allocate_page(page_no); + page[page_offset..page_offset + bytes_to_write] + .copy_from_slice(&data[buf_offset..buf_offset + bytes_to_write]); + } + + offset += bytes_to_write; + buf_offset += bytes_to_write; + remaining -= bytes_to_write; + } + total_written += buf_len; + } + c.complete(total_written as i32); + self.size + .set(core::cmp::max(pos + total_written, self.size.get())); + Ok(c) + } + fn size(&self) -> Result { Ok(self.size.get() as u64) } diff --git a/core/io/mod.rs b/core/io/mod.rs index 82ef51313..8560216e8 100644 --- a/core/io/mod.rs +++ b/core/io/mod.rs @@ -18,6 +18,46 @@ pub trait File: Send + Sync { fn pwrite(&self, pos: usize, buffer: Arc>, c: Completion) -> Result; fn sync(&self, c: Completion) -> Result; + fn pwritev( + &self, + pos: usize, + buffers: Vec>>, + c: Completion, + ) -> Result { + use std::sync::atomic::{AtomicUsize, Ordering}; + if buffers.is_empty() { + c.complete(0); + return Ok(c); + } + // naive default implementation can be overridden on backends where it makes sense to + let mut pos = pos; + let outstanding = Arc::new(AtomicUsize::new(buffers.len())); + let total_written = Arc::new(AtomicUsize::new(0)); + + for buf in buffers { + let len = buf.borrow().len(); + let child_c = { + let c_main = c.clone(); + let outstanding = outstanding.clone(); + let total_written = total_written.clone(); + Completion::new_write(move |n| { + // accumulate bytes actually reported by the backend + total_written.fetch_add(n as usize, Ordering::Relaxed); + if outstanding.fetch_sub(1, Ordering::AcqRel) == 1 { + // last one finished + c_main.complete(total_written.load(Ordering::Acquire) as i32); + } + }) + }; + if let Err(e) = self.pwrite(pos, buf.clone(), child_c) { + // best-effort: mark as done so caller won't wait forever + c.complete(-1); + return Err(e); + } + pos += len; + } + Ok(c) + } fn size(&self) -> Result; fn truncate(&self, len: usize, c: Completion) -> Result; } @@ -304,10 +344,10 @@ cfg_block! { pub use unix::UnixIO as PlatformIO; } - #[cfg(target_os = "windows")] { + #[cfg(target_os = "windows")] { mod windows; pub use windows::WindowsIO as PlatformIO; - pub use PlatformIO as SyscallIO; + pub use PlatformIO as SyscallIO; } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows", target_os = "android", target_os = "ios")))] { diff --git a/core/io/unix.rs b/core/io/unix.rs index 9cb50a3f8..7e73e6904 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -1,15 +1,15 @@ +use super::{Completion, File, MemoryIO, OpenFlags, IO}; use crate::error::LimboError; +use crate::io::clock::{Clock, Instant}; use crate::io::common; use crate::Result; - -use super::{Completion, File, MemoryIO, OpenFlags, IO}; -use crate::io::clock::{Clock, Instant}; use polling::{Event, Events, Poller}; use rustix::{ fd::{AsFd, AsRawFd}, fs::{self, FlockOperation, OFlags, OpenOptionsExt}, io::Errno, }; +use std::os::fd::RawFd; use std::{ cell::{RefCell, UnsafeCell}, mem::MaybeUninit, @@ -40,11 +40,6 @@ impl OwnedCallbacks { self.as_mut().inline_count == 0 } - fn get(&self, fd: usize) -> Option<&CompletionCallback> { - let callbacks = unsafe { &mut *self.0.get() }; - callbacks.get(fd) - } - fn remove(&self, fd: usize) -> Option { let callbacks = unsafe { &mut *self.0.get() }; callbacks.remove(fd) @@ -135,16 +130,6 @@ impl Callbacks { } } - fn get(&self, fd: usize) -> Option<&CompletionCallback> { - if let Some(pos) = self.find_inline(fd) { - let (_, callback) = unsafe { self.inline_entries[pos].assume_init_ref() }; - return Some(callback); - } else if let Some(pos) = self.heap_entries.iter().position(|&(k, _)| k == fd) { - return Some(&self.heap_entries[pos].1); - } - None - } - fn remove(&mut self, fd: usize) -> Option { if let Some(pos) = self.find_inline(fd) { let (_, callback) = unsafe { self.inline_entries[pos].assume_init_read() }; @@ -213,6 +198,35 @@ impl Clock for UnixIO { } } +fn try_pwritev_raw( + fd: RawFd, + off: u64, + bufs: &[Arc>], + start_idx: usize, + start_off: usize, +) -> std::io::Result { + const MAX_IOV: usize = 1024; + let iov_len = std::cmp::min(bufs.len() - start_idx, MAX_IOV); + let mut iov = Vec::with_capacity(iov_len); + + for (i, b) in bufs.iter().enumerate().skip(start_idx).take(iov_len) { + let r = b.borrow(); // borrow just to get pointer/len + let s = r.as_slice(); + let s = if i == start_idx { &s[start_off..] } else { s }; + iov.push(libc::iovec { + iov_base: s.as_ptr() as *mut _, + iov_len: s.len(), + }); + } + + let n = unsafe { libc::pwritev(fd, iov.as_ptr(), iov.len() as i32, off as i64) }; + if n < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(n as usize) + } +} + impl IO for UnixIO { fn open_file(&self, path: &str, flags: OpenFlags, _direct: bool) -> Result> { trace!("open_file(path = {})", path); @@ -243,46 +257,129 @@ impl IO for UnixIO { if self.callbacks.is_empty() { return Ok(()); } + self.events.clear(); trace!("run_once() waits for events"); self.poller.wait(self.events.as_mut(), None)?; for event in self.events.iter() { - if let Some(cf) = self.callbacks.get(event.key) { - let result = match cf { - CompletionCallback::Read(ref file, ref c, pos) => { - let file = file.lock().unwrap(); - let r = c.as_read(); - let mut buf = r.buf_mut(); - rustix::io::pread(file.as_fd(), buf.as_mut_slice(), *pos as u64) - } - CompletionCallback::Write(ref file, _, ref buf, pos) => { - let file = file.lock().unwrap(); - let buf = buf.borrow(); - rustix::io::pwrite(file.as_fd(), buf.as_slice(), *pos as u64) - } - }; - match result { - Ok(n) => { - let cf = self - .callbacks - .remove(event.key) - .expect("callback should exist"); - match cf { - CompletionCallback::Read(_, c, _) => c.complete(0), - CompletionCallback::Write(_, c, _, _) => c.complete(n as i32), - } - } - Err(Errno::AGAIN) => (), - Err(e) => { - self.callbacks.remove(event.key); + let key = event.key; + let cb = match self.callbacks.remove(key) { + Some(cb) => cb, + None => continue, // could have been completed/removed already + }; - trace!("run_once() error: {}", e); - return Err(e.into()); + match cb { + CompletionCallback::Read(ref file, c, pos) => { + let f = file + .lock() + .map_err(|e| LimboError::LockingError(e.to_string()))?; + let r = c.as_read(); + let mut buf = r.buf_mut(); + match rustix::io::pread(f.as_fd(), buf.as_mut_slice(), pos as u64) { + Ok(n) => c.complete(n as i32), + Err(Errno::AGAIN) => { + // re-arm + unsafe { self.poller.as_mut().add(&f.as_fd(), Event::readable(key))? }; + self.callbacks.as_mut().insert( + key, + CompletionCallback::Read(file.clone(), c.clone(), pos), + ); + } + Err(e) => return Err(e.into()), + } + } + + CompletionCallback::Write(ref file, c, buf, pos) => { + let f = file + .lock() + .map_err(|e| LimboError::LockingError(e.to_string()))?; + let b = buf.borrow(); + match rustix::io::pwrite(f.as_fd(), b.as_slice(), pos as u64) { + Ok(n) => c.complete(n as i32), + Err(Errno::AGAIN) => { + unsafe { self.poller.as_mut().add(&f.as_fd(), Event::writable(key))? }; + self.callbacks.as_mut().insert( + key, + CompletionCallback::Write(file.clone(), c, buf.clone(), pos), + ); + } + Err(e) => return Err(e.into()), + } + } + + CompletionCallback::Writev(file, c, bufs, mut pos, mut idx, mut off) => { + let f = file + .lock() + .map_err(|e| LimboError::LockingError(e.to_string()))?; + // keep trying until WouldBlock or we're done with this event + match try_pwritev_raw(f.as_raw_fd(), pos as u64, &bufs, idx, off) { + Ok(written) => { + // advance through buffers + let mut rem = written; + while rem > 0 { + let len = { + let r = bufs[idx].borrow(); + r.len() + }; + let left = len - off; + if rem < left { + off += rem; + rem = 0; + } else { + rem -= left; + idx += 1; + off = 0; + if idx == bufs.len() { + break; + } + } + } + pos += written; + + if idx == bufs.len() { + c.complete(pos as i32); + } else { + // Not finished; re-arm and store updated state + unsafe { + self.poller.as_mut().add(&f.as_fd(), Event::writable(key))? + }; + self.callbacks.as_mut().insert( + key, + CompletionCallback::Writev( + file.clone(), + c.clone(), + bufs, + pos, + idx, + off, + ), + ); + } + break; + } + Err(e) if e.kind() == ErrorKind::WouldBlock => { + // re-arm with same state + unsafe { self.poller.as_mut().add(&f.as_fd(), Event::writable(key))? }; + self.callbacks.as_mut().insert( + key, + CompletionCallback::Writev( + file.clone(), + c.clone(), + bufs, + pos, + idx, + off, + ), + ); + break; + } + Err(e) => return Err(e.into()), } } } } + Ok(()) } @@ -312,6 +409,14 @@ enum CompletionCallback { Arc>, usize, ), + Writev( + Arc>, + Completion, + Vec>>, + usize, // absolute file offset + usize, // buf index + usize, // intra-buf offset + ), } pub struct UnixFile<'io> { @@ -431,6 +536,52 @@ impl File for UnixFile<'_> { } } + #[instrument(err, skip_all, level = Level::TRACE)] + fn pwritev( + &self, + pos: usize, + buffers: Vec>>, + c: Completion, + ) -> Result { + let file = self + .file + .lock() + .map_err(|e| LimboError::LockingError(e.to_string()))?; + + match try_pwritev_raw(file.as_raw_fd(), pos as u64, &buffers, 0, 0) { + Ok(written) => { + trace!("pwritev wrote {written}"); + c.complete(written as i32); + Ok(c) + } + Err(e) => { + if e.kind() == ErrorKind::WouldBlock { + trace!("pwritev blocks"); + } else { + return Err(e.into()); + } + // Set up state so we can resume later + let fd = file.as_raw_fd(); + self.poller + .add(&file.as_fd(), Event::writable(fd as usize))?; + let buf_idx = 0; + let buf_offset = 0; + self.callbacks.insert( + fd as usize, + CompletionCallback::Writev( + self.file.clone(), + c.clone(), + buffers, + pos, + buf_idx, + buf_offset, + ), + ); + Ok(c) + } + } + } + #[instrument(err, skip_all, level = Level::TRACE)] fn sync(&self, c: Completion) -> Result { let file = self.file.lock().unwrap(); diff --git a/core/lib.rs b/core/lib.rs index 6176e9811..41fe11f36 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -18,6 +18,7 @@ pub mod result; mod schema; #[cfg(feature = "series")] mod series; +mod state_machine; mod storage; #[allow(dead_code)] #[cfg(feature = "time")] @@ -41,15 +42,12 @@ mod numeric; #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; -use crate::storage::header_accessor::get_schema_cookie; -use crate::storage::sqlite3_ondisk::is_valid_page_size; -use crate::storage::{header_accessor, wal::DummyWAL}; use crate::translate::optimizer::optimize_plan; use crate::translate::pragma::TURSO_CDC_DEFAULT_TABLE_NAME; #[cfg(feature = "fs")] use crate::types::WalInsertInfo; #[cfg(feature = "fs")] -use crate::util::{IOExt, OpenMode, OpenOptions}; +use crate::util::{OpenMode, OpenOptions}; use crate::vtab::VirtualTable; use core::str; pub use error::LimboError; @@ -80,6 +78,7 @@ use std::{ use storage::database::DatabaseFile; use storage::page_cache::DumbLruPageCache; use storage::pager::{AtomicDbState, DbState}; +use storage::sqlite3_ondisk::PageSize; pub use storage::{ buffer_pool::BufferPool, database::DatabaseStorage, @@ -93,7 +92,7 @@ use turso_sqlite3_parser::{ast, ast::Cmd, lexer::sql::Parser}; use types::IOResult; pub use types::RefValue; pub use types::Value; -use util::parse_schema_rows; +use util::{parse_schema_rows, IOExt as _}; use vdbe::builder::QueryMode; use vdbe::builder::TableRefIdCounter; @@ -121,7 +120,7 @@ static DATABASE_MANAGER: LazyLock>>> = /// The `Database` object contains per database file state that is shared /// between multiple connections. pub struct Database { - mv_store: Option>, + mv_store: Option>, schema: Mutex>, db_file: Arc, path: String, @@ -269,7 +268,7 @@ impl Database { let maybe_shared_wal = WalFileShared::open_shared_if_exists(&io, wal_path.as_str())?; let mv_store = if enable_mvcc { - Some(Rc::new(MvStore::new( + Some(Arc::new(MvStore::new( mvcc::LocalClock::new(), mvcc::persistent_storage::Storage::new_noop(), ))) @@ -333,10 +332,17 @@ impl Database { pub fn connect(self: &Arc) -> Result> { let pager = self.init_pager(None)?; - let page_size = header_accessor::get_page_size(&pager) - .unwrap_or(storage::sqlite3_ondisk::DEFAULT_PAGE_SIZE); - let default_cache_size = header_accessor::get_default_page_cache_size(&pager) - .unwrap_or(storage::sqlite3_ondisk::DEFAULT_CACHE_SIZE); + let page_size = pager + .io + .block(|| pager.with_header(|header| header.page_size)) + .unwrap_or_default() + .get(); + + let default_cache_size = pager + .io + .block(|| pager.with_header(|header| header.default_page_cache_size)) + .unwrap_or_default() + .get(); let conn = Arc::new(Connection { _db: self.clone(), @@ -391,7 +397,7 @@ impl Database { ))); let pager = Pager::new( self.db_file.clone(), - wal, + Some(wal), self.io.clone(), Arc::new(RwLock::new(DumbLruPageCache::default())), buffer_pool.clone(), @@ -403,12 +409,10 @@ impl Database { let buffer_pool = Arc::new(BufferPool::new(page_size)); // No existing WAL; create one. - // TODO: currently Pager needs to be instantiated with some implementation of trait Wal, so here's a workaround. - let dummy_wal = Rc::new(RefCell::new(DummyWAL {})); let db_state = self.db_state.clone(); let mut pager = Pager::new( self.db_file.clone(), - dummy_wal, + None, self.io.clone(), Arc::new(RwLock::new(DumbLruPageCache::default())), buffer_pool.clone(), @@ -419,8 +423,11 @@ impl Database { let size = match page_size { Some(size) => size as u32, None => { - let size = header_accessor::get_page_size(&pager) - .unwrap_or(storage::sqlite3_ondisk::DEFAULT_PAGE_SIZE); + let size = pager + .io + .block(|| pager.with_header(|header| header.page_size)) + .unwrap_or_default() + .get(); buffer_pool.set_page_size(size as usize); size } @@ -522,6 +529,10 @@ impl Database { } Ok(()) } + + pub fn get_mv_store(&self) -> Option<&Arc> { + self.mv_store.as_ref() + } } fn get_schema_version(conn: &Arc) -> Result { @@ -807,10 +818,12 @@ impl Connection { // first, quickly read schema_version from the root page in order to check if schema changed pager.begin_read_tx()?; - let db_schema_version = get_schema_cookie(&pager); + let db_schema_version = pager + .io + .block(|| pager.with_header(|header| header.schema_cookie)); pager.end_read_tx().expect("read txn must be finished"); - let db_schema_version = db_schema_version?; + let db_schema_version = db_schema_version?.get(); let conn_schema_version = self.schema.borrow().schema_version; turso_assert!( conn_schema_version <= db_schema_version, @@ -838,7 +851,10 @@ impl Connection { let mut fresh = Schema::new(false); // todo: indices! // read cookie before consuming statement program - otherwise we can end up reading cookie with closed transaction state - let cookie = get_schema_cookie(&pager)?; + let cookie = pager + .io + .block(|| pager.with_header(|header| header.schema_cookie))? + .get(); // TODO: This function below is synchronous, make it async parse_schema_rows(stmt, &mut fresh, &self.syms.borrow(), None)?; @@ -1170,13 +1186,19 @@ impl Connection { { let pager = self.pager.borrow(); + let Some(wal) = pager.wal.as_ref() else { + return Err(LimboError::InternalError( + "wal_insert_end called without a wal".to_string(), + )); + }; + { - let wal = pager.wal.borrow_mut(); + let wal = wal.borrow_mut(); wal.end_write_tx(); wal.end_read_tx(); } // remove all non-commited changes in case if WAL session left some suffix without commit frame - pager.rollback(false, self)?; + pager.rollback(false, self, true)?; } // let's re-parse schema from scratch if schema cookie changed compared to the our in-memory view of schema @@ -1315,7 +1337,7 @@ impl Connection { /// is first created, if it does not already exist when the page_size pragma is issued, /// or at the next VACUUM command that is run on the same database connection while not in WAL mode. pub fn reset_page_size(&self, size: u32) -> Result<()> { - if !is_valid_page_size(size) { + if PageSize::new(size).is_none() { return Ok(()); } @@ -1683,19 +1705,23 @@ impl Connection { databases.sort_by_key(|&(seq, _, _)| seq); databases } + + pub fn get_pager(&self) -> Rc { + self.pager.borrow().clone() + } } pub struct Statement { program: Rc, state: vdbe::ProgramState, - mv_store: Option>, + mv_store: Option>, pager: Rc, } impl Statement { pub fn new( program: Rc, - mv_store: Option>, + mv_store: Option>, pager: Rc, ) -> Self { let state = vdbe::ProgramState::new(program.max_registers, program.cursor_ref.len()); diff --git a/core/mvcc/cursor.rs b/core/mvcc/cursor.rs index db0d621a7..b6965a4c4 100644 --- a/core/mvcc/cursor.rs +++ b/core/mvcc/cursor.rs @@ -1,7 +1,10 @@ use crate::mvcc::clock::LogicalClock; -use crate::mvcc::database::{MvStore, Result, Row, RowID}; +use crate::mvcc::database::{MvStore, Row, RowID}; +use crate::Pager; +use crate::Result; use std::fmt::Debug; use std::rc::Rc; +use std::sync::Arc; #[derive(Debug, Copy, Clone)] enum CursorPosition { @@ -14,20 +17,27 @@ enum CursorPosition { } #[derive(Debug)] pub struct MvccLazyCursor { - pub db: Rc>, + pub db: Arc>, current_pos: CursorPosition, table_id: u64, tx_id: u64, } impl MvccLazyCursor { - pub fn new(db: Rc>, tx_id: u64, table_id: u64) -> Result> { - Ok(Self { + pub fn new( + db: Arc>, + tx_id: u64, + table_id: u64, + pager: Rc, + ) -> Result> { + db.maybe_initialize_table(table_id, pager)?; + let cursor = Self { db, tx_id, current_pos: CursorPosition::BeforeFirst, table_id, - }) + }; + Ok(cursor) } /// Insert a row into the table. @@ -40,18 +50,37 @@ impl MvccLazyCursor { Ok(()) } - pub fn current_row_id(&self) -> Option { + pub fn current_row_id(&mut self) -> Option { match self.current_pos { CursorPosition::Loaded(id) => Some(id), - CursorPosition::BeforeFirst => None, + CursorPosition::BeforeFirst => { + // If we are before first, we need to try and find the first row. + let maybe_rowid = self.db.get_next_row_id_for_table(self.table_id, i64::MIN); + if let Some(id) = maybe_rowid { + self.current_pos = CursorPosition::Loaded(id); + Some(id) + } else { + self.current_pos = CursorPosition::BeforeFirst; + None + } + } CursorPosition::End => None, } } - pub fn current_row(&self) -> Result> { + pub fn current_row(&mut self) -> Result> { match self.current_pos { CursorPosition::Loaded(id) => self.db.read(self.tx_id, id), - CursorPosition::BeforeFirst => Ok(None), + CursorPosition::BeforeFirst => { + // If we are before first, we need to try and find the first row. + let maybe_rowid = self.db.get_next_row_id_for_table(self.table_id, i64::MIN); + if let Some(id) = maybe_rowid { + self.current_pos = CursorPosition::Loaded(id); + self.db.read(self.tx_id, id) + } else { + Ok(None) + } + } CursorPosition::End => Ok(None), } } @@ -65,7 +94,8 @@ impl MvccLazyCursor { let before_first = matches!(self.current_pos, CursorPosition::BeforeFirst); let min_id = match self.current_pos { CursorPosition::Loaded(id) => id.row_id + 1, - CursorPosition::BeforeFirst => i64::MIN, // we need to find first row, so we look from the first id + // TODO: do we need to forward twice? + CursorPosition::BeforeFirst => i64::MIN, // we need to find first row, so we look from the first id, CursorPosition::End => { // let's keep same state, we reached the end so no point in moving forward. return false; diff --git a/core/mvcc/database/mod.rs b/core/mvcc/database/mod.rs index b3f71a871..7b9c60aaf 100644 --- a/core/mvcc/database/mod.rs +++ b/core/mvcc/database/mod.rs @@ -1,18 +1,30 @@ use crate::mvcc::clock::LogicalClock; -use crate::mvcc::errors::DatabaseError; use crate::mvcc::persistent_storage::Storage; +use crate::state_machine::StateMachine; +use crate::state_machine::StateTransition; +use crate::state_machine::TransitionResult; +use crate::storage::btree::BTreeCursor; +use crate::storage::btree::BTreeKey; +use crate::types::IOResult; +use crate::types::ImmutableRecord; +use crate::LimboError; +use crate::Result; +use crate::{Connection, Pager}; use crossbeam_skiplist::{SkipMap, SkipSet}; use parking_lot::RwLock; +use std::collections::HashSet; use std::fmt::Debug; +use std::marker::PhantomData; +use std::rc::Rc; use std::sync::atomic::{AtomicU64, Ordering}; - -pub type Result = std::result::Result; +use std::sync::Arc; #[cfg(test)] -mod tests; +pub mod tests; #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct RowID { + /// The table ID. Analogous to table's root page number. pub table_id: u64, pub row_id: i64, } @@ -28,11 +40,16 @@ impl RowID { pub struct Row { pub id: RowID, pub data: Vec, + pub column_count: usize, } impl Row { - pub fn new(id: RowID, data: Vec) -> Self { - Self { id, data } + pub fn new(id: RowID, data: Vec, column_count: usize) -> Self { + Self { + id, + data, + column_count, + } } } @@ -221,6 +238,444 @@ impl AtomicTransactionState { } } +#[derive(Debug)] +pub enum CommitState { + Initial, + BeginPagerTxn { end_ts: u64 }, + WriteRow { end_ts: u64, write_set_index: usize }, + WriteRowStateMachine { end_ts: u64, write_set_index: usize }, + CommitPagerTxn { end_ts: u64 }, + Commit { end_ts: u64 }, +} + +#[derive(Debug)] +pub enum WriteRowState { + Initial, + CreateCursor, + Seek, + Insert, +} + +pub struct CommitStateMachine { + state: CommitState, + is_finalized: bool, + pager: Rc, + tx_id: TxID, + connection: Arc, + write_set: Vec, + write_row_state_machine: Option>, + _phantom: PhantomData, +} + +pub struct WriteRowStateMachine { + state: WriteRowState, + is_finalized: bool, + pager: Rc, + row: Row, + record: Option, + cursor: Option, +} + +impl CommitStateMachine { + fn new(state: CommitState, pager: Rc, tx_id: TxID, connection: Arc) -> Self { + Self { + state, + is_finalized: false, + pager, + tx_id, + connection, + write_set: Vec::new(), + write_row_state_machine: None, + _phantom: PhantomData, + } + } +} + +impl WriteRowStateMachine { + fn new(pager: Rc, row: Row) -> Self { + Self { + state: WriteRowState::Initial, + is_finalized: false, + pager, + row, + record: None, + cursor: None, + } + } +} + +impl StateTransition for CommitStateMachine { + type State = CommitStateMachine; + type Context = MvStore; + type SMResult = (); + + #[tracing::instrument(fields(state = ?self.state), skip(self, mvcc_store))] + fn step(&mut self, mvcc_store: &Self::Context) -> Result> { + match self.state { + CommitState::Initial => { + let end_ts = mvcc_store.get_timestamp(); + // NOTICE: the first shadowed tx keeps the entry alive in the map + // for the duration of this whole function, which is important for correctness! + let tx = mvcc_store + .txs + .get(&self.tx_id) + .ok_or(LimboError::TxTerminated)?; + let tx = tx.value().write(); + match tx.state.load() { + TransactionState::Terminated => { + return Err(LimboError::TxTerminated); + } + _ => { + assert_eq!(tx.state, TransactionState::Active); + } + } + tx.state.store(TransactionState::Preparing); + tracing::trace!("prepare_tx(tx_id={})", self.tx_id); + + /* TODO: The code we have here is sufficient for snapshot isolation. + ** In order to implement serializability, we need the following steps: + ** + ** 1. Validate if all read versions are still visible by inspecting the read_set + ** 2. Validate if there are no phantoms by walking the scans from scan_set (which we don't even have yet) + ** - a phantom is a version that became visible in the middle of our transaction, + ** but wasn't taken into account during one of the scans from the scan_set + ** 3. Wait for commit dependencies, which we don't even track yet... + ** Excerpt from what's a commit dependency and how it's tracked in the original paper: + ** """ + A transaction T1 has a commit dependency on another transaction + T2, if T1 is allowed to commit only if T2 commits. If T2 aborts, + T1 must also abort, so cascading aborts are possible. T1 acquires a + commit dependency either by speculatively reading or speculatively ignoring a version, + instead of waiting for T2 to commit. + We implement commit dependencies by a register-and-report + approach: T1 registers its dependency with T2 and T2 informs T1 + when it has committed or aborted. Each transaction T contains a + counter, CommitDepCounter, that counts how many unresolved + commit dependencies it still has. A transaction cannot commit + until this counter is zero. In addition, T has a Boolean variable + AbortNow that other transactions can set to tell T to abort. Each + transaction T also has a set, CommitDepSet, that stores transaction IDs + of the transactions that depend on T. + To take a commit dependency on a transaction T2, T1 increments + its CommitDepCounter and adds its transaction ID to T2’s CommitDepSet. + When T2 has committed, it locates each transaction in + its CommitDepSet and decrements their CommitDepCounter. If + T2 aborted, it tells the dependent transactions to also abort by + setting their AbortNow flags. If a dependent transaction is not + found, this means that it has already aborted. + Note that a transaction with commit dependencies may not have to + wait at all - the dependencies may have been resolved before it is + ready to commit. Commit dependencies consolidate all waits into + a single wait and postpone the wait to just before commit. + Some transactions may have to wait before commit. + Waiting raises a concern of deadlocks. + However, deadlocks cannot occur because an older transaction never + waits on a younger transaction. In + a wait-for graph the direction of edges would always be from a + younger transaction (higher end timestamp) to an older transaction + (lower end timestamp) so cycles are impossible. + """ + ** If you're wondering when a speculative read happens, here you go: + ** Case 1: speculative read of TB: + """ + If transaction TB is in the Preparing state, it has acquired an end + timestamp TS which will be V’s begin timestamp if TB commits. + A safe approach in this situation would be to have transaction T + wait until transaction TB commits. However, we want to avoid all + blocking during normal processing so instead we continue with + the visibility test and, if the test returns true, allow T to + speculatively read V. Transaction T acquires a commit dependency on + TB, restricting the serialization order of the two transactions. That + is, T is allowed to commit only if TB commits. + """ + ** Case 2: speculative ignore of TE: + """ + If TE’s state is Preparing, it has an end timestamp TS that will become + the end timestamp of V if TE does commit. If TS is greater than the read + time RT, it is obvious that V will be visible if TE commits. If TE + aborts, V will still be visible, because any transaction that updates + V after TE has aborted will obtain an end timestamp greater than + TS. If TS is less than RT, we have a more complicated situation: + if TE commits, V will not be visible to T but if TE aborts, it will + be visible. We could handle this by forcing T to wait until TE + commits or aborts but we want to avoid all blocking during normal processing. + Instead we allow T to speculatively ignore V and + proceed with its processing. Transaction T acquires a commit + dependency (see Section 2.7) on TE, that is, T is allowed to commit + only if TE commits. + """ + */ + tx.state.store(TransactionState::Committed(end_ts)); + tracing::trace!("commit_tx(tx_id={})", self.tx_id); + self.write_set + .extend(tx.write_set.iter().map(|v| *v.value())); + self.state = CommitState::BeginPagerTxn { end_ts }; + Ok(TransitionResult::Continue) + } + CommitState::BeginPagerTxn { end_ts } => { + // FIXME: how do we deal with multiple concurrent writes? + // WAL requires a txn to be written sequentially. Either we: + // 1. Wait for currently writer to finish before second txn starts. + // 2. Choose a txn to write depending on some heuristics like amount of frames will be written. + // 3. .. + // + loop { + match self.pager.begin_write_tx() { + Ok(crate::types::IOResult::Done(result)) => { + if let crate::result::LimboResult::Busy = result { + return Err(LimboError::InternalError( + "Pager write transaction busy".to_string(), + )); + } + break; + } + Ok(crate::types::IOResult::IO) => { + // FIXME: this is a hack to make the pager run the IO loop + self.pager.io.run_once().unwrap(); + continue; + } + Err(e) => { + return Err(LimboError::InternalError(e.to_string())); + } + } + } + self.state = CommitState::WriteRow { + end_ts, + write_set_index: 0, + }; + return Ok(TransitionResult::Continue); + } + CommitState::WriteRow { + end_ts, + write_set_index, + } => { + if write_set_index == self.write_set.len() { + self.state = CommitState::CommitPagerTxn { end_ts }; + return Ok(TransitionResult::Continue); + } + let id = &self.write_set[write_set_index]; + if let Some(row_versions) = mvcc_store.rows.get(id) { + let row_versions = row_versions.value().read(); + // Find rows that were written by this transaction + for row_version in row_versions.iter() { + if let TxTimestampOrID::TxID(row_tx_id) = row_version.begin { + if row_tx_id == self.tx_id { + let state_machine = mvcc_store + .write_row_to_pager(self.pager.clone(), &row_version.row)?; + self.write_row_state_machine = Some(state_machine); + self.state = CommitState::WriteRowStateMachine { + end_ts, + write_set_index, + }; + break; + } + } + if let Some(TxTimestampOrID::Timestamp(row_tx_id)) = row_version.end { + if row_tx_id == self.tx_id { + let state_machine = mvcc_store + .write_row_to_pager(self.pager.clone(), &row_version.row)?; + self.write_row_state_machine = Some(state_machine); + self.state = CommitState::WriteRowStateMachine { + end_ts, + write_set_index, + }; + break; + } + } + } + } + Ok(TransitionResult::Continue) + } + CommitState::WriteRowStateMachine { + end_ts, + write_set_index, + } => { + let write_row_state_machine = self.write_row_state_machine.as_mut().unwrap(); + match write_row_state_machine.step(&())? { + TransitionResult::Io => return Ok(TransitionResult::Io), + TransitionResult::Continue => { + return Ok(TransitionResult::Continue); + } + TransitionResult::Done(_) => { + self.state = CommitState::WriteRow { + end_ts, + write_set_index: write_set_index + 1, + }; + return Ok(TransitionResult::Continue); + } + } + } + CommitState::CommitPagerTxn { end_ts } => { + // Write committed data to pager for persistence + // Flush dirty pages to WAL - this is critical for data persistence + // Similar to what step_end_write_txn does for legacy transactions + loop { + let result = self + .pager + .end_tx( + false, // rollback = false since we're committing + false, // schema_did_change = false for now (could be improved) + &self.connection, + self.connection.wal_checkpoint_disabled.get(), + ) + .map_err(|e| LimboError::InternalError(e.to_string())) + .unwrap(); + if let crate::types::IOResult::Done(_) = result { + break; + } + } + self.state = CommitState::Commit { end_ts }; + Ok(TransitionResult::Continue) + } + CommitState::Commit { end_ts } => { + let mut log_record = LogRecord::new(end_ts); + for id in &self.write_set { + if let Some(row_versions) = mvcc_store.rows.get(id) { + let mut row_versions = row_versions.value().write(); + for row_version in row_versions.iter_mut() { + if let TxTimestampOrID::TxID(id) = row_version.begin { + if id == self.tx_id { + // New version is valid STARTING FROM committing transaction's end timestamp + // See diagram on page 299: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf + row_version.begin = TxTimestampOrID::Timestamp(end_ts); + mvcc_store.insert_version_raw( + &mut log_record.row_versions, + row_version.clone(), + ); // FIXME: optimize cloning out + } + } + if let Some(TxTimestampOrID::TxID(id)) = row_version.end { + if id == self.tx_id { + // Old version is valid UNTIL committing transaction's end timestamp + // See diagram on page 299: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf + row_version.end = Some(TxTimestampOrID::Timestamp(end_ts)); + mvcc_store.insert_version_raw( + &mut log_record.row_versions, + row_version.clone(), + ); // FIXME: optimize cloning out + } + } + } + } + } + tracing::trace!("updated(tx_id={})", self.tx_id); + + // We have now updated all the versions with a reference to the + // transaction ID to a timestamp and can, therefore, remove the + // transaction. Please note that when we move to lockless, the + // invariant doesn't necessarily hold anymore because another thread + // might have speculatively read a version that we want to remove. + // But that's a problem for another day. + // FIXME: it actually just become a problem for today!!! + // TODO: test that reproduces this failure, and then a fix + mvcc_store.txs.remove(&self.tx_id); + if !log_record.row_versions.is_empty() { + mvcc_store.storage.log_tx(log_record)?; + } + tracing::trace!("logged(tx_id={})", self.tx_id); + self.finalize(mvcc_store)?; + Ok(TransitionResult::Done(())) + } + } + } + + fn finalize(&mut self, _context: &Self::Context) -> Result<()> { + self.is_finalized = true; + Ok(()) + } + + fn is_finalized(&self) -> bool { + self.is_finalized + } +} + +impl StateTransition for WriteRowStateMachine { + type State = WriteRowStateMachine; + type Context = (); + type SMResult = (); + + #[tracing::instrument(fields(state = ?self.state), skip(self, _context))] + fn step(&mut self, _context: &Self::Context) -> Result> { + use crate::storage::btree::BTreeCursor; + use crate::types::{IOResult, SeekKey, SeekOp}; + + match self.state { + WriteRowState::Initial => { + // Create the record and key + let mut record = ImmutableRecord::new(self.row.data.len()); + record.start_serialization(&self.row.data); + self.record = Some(record); + + self.state = WriteRowState::CreateCursor; + Ok(TransitionResult::Continue) + } + WriteRowState::CreateCursor => { + // Create the cursor + let root_page = self.row.id.table_id as usize; + let num_columns = self.row.column_count; + + let cursor = BTreeCursor::new_table( + None, // Write directly to B-tree + self.pager.clone(), + root_page, + num_columns, + ); + self.cursor = Some(cursor); + + self.state = WriteRowState::Seek; + Ok(TransitionResult::Continue) + } + WriteRowState::Seek => { + // Position the cursor by seeking to the row position + let seek_key = SeekKey::TableRowId(self.row.id.row_id); + let cursor = self.cursor.as_mut().unwrap(); + + match cursor.seek(seek_key, SeekOp::GE { eq_only: true })? { + IOResult::Done(_) => { + self.state = WriteRowState::Insert; + Ok(TransitionResult::Continue) + } + IOResult::IO => { + return Ok(TransitionResult::Io); + } + } + } + WriteRowState::Insert => { + // Insert the record into the B-tree + let cursor = self.cursor.as_mut().unwrap(); + let key = BTreeKey::new_table_rowid(self.row.id.row_id, self.record.as_ref()); + + match cursor + .insert(&key, true) + .map_err(|e| LimboError::InternalError(e.to_string()))? + { + IOResult::Done(()) => { + tracing::trace!( + "write_row_to_pager(table_id={}, row_id={})", + self.row.id.table_id, + self.row.id.row_id + ); + self.finalize(&())?; + Ok(TransitionResult::Done(())) + } + IOResult::IO => { + return Ok(TransitionResult::Io); + } + } + } + } + } + + fn finalize(&mut self, _context: &Self::Context) -> Result<()> { + self.is_finalized = true; + Ok(()) + } + + fn is_finalized(&self) -> bool { + self.is_finalized + } +} + /// A multi-version concurrency control database. #[derive(Debug)] pub struct MvStore { @@ -230,6 +685,7 @@ pub struct MvStore { next_rowid: AtomicU64, clock: Clock, storage: Storage, + loaded_tables: RwLock>, } impl MvStore { @@ -242,6 +698,7 @@ impl MvStore { next_rowid: AtomicU64::new(0), // TODO: determine this from B-Tree clock, storage, + loaded_tables: RwLock::new(HashSet::new()), } } @@ -264,7 +721,7 @@ impl MvStore { let tx = self .txs .get(&tx_id) - .ok_or(DatabaseError::NoSuchTransactionID(tx_id))?; + .ok_or(LimboError::NoSuchTransactionID(tx_id.to_string()))?; let mut tx = tx.value().write(); assert_eq!(tx.state, TransactionState::Active); let id = row.id; @@ -297,9 +754,9 @@ impl MvStore { /// # Returns /// /// Returns `true` if the row was successfully updated, and `false` otherwise. - pub fn update(&self, tx_id: TxID, row: Row) -> Result { + pub fn update(&self, tx_id: TxID, row: Row, pager: Rc) -> Result { tracing::trace!("update(tx_id={}, row.id={:?})", tx_id, row.id); - if !self.delete(tx_id, row.id)? { + if !self.delete(tx_id, row.id, pager)? { return Ok(false); } self.insert(tx_id, row)?; @@ -308,9 +765,9 @@ impl MvStore { /// Inserts a row in the database with new values, previously deleting /// any old data if it existed. Bails on a delete error, e.g. write-write conflict. - pub fn upsert(&self, tx_id: TxID, row: Row) -> Result<()> { + pub fn upsert(&self, tx_id: TxID, row: Row, pager: Rc) -> Result<()> { tracing::trace!("upsert(tx_id={}, row.id={:?})", tx_id, row.id); - self.delete(tx_id, row.id)?; + self.delete(tx_id, row.id, pager)?; self.insert(tx_id, row) } @@ -328,7 +785,7 @@ impl MvStore { /// /// Returns `true` if the row was successfully deleted, and `false` otherwise. /// - pub fn delete(&self, tx_id: TxID, id: RowID) -> Result { + pub fn delete(&self, tx_id: TxID, id: RowID, pager: Rc) -> Result { tracing::trace!("delete(tx_id={}, id={:?})", tx_id, id); let row_versions_opt = self.rows.get(&id); if let Some(ref row_versions) = row_versions_opt { @@ -337,7 +794,7 @@ impl MvStore { let tx = self .txs .get(&tx_id) - .ok_or(DatabaseError::NoSuchTransactionID(tx_id))?; + .ok_or(LimboError::NoSuchTransactionID(tx_id.to_string()))?; let tx = tx.value().read(); assert_eq!(tx.state, TransactionState::Active); // A transaction cannot delete a version that it cannot see, @@ -349,8 +806,8 @@ impl MvStore { drop(row_versions); drop(row_versions_opt); drop(tx); - self.rollback_tx(tx_id); - return Err(DatabaseError::WriteWriteConflict); + self.rollback_tx(tx_id, pager); + return Err(LimboError::WriteWriteConflict); } rv.end = Some(TxTimestampOrID::TxID(tx.tx_id)); @@ -360,7 +817,7 @@ impl MvStore { let tx = self .txs .get(&tx_id) - .ok_or(DatabaseError::NoSuchTransactionID(tx_id))?; + .ok_or(LimboError::NoSuchTransactionID(tx_id.to_string()))?; let mut tx = tx.value().write(); tx.insert_to_write_set(id); return Ok(true); @@ -409,24 +866,6 @@ impl MvStore { Ok(keys.collect()) } - /// Gets all row ids in the database for a given table. - pub fn scan_row_ids_for_table(&self, table_id: u64) -> Result> { - tracing::trace!("scan_row_ids_for_table(table_id={})", table_id); - Ok(self - .rows - .range( - RowID { - table_id, - row_id: 0, - }..RowID { - table_id, - row_id: i64::MAX, - }, - ) - .map(|entry| *entry.key()) - .collect()) - } - pub fn get_row_id_range( &self, table_id: u64, @@ -484,12 +923,16 @@ impl MvStore { /// This function starts a new transaction in the database and returns a `TxID` value /// that you can use to perform operations within the transaction. All changes made within the /// transaction are isolated from other transactions until you commit the transaction. - pub fn begin_tx(&self) -> TxID { + pub fn begin_tx(&self, pager: Rc) -> TxID { let tx_id = self.get_tx_id(); let begin_ts = self.get_timestamp(); let tx = Transaction::new(tx_id, begin_ts); tracing::trace!("begin_tx(tx_id={})", tx_id); self.txs.insert(tx_id, RwLock::new(tx)); + + // TODO: we need to tie a pager's read transaction to a transaction ID, so that future refactors to read + // pages from WAL/DB read from a consistent state to maintiain snapshot isolation. + pager.begin_read_tx().unwrap(); tx_id } @@ -502,145 +945,18 @@ impl MvStore { /// # Arguments /// /// * `tx_id` - The ID of the transaction to commit. - pub fn commit_tx(&self, tx_id: TxID) -> Result<()> { - let end_ts = self.get_timestamp(); - // NOTICE: the first shadowed tx keeps the entry alive in the map - // for the duration of this whole function, which is important for correctness! - let tx = self.txs.get(&tx_id).ok_or(DatabaseError::TxTerminated)?; - let tx = tx.value().write(); - match tx.state.load() { - TransactionState::Terminated => return Err(DatabaseError::TxTerminated), - _ => { - assert_eq!(tx.state, TransactionState::Active); - } - } - tx.state.store(TransactionState::Preparing); - tracing::trace!("prepare_tx(tx_id={})", tx_id); - - /* TODO: The code we have here is sufficient for snapshot isolation. - ** In order to implement serializability, we need the following steps: - ** - ** 1. Validate if all read versions are still visible by inspecting the read_set - ** 2. Validate if there are no phantoms by walking the scans from scan_set (which we don't even have yet) - ** - a phantom is a version that became visible in the middle of our transaction, - ** but wasn't taken into account during one of the scans from the scan_set - ** 3. Wait for commit dependencies, which we don't even track yet... - ** Excerpt from what's a commit dependency and how it's tracked in the original paper: - ** """ - A transaction T1 has a commit dependency on another transaction - T2, if T1 is allowed to commit only if T2 commits. If T2 aborts, - T1 must also abort, so cascading aborts are possible. T1 acquires a - commit dependency either by speculatively reading or speculatively ignoring a version, - instead of waiting for T2 to commit. - We implement commit dependencies by a register-and-report - approach: T1 registers its dependency with T2 and T2 informs T1 - when it has committed or aborted. Each transaction T contains a - counter, CommitDepCounter, that counts how many unresolved - commit dependencies it still has. A transaction cannot commit - until this counter is zero. In addition, T has a Boolean variable - AbortNow that other transactions can set to tell T to abort. Each - transaction T also has a set, CommitDepSet, that stores transaction IDs - of the transactions that depend on T. - To take a commit dependency on a transaction T2, T1 increments - its CommitDepCounter and adds its transaction ID to T2’s CommitDepSet. - When T2 has committed, it locates each transaction in - its CommitDepSet and decrements their CommitDepCounter. If - T2 aborted, it tells the dependent transactions to also abort by - setting their AbortNow flags. If a dependent transaction is not - found, this means that it has already aborted. - Note that a transaction with commit dependencies may not have to - wait at all - the dependencies may have been resolved before it is - ready to commit. Commit dependencies consolidate all waits into - a single wait and postpone the wait to just before commit. - Some transactions may have to wait before commit. - Waiting raises a concern of deadlocks. - However, deadlocks cannot occur because an older transaction never - waits on a younger transaction. In - a wait-for graph the direction of edges would always be from a - younger transaction (higher end timestamp) to an older transaction - (lower end timestamp) so cycles are impossible. - """ - ** If you're wondering when a speculative read happens, here you go: - ** Case 1: speculative read of TB: - """ - If transaction TB is in the Preparing state, it has acquired an end - timestamp TS which will be V’s begin timestamp if TB commits. - A safe approach in this situation would be to have transaction T - wait until transaction TB commits. However, we want to avoid all - blocking during normal processing so instead we continue with - the visibility test and, if the test returns true, allow T to - speculatively read V. Transaction T acquires a commit dependency on - TB, restricting the serialization order of the two transactions. That - is, T is allowed to commit only if TB commits. - """ - ** Case 2: speculative ignore of TE: - """ - If TE’s state is Preparing, it has an end timestamp TS that will become - the end timestamp of V if TE does commit. If TS is greater than the read - time RT, it is obvious that V will be visible if TE commits. If TE - aborts, V will still be visible, because any transaction that updates - V after TE has aborted will obtain an end timestamp greater than - TS. If TS is less than RT, we have a more complicated situation: - if TE commits, V will not be visible to T but if TE aborts, it will - be visible. We could handle this by forcing T to wait until TE - commits or aborts but we want to avoid all blocking during normal processing. - Instead we allow T to speculatively ignore V and - proceed with its processing. Transaction T acquires a commit - dependency (see Section 2.7) on TE, that is, T is allowed to commit - only if TE commits. - """ - */ - tx.state.store(TransactionState::Committed(end_ts)); - tracing::trace!("commit_tx(tx_id={})", tx_id); - let write_set: Vec = tx.write_set.iter().map(|v| *v.value()).collect(); - drop(tx); - // Postprocessing: inserting row versions and logging the transaction to persistent storage. - // TODO: we should probably save to persistent storage first, and only then update the in-memory structures. - let mut log_record = LogRecord::new(end_ts); - for ref id in write_set { - if let Some(row_versions) = self.rows.get(id) { - let mut row_versions = row_versions.value().write(); - for row_version in row_versions.iter_mut() { - if let TxTimestampOrID::TxID(id) = row_version.begin { - if id == tx_id { - // New version is valid STARTING FROM committing transaction's end timestamp - // See diagram on page 299: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf - row_version.begin = TxTimestampOrID::Timestamp(end_ts); - self.insert_version_raw( - &mut log_record.row_versions, - row_version.clone(), - ); // FIXME: optimize cloning out - } - } - if let Some(TxTimestampOrID::TxID(id)) = row_version.end { - if id == tx_id { - // Old version is valid UNTIL committing transaction's end timestamp - // See diagram on page 299: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf - row_version.end = Some(TxTimestampOrID::Timestamp(end_ts)); - self.insert_version_raw( - &mut log_record.row_versions, - row_version.clone(), - ); // FIXME: optimize cloning out - } - } - } - } - } - tracing::trace!("updated(tx_id={})", tx_id); - // We have now updated all the versions with a reference to the - // transaction ID to a timestamp and can, therefore, remove the - // transaction. Please note that when we move to lockless, the - // invariant doesn't necessarily hold anymore because another thread - // might have speculatively read a version that we want to remove. - // But that's a problem for another day. - // FIXME: it actually just become a problem for today!!! - // TODO: test that reproduces this failure, and then a fix - self.txs.remove(&tx_id); - if !log_record.row_versions.is_empty() { - self.storage.log_tx(log_record)?; - } - tracing::trace!("logged(tx_id={})", tx_id); - Ok(()) + pub fn commit_tx( + &self, + tx_id: TxID, + pager: Rc, + connection: &Arc, + ) -> Result>> { + let state_machine: StateMachine> = StateMachine::< + CommitStateMachine, + >::new( + CommitStateMachine::new(CommitState::Initial, pager, tx_id, connection.clone()), + ); + Ok(state_machine) } /// Rolls back a transaction with the specified ID. @@ -651,7 +967,7 @@ impl MvStore { /// # Arguments /// /// * `tx_id` - The ID of the transaction to abort. - pub fn rollback_tx(&self, tx_id: TxID) { + pub fn rollback_tx(&self, tx_id: TxID, pager: Rc) { let tx_unlocked = self.txs.get(&tx_id).unwrap(); let tx = tx_unlocked.value().write(); assert_eq!(tx.state, TransactionState::Active); @@ -673,6 +989,7 @@ impl MvStore { let tx = tx_unlocked.value().read(); tx.state.store(TransactionState::Terminated); tracing::trace!("terminate(tx_id={})", tx_id); + pager.end_read_tx().unwrap(); // FIXME: verify that we can already remove the transaction here! // Maybe it's fine for snapshot isolation, but too early for serializable? self.txs.remove(&tx_id); @@ -776,7 +1093,7 @@ impl MvStore { /// Inserts a new row version into the internal data structure for versions, /// while making sure that the row version is inserted in the correct order. - fn insert_version_raw(&self, versions: &mut Vec, row_version: RowVersion) { + pub fn insert_version_raw(&self, versions: &mut Vec, row_version: RowVersion) { // NOTICE: this is an insert a'la insertion sort, with pessimistic linear complexity. // However, we expect the number of versions to be nearly sorted, so we deem it worthy // to search linearly for the insertion point instead of paying the price of using @@ -798,6 +1115,115 @@ impl MvStore { } versions.insert(position, row_version); } + + pub fn write_row_to_pager( + &self, + pager: Rc, + row: &Row, + ) -> Result> { + let state_machine: StateMachine = + StateMachine::::new(WriteRowStateMachine::new( + pager, + row.clone(), + )); + + Ok(state_machine) + } + + /// Try to scan for row ids in the table. + /// + /// This function loads all row ids of a table if the rowids of table were not populated yet. + /// TODO: This is quite expensive so we should try and load rowids in a lazy way. + /// + /// # Arguments + /// + pub fn maybe_initialize_table(&self, table_id: u64, pager: Rc) -> Result<()> { + tracing::trace!("scan_row_ids_for_table(table_id={})", table_id); + + // First, check if the table is already loaded. + if self.loaded_tables.read().contains(&table_id) { + return Ok(()); + } + + // Then, scan the disk B-tree to find existing rows + self.scan_load_table(table_id, pager)?; + + self.loaded_tables.write().insert(table_id); + + Ok(()) + } + + /// Scans the table and inserts the rows into the database. + /// + /// This is initialization step for a table, where we still don't have any rows so we need to insert them if there are. + fn scan_load_table(&self, table_id: u64, pager: Rc) -> Result<()> { + let root_page = table_id as usize; + let mut cursor = BTreeCursor::new_table( + None, // No MVCC cursor for scanning + pager.clone(), + root_page, + 1, // We'll adjust this as needed + ); + loop { + match cursor + .rewind() + .map_err(|e| LimboError::InternalError(e.to_string()))? + { + IOResult::Done(()) => break, + IOResult::IO => { + pager.io.run_once().unwrap(); + continue; + } + } + } + loop { + let rowid_result = cursor + .rowid() + .map_err(|e| LimboError::InternalError(e.to_string()))?; + let row_id = match rowid_result { + IOResult::Done(Some(row_id)) => row_id, + IOResult::Done(None) => break, + IOResult::IO => { + pager.io.run_once().unwrap(); + continue; + } + }; + match cursor + .record() + .map_err(|e| LimboError::InternalError(e.to_string()))? + { + IOResult::Done(Some(record)) => { + let id = RowID { table_id, row_id }; + let column_count = record.column_count(); + // We insert row with 0 timestamp, because it's the only version we have on initialization. + self.insert_version( + id, + RowVersion { + begin: TxTimestampOrID::Timestamp(0), + end: None, + row: Row::new(id, record.get_payload().to_vec(), column_count), + }, + ); + } + IOResult::Done(None) => break, + IOResult::IO => unreachable!(), // FIXME: lazy me not wanting to do state machine right now + } + + // Move to next record + match cursor + .next() + .map_err(|e| LimboError::InternalError(e.to_string()))? + { + IOResult::Done(has_next) => { + if !has_next { + break; + } + } + IOResult::IO => unreachable!(), // FIXME: lazy me not wanting to do state machine right now + } + } + Ok(()) + } } /// A write-write conflict happens when transaction T_current attempts to update a diff --git a/core/mvcc/database/tests.rs b/core/mvcc/database/tests.rs index 6ae69456f..814ebe61f 100644 --- a/core/mvcc/database/tests.rs +++ b/core/mvcc/database/tests.rs @@ -1,26 +1,59 @@ use super::*; use crate::mvcc::clock::LocalClock; -fn test_db() -> MvStore { - let clock = LocalClock::new(); - let storage = crate::mvcc::persistent_storage::Storage::new_noop(); - MvStore::new(clock, storage) +pub(crate) struct MvccTestDbNoConn { + pub(crate) db: Arc, +} +pub(crate) struct MvccTestDb { + pub(crate) mvcc_store: Arc>, + + pub(crate) _db: Arc, + pub(crate) conn: Arc, +} + +impl MvccTestDb { + pub fn new() -> Self { + let io = Arc::new(MemoryIO::new()); + let db = Database::open_file(io.clone(), ":memory:", true, true).unwrap(); + let conn = db.connect().unwrap(); + let mvcc_store = db.mv_store.as_ref().unwrap().clone(); + Self { + mvcc_store, + _db: db, + conn, + } + } +} + +impl MvccTestDbNoConn { + pub fn new() -> Self { + let io = Arc::new(MemoryIO::new()); + let db = Database::open_file(io.clone(), ":memory:", true, true).unwrap(); + Self { db } + } +} + +pub(crate) fn generate_simple_string_row(table_id: u64, id: i64, data: &str) -> Row { + let record = ImmutableRecord::from_values(&[Value::Text(Text::new(data))], 1); + Row { + id: RowID { + table_id, + row_id: id, + }, + column_count: 1, + data: record.as_blob().to_vec(), + } } #[test] fn test_insert_read() { - let db = test_db(); + let db = MvccTestDb::new(); - let tx1 = db.begin_tx(); - let tx1_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx1, tx1_row.clone()).unwrap(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1_row = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -31,10 +64,11 @@ fn test_insert_read() { .unwrap() .unwrap(); assert_eq!(tx1_row, row); - db.commit_tx(tx1).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); - let tx2 = db.begin_tx(); + let tx2 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); let row = db + .mvcc_store .read( tx2, RowID { @@ -49,9 +83,9 @@ fn test_insert_read() { #[test] fn test_read_nonexistent() { - let db = test_db(); - let tx = db.begin_tx(); - let row = db.read( + let db = MvccTestDb::new(); + let tx = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let row = db.mvcc_store.read( tx, RowID { table_id: 1, @@ -63,18 +97,13 @@ fn test_read_nonexistent() { #[test] fn test_delete() { - let db = test_db(); + let db = MvccTestDb::new(); - let tx1 = db.begin_tx(); - let tx1_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx1, tx1_row.clone()).unwrap(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1_row = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -85,15 +114,18 @@ fn test_delete() { .unwrap() .unwrap(); assert_eq!(tx1_row, row); - db.delete( - tx1, - RowID { - table_id: 1, - row_id: 1, - }, - ) - .unwrap(); + db.mvcc_store + .delete( + tx1, + RowID { + table_id: 1, + row_id: 1, + }, + db.conn.pager.borrow().clone(), + ) + .unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -103,10 +135,11 @@ fn test_delete() { ) .unwrap(); assert!(row.is_none()); - db.commit_tx(tx1).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); - let tx2 = db.begin_tx(); + let tx2 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); let row = db + .mvcc_store .read( tx2, RowID { @@ -120,32 +153,29 @@ fn test_delete() { #[test] fn test_delete_nonexistent() { - let db = test_db(); - let tx = db.begin_tx(); + let db = MvccTestDb::new(); + let tx = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); assert!(!db + .mvcc_store .delete( tx, RowID { table_id: 1, row_id: 1 - } + }, + db.conn.pager.borrow().clone(), ) .unwrap()); } #[test] fn test_commit() { - let db = test_db(); - let tx1 = db.begin_tx(); - let tx1_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx1, tx1_row.clone()).unwrap(); + let db = MvccTestDb::new(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1_row = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -156,15 +186,12 @@ fn test_commit() { .unwrap() .unwrap(); assert_eq!(tx1_row, row); - let tx1_updated_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "World".to_string().into_bytes(), - }; - db.update(tx1, tx1_updated_row.clone()).unwrap(); + let tx1_updated_row = generate_simple_string_row(1, 1, "World"); + db.mvcc_store + .update(tx1, tx1_updated_row.clone(), db.conn.pager.borrow().clone()) + .unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -175,10 +202,11 @@ fn test_commit() { .unwrap() .unwrap(); assert_eq!(tx1_updated_row, row); - db.commit_tx(tx1).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); - let tx2 = db.begin_tx(); + let tx2 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); let row = db + .mvcc_store .read( tx2, RowID { @@ -188,24 +216,19 @@ fn test_commit() { ) .unwrap() .unwrap(); - db.commit_tx(tx2).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx2).unwrap(); assert_eq!(tx1_updated_row, row); - db.drop_unused_row_versions(); + db.mvcc_store.drop_unused_row_versions(); } #[test] fn test_rollback() { - let db = test_db(); - let tx1 = db.begin_tx(); - let row1 = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx1, row1.clone()).unwrap(); + let db = MvccTestDb::new(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let row1 = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx1, row1.clone()).unwrap(); let row2 = db + .mvcc_store .read( tx1, RowID { @@ -216,15 +239,12 @@ fn test_rollback() { .unwrap() .unwrap(); assert_eq!(row1, row2); - let row3 = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "World".to_string().into_bytes(), - }; - db.update(tx1, row3.clone()).unwrap(); + let row3 = generate_simple_string_row(1, 1, "World"); + db.mvcc_store + .update(tx1, row3.clone(), db.conn.pager.borrow().clone()) + .unwrap(); let row4 = db + .mvcc_store .read( tx1, RowID { @@ -235,9 +255,11 @@ fn test_rollback() { .unwrap() .unwrap(); assert_eq!(row3, row4); - db.rollback_tx(tx1); - let tx2 = db.begin_tx(); + db.mvcc_store + .rollback_tx(tx1, db.conn.pager.borrow().clone()); + let tx2 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); let row5 = db + .mvcc_store .read( tx2, RowID { @@ -251,19 +273,14 @@ fn test_rollback() { #[test] fn test_dirty_write() { - let db = test_db(); + let db = MvccTestDb::new(); // T1 inserts a row with ID 1, but does not commit. - let tx1 = db.begin_tx(); - let tx1_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx1, tx1_row.clone()).unwrap(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1_row = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -275,18 +292,17 @@ fn test_dirty_write() { .unwrap(); assert_eq!(tx1_row, row); + let conn2 = db._db.connect().unwrap(); // T2 attempts to delete row with ID 1, but fails because T1 has not committed. - let tx2 = db.begin_tx(); - let tx2_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "World".to_string().into_bytes(), - }; - assert!(!db.update(tx2, tx2_row).unwrap()); + let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); + let tx2_row = generate_simple_string_row(1, 1, "World"); + assert!(!db + .mvcc_store + .update(tx2, tx2_row, conn2.pager.borrow().clone()) + .unwrap()); let row = db + .mvcc_store .read( tx1, RowID { @@ -301,22 +317,18 @@ fn test_dirty_write() { #[test] fn test_dirty_read() { - let db = test_db(); + let db = MvccTestDb::new(); // T1 inserts a row with ID 1, but does not commit. - let tx1 = db.begin_tx(); - let row1 = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx1, row1).unwrap(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let row1 = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx1, row1).unwrap(); // T2 attempts to read row with ID 1, but doesn't see one because T1 has not committed. - let tx2 = db.begin_tx(); + let conn2 = db._db.connect().unwrap(); + let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); let row2 = db + .mvcc_store .read( tx2, RowID { @@ -330,35 +342,34 @@ fn test_dirty_read() { #[test] fn test_dirty_read_deleted() { - let db = test_db(); + let db = MvccTestDb::new(); // T1 inserts a row with ID 1 and commits. - let tx1 = db.begin_tx(); - let tx1_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx1, tx1_row.clone()).unwrap(); - db.commit_tx(tx1).unwrap(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1_row = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); // T2 deletes row with ID 1, but does not commit. - let tx2 = db.begin_tx(); + let conn2 = db._db.connect().unwrap(); + let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); assert!(db + .mvcc_store .delete( tx2, RowID { table_id: 1, row_id: 1 - } + }, + conn2.pager.borrow().clone(), ) .unwrap()); // T3 reads row with ID 1, but doesn't see the delete because T2 hasn't committed. - let tx3 = db.begin_tx(); + let conn3 = db._db.connect().unwrap(); + let tx3 = db.mvcc_store.begin_tx(conn3.pager.borrow().clone()); let row = db + .mvcc_store .read( tx3, RowID { @@ -373,19 +384,14 @@ fn test_dirty_read_deleted() { #[test] fn test_fuzzy_read() { - let db = test_db(); + let db = MvccTestDb::new(); // T1 inserts a row with ID 1 and commits. - let tx1 = db.begin_tx(); - let tx1_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "First".to_string().into_bytes(), - }; - db.insert(tx1, tx1_row.clone()).unwrap(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1_row = generate_simple_string_row(1, 1, "First"); + db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -396,11 +402,13 @@ fn test_fuzzy_read() { .unwrap() .unwrap(); assert_eq!(tx1_row, row); - db.commit_tx(tx1).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); // T2 reads the row with ID 1 within an active transaction. - let tx2 = db.begin_tx(); + let conn2 = db._db.connect().unwrap(); + let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); let row = db + .mvcc_store .read( tx2, RowID { @@ -413,19 +421,17 @@ fn test_fuzzy_read() { assert_eq!(tx1_row, row); // T3 updates the row and commits. - let tx3 = db.begin_tx(); - let tx3_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Second".to_string().into_bytes(), - }; - db.update(tx3, tx3_row).unwrap(); - db.commit_tx(tx3).unwrap(); + let conn3 = db._db.connect().unwrap(); + let tx3 = db.mvcc_store.begin_tx(conn3.pager.borrow().clone()); + let tx3_row = generate_simple_string_row(1, 1, "Second"); + db.mvcc_store + .update(tx3, tx3_row, conn3.pager.borrow().clone()) + .unwrap(); + commit_tx(db.mvcc_store.clone(), &conn3, tx3).unwrap(); // T2 still reads the same version of the row as before. let row = db + .mvcc_store .read( tx2, RowID { @@ -439,32 +445,23 @@ fn test_fuzzy_read() { // T2 tries to update the row, but fails because T3 has already committed an update to the row, // so T2 trying to write would violate snapshot isolation if it succeeded. - let tx2_newrow = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Third".to_string().into_bytes(), - }; - let update_result = db.update(tx2, tx2_newrow); - assert_eq!(Err(DatabaseError::WriteWriteConflict), update_result); + let tx2_newrow = generate_simple_string_row(1, 1, "Third"); + let update_result = db + .mvcc_store + .update(tx2, tx2_newrow, conn2.pager.borrow().clone()); + assert!(matches!(update_result, Err(LimboError::WriteWriteConflict))); } #[test] fn test_lost_update() { - let db = test_db(); + let db = MvccTestDb::new(); // T1 inserts a row with ID 1 and commits. - let tx1 = db.begin_tx(); - let tx1_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx1, tx1_row.clone()).unwrap(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1_row = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -475,38 +472,37 @@ fn test_lost_update() { .unwrap() .unwrap(); assert_eq!(tx1_row, row); - db.commit_tx(tx1).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); // T2 attempts to update row ID 1 within an active transaction. - let tx2 = db.begin_tx(); - let tx2_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "World".to_string().into_bytes(), - }; - assert!(db.update(tx2, tx2_row.clone()).unwrap()); + let conn2 = db._db.connect().unwrap(); + let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); + let tx2_row = generate_simple_string_row(1, 1, "World"); + assert!(db + .mvcc_store + .update(tx2, tx2_row.clone(), conn2.pager.borrow().clone()) + .unwrap()); // T3 also attempts to update row ID 1 within an active transaction. - let tx3 = db.begin_tx(); - let tx3_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "Hello, world!".to_string().into_bytes(), - }; - assert_eq!( - Err(DatabaseError::WriteWriteConflict), - db.update(tx3, tx3_row) - ); + let conn3 = db._db.connect().unwrap(); + let tx3 = db.mvcc_store.begin_tx(conn3.pager.borrow().clone()); + let tx3_row = generate_simple_string_row(1, 1, "Hello, world!"); + assert!(matches!( + db.mvcc_store + .update(tx3, tx3_row, conn3.pager.borrow().clone(),), + Err(LimboError::WriteWriteConflict) + )); - db.commit_tx(tx2).unwrap(); - assert_eq!(Err(DatabaseError::TxTerminated), db.commit_tx(tx3)); + commit_tx(db.mvcc_store.clone(), &conn2, tx2).unwrap(); + assert!(matches!( + commit_tx(db.mvcc_store.clone(), &conn3, tx3), + Err(LimboError::TxTerminated) + )); - let tx4 = db.begin_tx(); + let conn4 = db._db.connect().unwrap(); + let tx4 = db.mvcc_store.begin_tx(conn4.pager.borrow().clone()); let row = db + .mvcc_store .read( tx4, RowID { @@ -523,31 +519,24 @@ fn test_lost_update() { // This test checks for the typo present in the paper, explained in https://github.com/penberg/mvcc-rs/issues/15 #[test] fn test_committed_visibility() { - let db = test_db(); + let db = MvccTestDb::new(); // let's add $10 to my account since I like money - let tx1 = db.begin_tx(); - let tx1_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "10".to_string().into_bytes(), - }; - db.insert(tx1, tx1_row.clone()).unwrap(); - db.commit_tx(tx1).unwrap(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1_row = generate_simple_string_row(1, 1, "10"); + db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); // but I like more money, so let me try adding $10 more - let tx2 = db.begin_tx(); - let tx2_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "20".to_string().into_bytes(), - }; - assert!(db.update(tx2, tx2_row.clone()).unwrap()); + let conn2 = db._db.connect().unwrap(); + let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); + let tx2_row = generate_simple_string_row(1, 1, "20"); + assert!(db + .mvcc_store + .update(tx2, tx2_row.clone(), conn2.pager.borrow().clone()) + .unwrap()); let row = db + .mvcc_store .read( tx2, RowID { @@ -560,8 +549,10 @@ fn test_committed_visibility() { assert_eq!(row, tx2_row); // can I check how much money I have? - let tx3 = db.begin_tx(); + let conn3 = db._db.connect().unwrap(); + let tx3 = db.mvcc_store.begin_tx(conn3.pager.borrow().clone()); let row = db + .mvcc_store .read( tx3, RowID { @@ -577,22 +568,18 @@ fn test_committed_visibility() { // Test to check if a older transaction can see (un)committed future rows #[test] fn test_future_row() { - let db = test_db(); + let db = MvccTestDb::new(); - let tx1 = db.begin_tx(); + let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); - let tx2 = db.begin_tx(); - let tx2_row = Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "10".to_string().into_bytes(), - }; - db.insert(tx2, tx2_row).unwrap(); + let conn2 = db._db.connect().unwrap(); + let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); + let tx2_row = generate_simple_string_row(1, 1, "Hello"); + db.mvcc_store.insert(tx2, tx2_row).unwrap(); // transaction in progress, so tx1 shouldn't be able to see the value let row = db + .mvcc_store .read( tx1, RowID { @@ -604,8 +591,9 @@ fn test_future_row() { assert_eq!(row, None); // lets commit the transaction and check if tx1 can see it - db.commit_tx(tx2).unwrap(); + commit_tx(db.mvcc_store.clone(), &conn2, tx2).unwrap(); let row = db + .mvcc_store .read( tx1, RowID { @@ -617,92 +605,108 @@ fn test_future_row() { assert_eq!(row, None); } -use crate::mvcc::clock::LogicalClock; use crate::mvcc::cursor::MvccLazyCursor; use crate::mvcc::database::{MvStore, Row, RowID}; -use crate::mvcc::persistent_storage::Storage; -use std::rc::Rc; -use std::sync::atomic::{AtomicU64, Ordering}; +use crate::types::Text; +use crate::Database; +use crate::MemoryIO; +use crate::RefValue; +use crate::Value; // Simple atomic clock implementation for testing -struct TestClock { - counter: AtomicU64, -} -impl TestClock { - fn new(start: u64) -> Self { - Self { - counter: AtomicU64::new(start), - } - } -} - -impl LogicalClock for TestClock { - fn get_timestamp(&self) -> u64 { - self.counter.fetch_add(1, Ordering::SeqCst) - } - - fn reset(&self, ts: u64) { - let current = self.counter.load(Ordering::SeqCst); - if ts > current { - self.counter.store(ts, Ordering::SeqCst); - } - } -} - -fn setup_test_db() -> (Rc>, u64) { - let clock = TestClock::new(1); - let storage = Storage::new_noop(); - let db = Rc::new(MvStore::new(clock, storage)); - let tx_id = db.begin_tx(); +fn setup_test_db() -> (MvccTestDb, u64) { + let db = MvccTestDb::new(); + let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); let table_id = 1; let test_rows = [ - (5, b"row5".to_vec()), - (10, b"row10".to_vec()), - (15, b"row15".to_vec()), - (20, b"row20".to_vec()), - (30, b"row30".to_vec()), + (5, "row5"), + (10, "row10"), + (15, "row15"), + (20, "row20"), + (30, "row30"), ]; for (row_id, data) in test_rows.iter() { let id = RowID::new(table_id, *row_id); - let row = Row::new(id, data.clone()); - db.insert(tx_id, row).unwrap(); + let record = ImmutableRecord::from_values(&[Value::Text(Text::new(data))], 1); + let row = Row::new(id, record.as_blob().to_vec(), 1); + db.mvcc_store.insert(tx_id, row).unwrap(); } - db.commit_tx(tx_id).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx_id).unwrap(); - let tx_id = db.begin_tx(); + let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); (db, tx_id) } -fn setup_lazy_db(initial_keys: &[i64]) -> (Rc>, u64) { - let clock = TestClock::new(1); - let storage = Storage::new_noop(); - let db = Rc::new(MvStore::new(clock, storage)); - let tx_id = db.begin_tx(); +fn setup_lazy_db(initial_keys: &[i64]) -> (MvccTestDb, u64) { + let db = MvccTestDb::new(); + let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); let table_id = 1; for i in initial_keys { let id = RowID::new(table_id, *i); - let data = format!("row{i}").into_bytes(); - let row = Row::new(id, data); - db.insert(tx_id, row).unwrap(); + let data = format!("row{i}"); + let record = ImmutableRecord::from_values(&[Value::Text(Text::new(&data))], 1); + let row = Row::new(id, record.as_blob().to_vec(), 1); + db.mvcc_store.insert(tx_id, row).unwrap(); } - db.commit_tx(tx_id).unwrap(); + commit_tx(db.mvcc_store.clone(), &db.conn, tx_id).unwrap(); - let tx_id = db.begin_tx(); + let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); (db, tx_id) } +pub(crate) fn commit_tx( + mv_store: Arc>, + conn: &Arc, + tx_id: u64, +) -> Result<()> { + let mut sm = mv_store + .commit_tx(tx_id, conn.pager.borrow().clone(), conn) + .unwrap(); + let result = sm.step(&mv_store)?; + assert!(sm.is_finalized()); + match result { + TransitionResult::Done(()) => Ok(()), + _ => unreachable!(), + } +} + +pub(crate) fn commit_tx_no_conn( + db: &MvccTestDbNoConn, + tx_id: u64, + conn: &Arc, +) -> Result<(), LimboError> { + let mut sm = db + .db + .get_mv_store() + .unwrap() + .commit_tx(tx_id, conn.pager.borrow().clone(), conn) + .unwrap(); + let result = sm.step(db.db.mv_store.as_ref().unwrap())?; + assert!(sm.is_finalized()); + match result { + TransitionResult::Done(()) => Ok(()), + _ => unreachable!(), + } +} + #[test] fn test_lazy_scan_cursor_basic() { let (db, tx_id) = setup_lazy_db(&[1, 2, 3, 4, 5]); let table_id = 1; - let mut cursor = MvccLazyCursor::new(db.clone(), tx_id, table_id).unwrap(); + let mut cursor = MvccLazyCursor::new( + db.mvcc_store.clone(), + tx_id, + table_id, + db.conn.pager.borrow().clone(), + ) + .unwrap(); // Check first row assert!(cursor.forward()); @@ -731,7 +735,13 @@ fn test_lazy_scan_cursor_with_gaps() { let (db, tx_id) = setup_test_db(); let table_id = 1; - let mut cursor = MvccLazyCursor::new(db.clone(), tx_id, table_id).unwrap(); + let mut cursor = MvccLazyCursor::new( + db.mvcc_store.clone(), + tx_id, + table_id, + db.conn.pager.borrow().clone(), + ) + .unwrap(); // Check first row assert!(cursor.forward()); @@ -761,7 +771,13 @@ fn test_cursor_basic() { let (db, tx_id) = setup_lazy_db(&[1, 2, 3, 4, 5]); let table_id = 1; - let mut cursor = MvccLazyCursor::new(db.clone(), tx_id, table_id).unwrap(); + let mut cursor = MvccLazyCursor::new( + db.mvcc_store.clone(), + tx_id, + table_id, + db.conn.pager.borrow().clone(), + ) + .unwrap(); cursor.forward(); @@ -788,24 +804,40 @@ fn test_cursor_basic() { #[test] fn test_cursor_with_empty_table() { - let clock = TestClock::new(1); - let storage = Storage::new_noop(); - let db = Rc::new(MvStore::new(clock, storage)); - let tx_id = db.begin_tx(); + let db = MvccTestDb::new(); + { + // FIXME: force page 1 initialization + let pager = db.conn.pager.borrow().clone(); + let tx_id = db.mvcc_store.begin_tx(pager.clone()); + commit_tx(db.mvcc_store.clone(), &db.conn, tx_id).unwrap(); + } + let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); let table_id = 1; // Empty table // Test LazyScanCursor with empty table - let cursor = MvccLazyCursor::new(db.clone(), tx_id, table_id).unwrap(); + let mut cursor = MvccLazyCursor::new( + db.mvcc_store.clone(), + tx_id, + table_id, + db.conn.pager.borrow().clone(), + ) + .unwrap(); assert!(cursor.is_empty()); assert!(cursor.current_row_id().is_none()); } #[test] fn test_cursor_modification_during_scan() { - let (db, tx_id) = setup_lazy_db(&[1, 2, 3, 4, 5]); + let (db, tx_id) = setup_lazy_db(&[1, 2, 4, 5]); let table_id = 1; - let mut cursor = MvccLazyCursor::new(db.clone(), tx_id, table_id).unwrap(); + let mut cursor = MvccLazyCursor::new( + db.mvcc_store.clone(), + tx_id, + table_id, + db.conn.pager.borrow().clone(), + ) + .unwrap(); // Read first row assert!(cursor.forward()); @@ -814,21 +846,36 @@ fn test_cursor_modification_during_scan() { // Insert a new row with ID between existing rows let new_row_id = RowID::new(table_id, 3); - let new_row_data = b"new_row".to_vec(); - let new_row = Row::new(new_row_id, new_row_data); + let new_row = generate_simple_string_row(table_id, new_row_id.row_id, "new_row"); cursor.insert(new_row).unwrap(); - let row = cursor.current_row().unwrap().unwrap(); + let row = db.mvcc_store.read(tx_id, new_row_id).unwrap().unwrap(); + let mut record = ImmutableRecord::new(1024); + record.start_serialization(&row.data); + let value = record.get_value(0).unwrap(); + match value { + RefValue::Text(text) => { + assert_eq!(text.as_str(), "new_row"); + } + _ => panic!("Expected Text value"), + } assert_eq!(row.id.row_id, 3); - assert_eq!(row.data, b"new_row".to_vec()); // Continue scanning - the cursor should still work correctly cursor.forward(); // Move to 4 - let row = cursor.current_row().unwrap().unwrap(); + let row = db + .mvcc_store + .read(tx_id, RowID::new(table_id, 4)) + .unwrap() + .unwrap(); assert_eq!(row.id.row_id, 4); cursor.forward(); // Move to 5 (our new row) - let row = cursor.current_row().unwrap().unwrap(); + let row = db + .mvcc_store + .read(tx_id, RowID::new(table_id, 5)) + .unwrap() + .unwrap(); assert_eq!(row.id.row_id, 5); assert!(!cursor.forward()); assert!(cursor.is_empty()); @@ -907,13 +954,7 @@ fn test_snapshot_isolation_tx_visible1() { let row_version = RowVersion { begin, end, - row: Row { - id: RowID { - table_id: 1, - row_id: 1, - }, - data: "testme".to_string().into_bytes(), - }, + row: generate_simple_string_row(1, 1, "testme"), }; tracing::debug!("Testing visibility of {row_version:?}"); row_version.is_visible_to(¤t_tx, &txs) diff --git a/core/mvcc/errors.rs b/core/mvcc/errors.rs deleted file mode 100644 index 6cdad8ca3..000000000 --- a/core/mvcc/errors.rs +++ /dev/null @@ -1,13 +0,0 @@ -use thiserror::Error; - -#[derive(Error, Debug, PartialEq)] -pub enum DatabaseError { - #[error("no such transaction ID: `{0}`")] - NoSuchTransactionID(u64), - #[error("transaction aborted because of a write-write conflict")] - WriteWriteConflict, - #[error("transaction is terminated")] - TxTerminated, - #[error("I/O error: {0}")] - Io(String), -} diff --git a/core/mvcc/mod.rs b/core/mvcc/mod.rs index a1cf680d8..32b8ce807 100644 --- a/core/mvcc/mod.rs +++ b/core/mvcc/mod.rs @@ -34,7 +34,6 @@ pub mod clock; pub mod cursor; pub mod database; -pub mod errors; pub mod persistent_storage; pub use clock::LocalClock; @@ -42,8 +41,10 @@ pub use database::MvStore; #[cfg(test)] mod tests { - use crate::mvcc::clock::LocalClock; - use crate::mvcc::database::{MvStore, Row, RowID}; + use crate::mvcc::database::tests::{ + commit_tx_no_conn, generate_simple_string_row, MvccTestDbNoConn, + }; + use crate::mvcc::database::RowID; use std::sync::atomic::AtomicI64; use std::sync::atomic::Ordering; use std::sync::Arc; @@ -51,55 +52,52 @@ mod tests { static IDS: AtomicI64 = AtomicI64::new(1); #[test] + #[ignore = "FIXME: This test fails because there is write busy lock yet to be fixed"] fn test_non_overlapping_concurrent_inserts() { // Two threads insert to the database concurrently using non-overlapping // row IDs. - let clock = LocalClock::default(); - let storage = crate::mvcc::persistent_storage::Storage::new_noop(); - let db = Arc::new(MvStore::new(clock, storage)); + let db = Arc::new(MvccTestDbNoConn::new()); let iterations = 100000; let th1 = { let db = db.clone(); std::thread::spawn(move || { + let conn = db.db.connect().unwrap(); + let mvcc_store = db.db.mv_store.as_ref().unwrap().clone(); for _ in 0..iterations { - let tx = db.begin_tx(); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); let id = IDS.fetch_add(1, Ordering::SeqCst); let id = RowID { table_id: 1, row_id: id, }; - let row = Row { - id, - data: "Hello".to_string().into_bytes(), - }; - db.insert(tx, row.clone()).unwrap(); - db.commit_tx(tx).unwrap(); - let tx = db.begin_tx(); - let committed_row = db.read(tx, id).unwrap(); - db.commit_tx(tx).unwrap(); + let row = generate_simple_string_row(1, id.row_id, "Hello"); + mvcc_store.insert(tx, row.clone()).unwrap(); + commit_tx_no_conn(&db, tx, &conn).unwrap(); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let committed_row = mvcc_store.read(tx, id).unwrap(); + commit_tx_no_conn(&db, tx, &conn).unwrap(); assert_eq!(committed_row, Some(row)); } }) }; let th2 = { std::thread::spawn(move || { + let conn = db.db.connect().unwrap(); + let mvcc_store = db.db.mv_store.as_ref().unwrap().clone(); for _ in 0..iterations { - let tx = db.begin_tx(); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); let id = IDS.fetch_add(1, Ordering::SeqCst); let id = RowID { table_id: 1, row_id: id, }; - let row = Row { - id, - data: "World".to_string().into_bytes(), - }; - db.insert(tx, row.clone()).unwrap(); - db.commit_tx(tx).unwrap(); - let tx = db.begin_tx(); - let committed_row = db.read(tx, id).unwrap(); - db.commit_tx(tx).unwrap(); + let row = generate_simple_string_row(1, id.row_id, "World"); + mvcc_store.insert(tx, row.clone()).unwrap(); + commit_tx_no_conn(&db, tx, &conn).unwrap(); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let committed_row = mvcc_store.read(tx, id).unwrap(); + commit_tx_no_conn(&db, tx, &conn).unwrap(); assert_eq!(committed_row, Some(row)); } }) @@ -112,40 +110,38 @@ mod tests { #[test] #[ignore] fn test_overlapping_concurrent_inserts_read_your_writes() { - let clock = LocalClock::default(); - let storage = crate::mvcc::persistent_storage::Storage::new_noop(); - let db = Arc::new(MvStore::new(clock, storage)); + let db = Arc::new(MvccTestDbNoConn::new()); let iterations = 100000; let work = |prefix: &'static str| { let db = db.clone(); std::thread::spawn(move || { + let conn = db.db.connect().unwrap(); + let mvcc_store = db.db.mv_store.as_ref().unwrap().clone(); let mut failed_upserts = 0; for i in 0..iterations { if i % 1000 == 0 { tracing::debug!("{prefix}: {i}"); } if i % 10000 == 0 { - let dropped = db.drop_unused_row_versions(); + let dropped = mvcc_store.drop_unused_row_versions(); tracing::debug!("garbage collected {dropped} versions"); } - let tx = db.begin_tx(); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); let id = i % 16; let id = RowID { table_id: 1, row_id: id, }; - let row = Row { - id, - data: format!("{prefix} @{tx}").into_bytes(), - }; - if let Err(e) = db.upsert(tx, row.clone()) { + let row = generate_simple_string_row(1, id.row_id, &format!("{prefix} @{tx}")); + if let Err(e) = mvcc_store.upsert(tx, row.clone(), conn.pager.borrow().clone()) + { tracing::trace!("upsert failed: {e}"); failed_upserts += 1; continue; } - let committed_row = db.read(tx, id).unwrap(); - db.commit_tx(tx).unwrap(); + let committed_row = mvcc_store.read(tx, id).unwrap(); + commit_tx_no_conn(&db, tx, &conn).unwrap(); assert_eq!(committed_row, Some(row)); } tracing::info!( diff --git a/core/mvcc/persistent_storage/mod.rs b/core/mvcc/persistent_storage/mod.rs index 4b9b06407..3dbd891a0 100644 --- a/core/mvcc/persistent_storage/mod.rs +++ b/core/mvcc/persistent_storage/mod.rs @@ -1,7 +1,7 @@ use std::fmt::Debug; -use crate::mvcc::database::{LogRecord, Result}; -use crate::mvcc::errors::DatabaseError; +use crate::mvcc::database::LogRecord; +use crate::{LimboError, Result}; #[derive(Debug)] pub enum Storage { @@ -24,7 +24,7 @@ impl Storage { pub fn read_tx_log(&self) -> Result> { match self { - Self::Noop => Err(DatabaseError::Io( + Self::Noop => Err(LimboError::InternalError( "cannot read from Noop storage".to_string(), )), } diff --git a/core/state_machine.rs b/core/state_machine.rs new file mode 100644 index 000000000..fc8361480 --- /dev/null +++ b/core/state_machine.rs @@ -0,0 +1,81 @@ +use crate::Result; + +pub enum TransitionResult { + Io, + Continue, + Done(Result), +} + +/// A generic trait for state machines. +pub trait StateTransition { + type State; + type Context; + type SMResult; + + /// Transition the state machine to the next state. + /// + /// Returns `TransitionResult::Io` if the state machine needs to perform an IO operation. + /// Returns `TransitionResult::Continue` if the state machine needs to continue. + /// Returns `TransitionResult::Done` if the state machine is done. + fn step(&mut self, context: &Self::Context) -> Result>; + + /// Finalize the state machine. + /// + /// This is called when the state machine is done. + fn finalize(&mut self, context: &Self::Context) -> Result<()>; + + /// Check if the state machine is finalized. + fn is_finalized(&self) -> bool; +} + +pub struct StateMachine { + state: State, + is_finalized: bool, +} + +/// A generic state machine that loops calling `transition` until it returns `TransitionResult::Done` or `TransitionResult::Io`. +impl StateMachine { + pub fn new(state: State) -> Self { + Self { + state, + is_finalized: false, + } + } +} + +impl StateTransition for StateMachine { + type State = State; + type Context = State::Context; + type SMResult = State::SMResult; + + fn step(&mut self, context: &Self::Context) -> Result> { + loop { + if self.is_finalized { + unreachable!("StateMachine::transition: state machine is finalized"); + } + match self.state.step(context)? { + TransitionResult::Io => { + return Ok(TransitionResult::Io); + } + TransitionResult::Continue => { + continue; + } + TransitionResult::Done(result) => { + assert!(self.state.is_finalized()); + self.is_finalized = true; + return Ok(TransitionResult::Done(result)); + } + } + } + } + + fn finalize(&mut self, context: &Self::Context) -> Result<()> { + self.state.finalize(context)?; + self.is_finalized = true; + Ok(()) + } + + fn is_finalized(&self) -> bool { + self.is_finalized + } +} diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 741900ae8..7b86eb340 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -4,13 +4,13 @@ use tracing::{instrument, Level}; use crate::{ schema::Index, storage::{ - header_accessor, pager::{BtreePageAllocMode, Pager}, sqlite3_ondisk::{ - read_u32, read_varint, BTreeCell, PageContent, PageType, TableInteriorCell, - TableLeafCell, CELL_PTR_SIZE_BYTES, INTERIOR_PAGE_HEADER_SIZE_BYTES, + read_u32, read_varint, BTreeCell, DatabaseHeader, PageContent, PageType, + TableInteriorCell, TableLeafCell, CELL_PTR_SIZE_BYTES, INTERIOR_PAGE_HEADER_SIZE_BYTES, LEAF_PAGE_HEADER_SIZE_BYTES, LEFT_CHILD_PTR_SIZE_BYTES, }, + state_machines::{EmptyTableState, MoveToRightState, SeekToLastState}, }, translate::plan::IterationDirection, turso_assert, @@ -18,6 +18,7 @@ use crate::{ find_compare, get_tie_breaker_from_seek_op, IndexInfo, ParseRecordState, RecordCompare, RecordCursor, SeekResult, }, + util::IOExt, Completion, MvCursor, }; @@ -30,8 +31,7 @@ use crate::{ use super::{ pager::PageRef, sqlite3_ondisk::{ - write_varint_to_vec, IndexInteriorCell, IndexLeafCell, OverflowCell, DATABASE_HEADER_SIZE, - MINIMUM_CELL_SIZE, + write_varint_to_vec, IndexInteriorCell, IndexLeafCell, OverflowCell, MINIMUM_CELL_SIZE, }, }; #[cfg(debug_assertions)] @@ -137,7 +137,7 @@ macro_rules! return_if_locked_maybe_load { return Ok(IOResult::IO); } if !$btree_page.get().is_loaded() { - let (page, c) = $pager.read_page($btree_page.get().get().id)?; + let (page, _c) = $pager.read_page($btree_page.get().get().id)?; $btree_page.page.replace(page); return Ok(IOResult::IO); } @@ -571,6 +571,11 @@ pub struct BTreeCursor { /// - Moving to a different record/row /// - The underlying `ImmutableRecord` is modified pub record_cursor: RefCell, + /// State machine for [BTreeCursor::is_empty_table] + is_empty_table_state: RefCell, + /// State machine for [BTreeCursor::move_to_rightmost] + move_to_right_state: MoveToRightState, + seek_to_last_state: SeekToLastState, } /// We store the cell index and cell count for each page in the stack. @@ -625,6 +630,9 @@ impl BTreeCursor { read_overflow_state: RefCell::new(None), parse_record_state: RefCell::new(ParseRecordState::Init), record_cursor: RefCell::new(RecordCursor::with_capacity(num_columns)), + is_empty_table_state: RefCell::new(EmptyTableState::Start), + move_to_right_state: MoveToRightState::Start, + seek_to_last_state: SeekToLastState::Start, } } @@ -680,15 +688,24 @@ impl BTreeCursor { /// This is done by checking if the root page has no cells. #[instrument(skip_all, level = Level::DEBUG)] fn is_empty_table(&self) -> Result> { - if let Some(mv_cursor) = &self.mv_cursor { - let mv_cursor = mv_cursor.borrow(); - return Ok(IOResult::Done(mv_cursor.is_empty())); + let state = self.is_empty_table_state.borrow().clone(); + match state { + EmptyTableState::Start => { + if let Some(mv_cursor) = &self.mv_cursor { + let mv_cursor = mv_cursor.borrow(); + return Ok(IOResult::Done(mv_cursor.is_empty())); + } + let (page, _c) = self.pager.read_page(self.root_page)?; + *self.is_empty_table_state.borrow_mut() = EmptyTableState::ReadPage { page }; + Ok(IOResult::IO) + } + EmptyTableState::ReadPage { page } => { + // TODO: Remove this line after we start awaiting for completions + return_if_locked!(page); + let cell_count = page.get().contents.as_ref().unwrap().cell_count(); + Ok(IOResult::Done(cell_count == 0)) + } } - let (page, c) = self.pager.read_page(self.root_page)?; - return_if_locked!(page); - - let cell_count = page.get().contents.as_ref().unwrap().cell_count(); - Ok(IOResult::Done(cell_count == 0)) } /// Move the cursor to the previous record and return it. @@ -701,6 +718,8 @@ impl BTreeCursor { return_if_locked_maybe_load!(self.pager, page); let page = page.get(); let contents = page.get().contents.as_ref().unwrap(); + let page_type = contents.page_type(); + let is_index = page.is_index(); let cell_count = contents.cell_count(); let cell_idx = self.stack.current_cell_index(); @@ -712,7 +731,7 @@ impl BTreeCursor { if let Some(rightmost_pointer) = rightmost_pointer { let past_rightmost_pointer = cell_count as i32 + 1; self.stack.set_cell_index(past_rightmost_pointer); - let (page, c) = self.read_page(rightmost_pointer as usize)?; + let (page, _c) = self.read_page(rightmost_pointer as usize)?; self.stack.push_backwards(page); continue; } @@ -720,10 +739,8 @@ impl BTreeCursor { if cell_idx >= cell_count as i32 { self.stack.set_cell_index(cell_count as i32 - 1); } else if !self.stack.current_cell_index_less_than_min() { - let is_index = page.is_index(); // skip retreat in case we still haven't visited this cell in index let should_visit_internal_node = is_index && self.going_upwards; // we are going upwards, this means we still need to visit divider cell in an index - let page_type = contents.page_type(); if should_visit_internal_node { self.going_upwards = false; return Ok(IOResult::Done(true)); @@ -753,48 +770,34 @@ impl BTreeCursor { // continue to next loop to get record from the new page continue; } - let cell_idx = self.stack.current_cell_index() as usize; - - let cell = contents.cell_get(cell_idx, self.usable_space())?; - - match cell { - BTreeCell::TableInteriorCell(TableInteriorCell { - left_child_page, .. - }) => { - let (mem_page, c) = self.read_page(left_child_page as usize)?; - self.stack.push_backwards(mem_page); - continue; - } - BTreeCell::TableLeafCell(TableLeafCell { .. }) => { - return Ok(IOResult::Done(true)); - } - BTreeCell::IndexInteriorCell(IndexInteriorCell { - left_child_page, .. - }) => { - if !self.going_upwards { - // In backwards iteration, if we haven't just moved to this interior node from the - // right child, but instead are about to move to the left child, we need to retreat - // so that we don't come back to this node again. - // For example: - // this parent: key 666 - // left child has: key 663, key 664, key 665 - // we need to move to the previous parent (with e.g. key 662) when iterating backwards. - let (mem_page, c) = self.read_page(left_child_page as usize)?; - self.stack.retreat(); - self.stack.push_backwards(mem_page); - continue; - } - - // Going upwards = we just moved to an interior cell from the right child. - // On the first pass we must take the record from the interior cell (since unlike table btrees, index interior cells have payloads) - // We then mark going_upwards=false so that we go back down the tree on the next invocation. - self.going_upwards = false; - return Ok(IOResult::Done(true)); - } - BTreeCell::IndexLeafCell(IndexLeafCell { .. }) => { - return Ok(IOResult::Done(true)); - } + if contents.is_leaf() { + return Ok(IOResult::Done(true)); } + + if is_index && self.going_upwards { + // If we are going upwards, we need to visit the divider cell before going back to another child page. + // This is because index interior cells have payloads, so unless we do this we will be skipping an entry when traversing the tree. + self.going_upwards = false; + return Ok(IOResult::Done(true)); + } + + let cell_idx = self.stack.current_cell_index() as usize; + let left_child_page = contents.cell_interior_read_left_child_page(cell_idx); + + if page_type == PageType::IndexInterior { + // In backwards iteration, if we haven't just moved to this interior node from the + // right child, but instead are about to move to the left child, we need to retreat + // so that we don't come back to this node again. + // For example: + // this parent: key 666 + // left child has: key 663, key 664, key 665 + // we need to move to the previous parent (with e.g. key 662) when iterating backwards. + self.stack.retreat(); + } + + let (mem_page, _c) = self.read_page(left_child_page as usize)?; + self.stack.push_backwards(mem_page); + continue; } } @@ -808,7 +811,7 @@ impl BTreeCursor { payload_size: u64, ) -> Result> { if self.read_overflow_state.borrow().is_none() { - let (page, c) = self.read_page(start_next_page as usize)?; + let (page, _c) = self.read_page(start_next_page as usize)?; *self.read_overflow_state.borrow_mut() = Some(ReadPayloadOverflow { payload: payload.to_vec(), next_page: start_next_page, @@ -840,7 +843,7 @@ impl BTreeCursor { *remaining_to_read -= to_read; if *remaining_to_read != 0 && next != 0 { - let (new_page, c) = self.pager.read_page(next as usize).map(|(page, c)| { + let (new_page, _c) = self.pager.read_page(next as usize).map(|(page, c)| { ( Arc::new(BTreePageInner { page: RefCell::new(page), @@ -1047,7 +1050,7 @@ impl BTreeCursor { is_write, }) => { if *pages_left_to_skip == 0 { - let (page, c) = self.read_page(*next_page as usize)?; + let (page, _c) = self.read_page(*next_page as usize)?; return_if_locked_maybe_load!(self.pager, page); self.state = CursorState::ReadWritePayload(PayloadOverflowWithOffset::ProcessPage { @@ -1062,7 +1065,7 @@ impl BTreeCursor { continue; } - let (page, c) = self.read_page(*next_page as usize)?; + let (page, _c) = self.read_page(*next_page as usize)?; return_if_locked_maybe_load!(self.pager, page); let page = page.get(); let contents = page.get_contents(); @@ -1157,7 +1160,7 @@ impl BTreeCursor { // Load next page *next_page = next; *current_offset = 0; // Reset offset for new page - let (page, c) = self.read_page(next as usize)?; + let (page, _c) = self.read_page(next as usize)?; *page_btree = page; // Return IO to allow other operations @@ -1269,7 +1272,7 @@ impl BTreeCursor { (Some(right_most_pointer), false) => { // do rightmost self.stack.advance(); - let (mem_page, c) = self.read_page(right_most_pointer as usize)?; + let (mem_page, _c) = self.read_page(right_most_pointer as usize)?; self.stack.push(mem_page); continue; } @@ -1296,34 +1299,20 @@ impl BTreeCursor { mem_page_rc.get().get().id ); - let cell = contents.cell_get(cell_idx, self.usable_space())?; - match &cell { - BTreeCell::TableInteriorCell(TableInteriorCell { - left_child_page, .. - }) => { - let (mem_page, c) = self.read_page(*left_child_page as usize)?; - self.stack.push(mem_page); - continue; - } - BTreeCell::TableLeafCell(TableLeafCell { .. }) => { - return Ok(IOResult::Done(true)); - } - BTreeCell::IndexInteriorCell(IndexInteriorCell { - left_child_page, .. - }) => { - if self.going_upwards { - self.going_upwards = false; - return Ok(IOResult::Done(true)); - } else { - let (mem_page, c) = self.read_page(*left_child_page as usize)?; - self.stack.push(mem_page); - continue; - } - } - BTreeCell::IndexLeafCell(IndexLeafCell { .. }) => { - return Ok(IOResult::Done(true)); - } + if contents.is_leaf() { + return Ok(IOResult::Done(true)); } + if is_index && self.going_upwards { + // This means we just came up from a child, so now we need to visit the divider cell before going back to another child page. + // This is because index interior cells have payloads, so unless we do this we will be skipping an entry when traversing the tree. + self.going_upwards = false; + return Ok(IOResult::Done(true)); + } + + let left_child_page = contents.cell_interior_read_left_child_page(cell_idx); + let (mem_page, _c) = self.read_page(left_child_page as usize)?; + self.stack.push(mem_page); + continue; } } @@ -1346,46 +1335,52 @@ impl BTreeCursor { /// Move the cursor to the root page of the btree. #[instrument(skip_all, level = Level::DEBUG)] - fn move_to_root(&mut self) -> Result<()> { + fn move_to_root(&mut self) -> Result { self.seek_state = CursorSeekState::Start; self.going_upwards = false; tracing::trace!(root_page = self.root_page); let (mem_page, c) = self.read_page(self.root_page)?; self.stack.clear(); self.stack.push(mem_page); - Ok(()) + Ok(c) } /// Move the cursor to the rightmost record in the btree. #[instrument(skip(self), level = Level::DEBUG)] fn move_to_rightmost(&mut self) -> Result> { - self.move_to_root()?; - - loop { - let mem_page = self.stack.top(); - let page_idx = mem_page.get().get().id; - let (page, c) = self.read_page(page_idx)?; - return_if_locked_maybe_load!(self.pager, page); - let page = page.get(); - let contents = page.get().contents.as_ref().unwrap(); - if contents.is_leaf() { - if contents.cell_count() > 0 { - self.stack.set_cell_index(contents.cell_count() as i32 - 1); - return Ok(IOResult::Done(true)); - } - return Ok(IOResult::Done(false)); + match self.move_to_right_state { + MoveToRightState::Start => { + let _c = self.move_to_root()?; + self.move_to_right_state = MoveToRightState::ProcessPage; + return Ok(IOResult::IO); } - - match contents.rightmost_pointer() { - Some(right_most_pointer) => { - self.stack.set_cell_index(contents.cell_count() as i32 + 1); - let (mem_page, c) = self.read_page(right_most_pointer as usize)?; - self.stack.push(mem_page); - continue; + MoveToRightState::ProcessPage => { + let mem_page = self.stack.top(); + let page_idx = mem_page.get().get().id; + let (page, _c) = self.read_page(page_idx)?; + return_if_locked_maybe_load!(self.pager, page); + let page = page.get(); + let contents = page.get().contents.as_ref().unwrap(); + if contents.is_leaf() { + self.move_to_right_state = MoveToRightState::Start; + if contents.cell_count() > 0 { + self.stack.set_cell_index(contents.cell_count() as i32 - 1); + return Ok(IOResult::Done(true)); + } + return Ok(IOResult::Done(false)); } - None => { - unreachable!("interior page should have a rightmost pointer"); + match contents.rightmost_pointer() { + Some(right_most_pointer) => { + self.stack.set_cell_index(contents.cell_count() as i32 + 1); + let (mem_page, _c) = self.read_page(right_most_pointer as usize)?; + self.stack.push(mem_page); + return Ok(IOResult::IO); + } + + None => { + unreachable!("interior page should have a rightmost pointer"); + } } } } @@ -1446,7 +1441,7 @@ impl BTreeCursor { let left_child_page = contents.cell_interior_read_left_child_page(nearest_matching_cell); self.stack.set_cell_index(nearest_matching_cell as i32); - let (mem_page, c) = self.read_page(left_child_page as usize)?; + let (mem_page, _c) = self.read_page(left_child_page as usize)?; self.stack.push(mem_page); self.seek_state = CursorSeekState::MovingBetweenPages { eq_seen: Cell::new(eq_seen.get()), @@ -1456,7 +1451,7 @@ impl BTreeCursor { self.stack.set_cell_index(cell_count as i32 + 1); match contents.rightmost_pointer() { Some(right_most_pointer) => { - let (mem_page, c) = self.read_page(right_most_pointer as usize)?; + let (mem_page, _c) = self.read_page(right_most_pointer as usize)?; self.stack.push(mem_page); self.seek_state = CursorSeekState::MovingBetweenPages { eq_seen: Cell::new(eq_seen.get()), @@ -1587,7 +1582,7 @@ impl BTreeCursor { self.stack.set_cell_index(contents.cell_count() as i32 + 1); match contents.rightmost_pointer() { Some(right_most_pointer) => { - let (mem_page, c) = self.read_page(right_most_pointer as usize)?; + let (mem_page, _c) = self.read_page(right_most_pointer as usize)?; self.stack.push(mem_page); self.seek_state = CursorSeekState::MovingBetweenPages { eq_seen: Cell::new(eq_seen.get()), @@ -1627,7 +1622,7 @@ impl BTreeCursor { page.get().id ); - let (mem_page, c) = self.read_page(*left_child_page as usize)?; + let (mem_page, _c) = self.read_page(*left_child_page as usize)?; self.stack.push(mem_page); self.seek_state = CursorSeekState::MovingBetweenPages { eq_seen: Cell::new(eq_seen.get()), @@ -2129,7 +2124,7 @@ impl BTreeCursor { self.seek_state = CursorSeekState::Start; } if matches!(self.seek_state, CursorSeekState::Start) { - self.move_to_root()?; + let _c = self.move_to_root()?; } let ret = match key { @@ -2447,7 +2442,7 @@ impl BTreeCursor { } if !self.stack.has_parent() { - let res = self.balance_root()?; + let _res = self.balance_root()?; } let write_info = self.state.mut_write_info().unwrap(); @@ -2615,7 +2610,7 @@ impl BTreeCursor { let mut pgno: u32 = unsafe { right_pointer.cast::().read().swap_bytes() }; let current_sibling = sibling_pointer; for i in (0..=current_sibling).rev() { - let (page, c) = self.read_page(pgno as usize)?; + let (page, _c) = self.read_page(pgno as usize)?; { // mark as dirty let sibling_page = page.get(); @@ -3362,7 +3357,11 @@ impl BTreeCursor { "left pointer is not the same as page id" ); // FIXME: remove this lock - let database_size = header_accessor::get_database_size(&self.pager)?; + let database_size = self + .pager + .io + .block(|| self.pager.with_header(|header| header.database_size))? + .get(); turso_assert!( left_pointer <= database_size, "invalid page number divider left pointer {} > database number of pages {}", @@ -3521,7 +3520,7 @@ impl BTreeCursor { // sub-algorithm in some documentation. assert!(sibling_count_new == 1); let parent_offset = if parent_page.get().id == 1 { - DATABASE_HEADER_SIZE + DatabaseHeader::SIZE } else { 0 }; @@ -4072,7 +4071,7 @@ impl BTreeCursor { current_root.get().get().id == 1 }; - let offset = if is_page_1 { DATABASE_HEADER_SIZE } else { 0 }; + let offset = if is_page_1 { DatabaseHeader::SIZE } else { 0 }; let root_btree = self.stack.top(); let root = root_btree.get(); @@ -4160,11 +4159,11 @@ impl BTreeCursor { pub fn seek_end(&mut self) -> Result> { assert!(self.mv_cursor.is_none()); // unsure about this -_- - self.move_to_root()?; + let _c = self.move_to_root()?; loop { let mem_page = self.stack.top(); let page_id = mem_page.get().get().id; - let (page, c) = self.read_page(page_id)?; + let (page, _c) = self.read_page(page_id)?; return_if_locked_maybe_load!(self.pager, page); let page = page.get(); @@ -4178,7 +4177,7 @@ impl BTreeCursor { match contents.rightmost_pointer() { Some(right_most_pointer) => { self.stack.set_cell_index(contents.cell_count() as i32 + 1); // invalid on interior - let (child, c) = self.read_page(right_most_pointer as usize)?; + let (child, _c) = self.read_page(right_most_pointer as usize)?; self.stack.push(child); } None => unreachable!("interior page must have rightmost pointer"), @@ -4188,16 +4187,27 @@ impl BTreeCursor { #[instrument(skip_all, level = Level::DEBUG)] pub fn seek_to_last(&mut self) -> Result> { - assert!(self.mv_cursor.is_none()); - let has_record = return_if_io!(self.move_to_rightmost()); - self.invalidate_record(); - self.has_record.replace(has_record); - if !has_record { - let is_empty = return_if_io!(self.is_empty_table()); - assert!(is_empty); - return Ok(IOResult::Done(())); + loop { + match self.seek_to_last_state { + SeekToLastState::Start => { + assert!(self.mv_cursor.is_none()); + let has_record = return_if_io!(self.move_to_rightmost()); + self.invalidate_record(); + self.has_record.replace(has_record); + if !has_record { + self.seek_to_last_state = SeekToLastState::IsEmpty; + continue; + } + return Ok(IOResult::Done(())); + } + SeekToLastState::IsEmpty => { + let is_empty = return_if_io!(self.is_empty_table()); + assert!(is_empty); + self.seek_to_last_state = SeekToLastState::Start; + return Ok(IOResult::Done(())); + } + } } - Ok(IOResult::Done(())) } pub fn is_empty(&self) -> bool { @@ -4219,7 +4229,7 @@ impl BTreeCursor { self.invalidate_record(); self.has_record.replace(cursor_has_record); } else { - self.move_to_root()?; + let _c = self.move_to_root()?; let cursor_has_record = return_if_io!(self.get_next_record()); self.invalidate_record(); @@ -4268,7 +4278,7 @@ impl BTreeCursor { pub fn rowid(&mut self) -> Result>> { if let Some(mv_cursor) = &self.mv_cursor { if self.has_record.get() { - let mv_cursor = mv_cursor.borrow(); + let mut mv_cursor = mv_cursor.borrow_mut(); return Ok(IOResult::Done( mv_cursor.current_row_id().map(|rowid| rowid.row_id), )); @@ -4279,22 +4289,15 @@ impl BTreeCursor { if self.has_record.get() { let page = self.stack.top(); return_if_locked_maybe_load!(self.pager, page); - // load record - let _ = return_if_io!(self.record()); - let page_type = page.get().get_contents().page_type(); let page = page.get(); let contents = page.get_contents(); - let cell_idx = self.stack.current_cell_index(); - let cell = contents.cell_get(cell_idx as usize, self.usable_space())?; + let page_type = contents.page_type(); if page_type.is_table() { - let BTreeCell::TableLeafCell(TableLeafCell { rowid, .. }) = cell else { - unreachable!( - "BTreeCursor::rowid(): unexpected page_type: {:?}", - page_type - ); - }; + let cell_idx = self.stack.current_cell_index(); + let rowid = contents.cell_table_leaf_read_rowid(cell_idx as usize)?; Ok(IOResult::Done(Some(rowid))) } else { + let _ = return_if_io!(self.record()); Ok(IOResult::Done(self.get_index_rowid_from_record())) } } else { @@ -4340,7 +4343,7 @@ impl BTreeCursor { return Ok(IOResult::Done(Some(record_ref))); } if self.mv_cursor.is_some() { - let mv_cursor = self.mv_cursor.as_ref().unwrap().borrow(); + let mut mv_cursor = self.mv_cursor.as_ref().unwrap().borrow_mut(); let row = mv_cursor.current_row().unwrap().unwrap(); self.get_immutable_record_or_create() .as_mut() @@ -4423,7 +4426,13 @@ impl BTreeCursor { Some(rowid) => { let row_id = crate::mvcc::database::RowID::new(self.table_id() as u64, rowid); let record_buf = key.get_record().unwrap().get_payload().to_vec(); - let row = crate::mvcc::database::Row::new(row_id, record_buf); + let num_columns = match key { + BTreeKey::IndexKey(record) => record.column_count(), + BTreeKey::TableRowId((_, record)) => { + record.as_ref().unwrap().column_count() + } + }; + let row = crate::mvcc::database::Row::new(row_id, record_buf, num_columns); mv_cursor.borrow_mut().insert(row).unwrap(); } None => todo!("Support mvcc inserts with index btrees"), @@ -4963,12 +4972,16 @@ impl BTreeCursor { OverflowState::ProcessPage { next_page } => { if next_page < 2 || next_page as usize - > header_accessor::get_database_size(&self.pager)? as usize + > self + .pager + .io + .block(|| self.pager.with_header(|header| header.database_size))? + .get() as usize { self.overflow_state = None; return Err(LimboError::Corrupt("Invalid overflow page number".into())); } - let (page, c) = self.read_page(next_page as usize)?; + let (page, _c) = self.read_page(next_page as usize)?; return_if_locked_maybe_load!(self.pager, page); let page = page.get(); @@ -5009,7 +5022,7 @@ impl BTreeCursor { #[instrument(skip(self), level = Level::DEBUG)] pub fn btree_destroy(&mut self) -> Result>> { if let CursorState::None = &self.state { - self.move_to_root()?; + let _c = self.move_to_root()?; self.state = CursorState::Destroy(DestroyInfo { state: DestroyState::Start, }); @@ -5064,7 +5077,8 @@ impl BTreeCursor { // Non-leaf page which has processed all children but not it's potential right child (false, n) if n == contents.cell_count() as i32 => { if let Some(rightmost) = contents.rightmost_pointer() { - let (rightmost_page, c) = self.read_page(rightmost as usize)?; + let (rightmost_page, _c) = + self.read_page(rightmost as usize)?; self.stack.push(rightmost_page); let destroy_info = self.state.mut_destroy_info().expect( "unable to get a mut reference to destroy state in cursor", @@ -5121,7 +5135,7 @@ impl BTreeCursor { BTreeCell::IndexInteriorCell(cell) => cell.left_child_page, _ => panic!("expected interior cell"), }; - let (child_page, c) = self.read_page(child_page_id as usize)?; + let (child_page, _c) = self.read_page(child_page_id as usize)?; self.stack.push(child_page); let destroy_info = self.state.mut_destroy_info().expect( "unable to get a mut reference to destroy state in cursor", @@ -5137,7 +5151,7 @@ impl BTreeCursor { IOResult::Done(_) => match cell { // For an index interior cell, clear the left child page now that overflow pages have been cleared BTreeCell::IndexInteriorCell(index_int_cell) => { - let (child_page, c) = + let (child_page, _c) = self.read_page(index_int_cell.left_child_page as usize)?; self.stack.push(child_page); let destroy_info = self.state.mut_destroy_info().expect( @@ -5260,7 +5274,7 @@ impl BTreeCursor { let new_payload = &mut *new_payload; // if it all fits in local space and old_local_size is enough, do an in-place overwrite if new_payload.len() == *old_local_size { - let res = + let _res = self.overwrite_content(page_ref.clone(), *old_offset, new_payload)?; return Ok(IOResult::Done(())); } @@ -5318,7 +5332,7 @@ impl BTreeCursor { #[instrument(skip(self), level = Level::DEBUG)] pub fn count(&mut self) -> Result> { if self.count == 0 { - self.move_to_root()?; + let _c = self.move_to_root()?; } if let Some(_mv_cursor) = &self.mv_cursor { @@ -5351,7 +5365,7 @@ impl BTreeCursor { loop { if !self.stack.has_parent() { // All pages of the b-tree have been visited. Return successfully - self.move_to_root()?; + let _c = self.move_to_root()?; return Ok(IOResult::Done(self.count)); } @@ -5382,7 +5396,7 @@ impl BTreeCursor { // should be safe as contents is not a leaf page let right_most_pointer = contents.rightmost_pointer().unwrap(); self.stack.advance(); - let (mem_page, c) = self.read_page(right_most_pointer as usize)?; + let (mem_page, _c) = self.read_page(right_most_pointer as usize)?; self.stack.push(mem_page); } else { // Move to child left page @@ -5396,7 +5410,7 @@ impl BTreeCursor { left_child_page, .. }) => { self.stack.advance(); - let (mem_page, c) = self.read_page(left_child_page as usize)?; + let (mem_page, _c) = self.read_page(left_child_page as usize)?; self.stack.push(mem_page); } _ => unreachable!(), @@ -5576,7 +5590,7 @@ pub fn integrity_check( else { return Ok(IOResult::Done(())); }; - let (page, c) = btree_read_page(pager, page_idx)?; + let (page, _c) = btree_read_page(pager, page_idx)?; return_if_locked_maybe_load!(pager, page); state.page_stack.pop(); @@ -7023,9 +7037,9 @@ mod tests { database::DatabaseFile, page_cache::DumbLruPageCache, pager::{AtomicDbState, DbState}, + sqlite3_ondisk::PageSize, }, types::Text, - util::IOExt as _, vdbe::Register, BufferPool, Completion, Connection, StepResult, WalFile, WalFileShared, }; @@ -7188,7 +7202,7 @@ mod tests { fn validate_btree(pager: Rc, page_idx: usize) -> (usize, bool) { let num_columns = 5; let cursor = BTreeCursor::new_table(None, pager.clone(), page_idx, num_columns); - let (page, c) = cursor.read_page(page_idx).unwrap(); + let (page, _c) = cursor.read_page(page_idx).unwrap(); while page.get().is_locked() { pager.io.run_once().unwrap(); } @@ -7208,7 +7222,7 @@ mod tests { BTreeCell::TableInteriorCell(TableInteriorCell { left_child_page, .. }) => { - let (child_page, c) = cursor.read_page(left_child_page as usize).unwrap(); + let (child_page, _c) = cursor.read_page(left_child_page as usize).unwrap(); while child_page.get().is_locked() { pager.io.run_once().unwrap(); } @@ -7265,7 +7279,7 @@ mod tests { } let first_page_type = child_pages.first().map(|p| { if !p.get().is_loaded() { - let (new_page, c) = pager.read_page(p.get().get().id).unwrap(); + let (new_page, _c) = pager.read_page(p.get().get().id).unwrap(); p.page.replace(new_page); } while p.get().is_locked() { @@ -7276,7 +7290,7 @@ mod tests { if let Some(child_type) = first_page_type { for page in child_pages.iter().skip(1) { if !page.get().is_loaded() { - let (new_page, c) = pager.read_page(page.get().get().id).unwrap(); + let (new_page, _c) = pager.read_page(page.get().get().id).unwrap(); page.page.replace(new_page); } while page.get().is_locked() { @@ -7299,7 +7313,7 @@ mod tests { let num_columns = 5; let cursor = BTreeCursor::new_table(None, pager.clone(), page_idx, num_columns); - let (page, c) = cursor.read_page(page_idx).unwrap(); + let (page, _c) = cursor.read_page(page_idx).unwrap(); while page.get().is_locked() { pager.io.run_once().unwrap(); } @@ -7392,7 +7406,10 @@ mod tests { // Create cursor for the table let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page, num_columns); - let initial_pagecount = header_accessor::get_database_size(&pager).unwrap(); + let initial_pagecount = pager + .io + .block(|| pager.with_header(|header| header.database_size.get())) + .unwrap(); assert_eq!( initial_pagecount, 2, "Page count should be 2 after initial insert, was {initial_pagecount}" @@ -7413,12 +7430,18 @@ mod tests { // Verify that overflow pages were created by checking freelist count // The freelist count should be 0 initially, and after inserting a large record, // some pages should be allocated for overflow, but they won't be in freelist yet - let freelist_after_insert = header_accessor::get_freelist_pages(&pager).unwrap(); + let freelist_after_insert = pager + .io + .block(|| pager.with_header(|header| header.freelist_pages.get())) + .unwrap(); assert_eq!( freelist_after_insert, 0, "Freelist count should be 0 after insert, was {freelist_after_insert}" ); - let pagecount_after_insert = header_accessor::get_database_size(&pager).unwrap(); + let pagecount_after_insert = pager + .io + .block(|| pager.with_header(|header| header.database_size.get())) + .unwrap(); const EXPECTED_OVERFLOW_PAGES: u32 = 3; assert_eq!( pagecount_after_insert, @@ -7447,7 +7470,10 @@ mod tests { run_until_done(|| cursor.insert(&key, true), pager.deref()).unwrap(); // Check that the freelist count has increased, indicating overflow pages were cleared - let freelist_after_overwrite = header_accessor::get_freelist_pages(&pager).unwrap(); + let freelist_after_overwrite = pager + .io + .block(|| pager.with_header(|header| header.freelist_pages.get())) + .unwrap(); assert_eq!(freelist_after_overwrite, EXPECTED_OVERFLOW_PAGES, "Freelist count should be {EXPECTED_OVERFLOW_PAGES} after overwrite, was {freelist_after_overwrite}"); // Verify the record was actually overwritten by reading it back @@ -7655,10 +7681,10 @@ mod tests { } pager.begin_read_tx().unwrap(); // FIXME: add sorted vector instead, should be okay for small amounts of keys for now :P, too lazy to fix right now - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); let mut valid = true; if do_validate { - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); for key in keys.iter() { tracing::trace!("seeking key: {}", key); run_until_done(|| cursor.next(), pager.deref()).unwrap(); @@ -7691,7 +7717,7 @@ mod tests { if matches!(validate_btree(pager.clone(), root_page), (_, false)) { panic!("invalid btree"); } - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); for key in keys.iter() { tracing::trace!("seeking key: {}", key); run_until_done(|| cursor.next(), pager.deref()).unwrap(); @@ -7759,7 +7785,7 @@ mod tests { tracing::info!("seed: {seed}"); for i in 0..inserts { pager.begin_read_tx().unwrap(); - let res = pager.begin_write_tx().unwrap(); + let _res = pager.begin_write_tx().unwrap(); let key = { let result; loop { @@ -7802,7 +7828,7 @@ mod tests { pager.deref(), ) .unwrap(); - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); loop { match pager.end_tx(false, false, &conn, false).unwrap() { IOResult::Done(_) => break, @@ -7815,7 +7841,7 @@ mod tests { // Check that all keys can be found by seeking pager.begin_read_tx().unwrap(); - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); for (i, key) in keys.iter().enumerate() { tracing::info!("seeking key {}/{}: {:?}", i + 1, keys.len(), key); let exists = run_until_done( @@ -7839,7 +7865,7 @@ mod tests { assert!(found, "key {key:?} is not found"); } // Check that key count is right - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); let mut count = 0; while run_until_done(|| cursor.next(), pager.deref()).unwrap() { count += 1; @@ -7852,7 +7878,7 @@ mod tests { keys.len() ); // Check that all keys can be found in-order, by iterating the btree - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); let mut prev = None; for (i, key) in keys.iter().enumerate() { tracing::info!("iterating key {}/{}: {:?}", i + 1, keys.len(), key); @@ -7929,7 +7955,7 @@ mod tests { for i in 0..operations { let print_progress = i % 100 == 0; pager.begin_read_tx().unwrap(); - let res = pager.begin_write_tx().unwrap(); + let _res = pager.begin_write_tx().unwrap(); // Decide whether to insert or delete (80% chance of insert) let is_insert = rng.next_u64() % 100 < (insert_chance * 100.0) as u64; @@ -8020,7 +8046,7 @@ mod tests { } } - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); loop { match pager.end_tx(false, false, &conn, false).unwrap() { IOResult::Done(_) => break, @@ -8048,7 +8074,7 @@ mod tests { ) { // Check that all expected keys can be found by seeking pager.begin_read_tx().unwrap(); - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); for (i, key) in expected_keys.iter().enumerate() { tracing::info!( "validating key {}/{}, seed: {seed}", @@ -8074,7 +8100,7 @@ mod tests { } // Check key count - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); run_until_done(|| cursor.rewind(), pager.deref()).unwrap(); if !cursor.has_record.get() { panic!("no keys in tree"); @@ -8096,7 +8122,7 @@ mod tests { ); // Check that all keys can be found in-order, by iterating the btree - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); for (i, key) in expected_keys.iter().enumerate() { run_until_done(|| cursor.next(), pager.deref()).unwrap(); tracing::info!( @@ -8296,7 +8322,7 @@ mod tests { let pager = Rc::new( Pager::new( db_file, - wal, + Some(wal), io, Arc::new(parking_lot::RwLock::new(DumbLruPageCache::new(10))), buffer_pool, @@ -8310,10 +8336,15 @@ mod tests { let _ = run_until_done(|| pager.allocate_page1(), &pager); for _ in 0..(database_size - 1) { - let res = pager.allocate_page().unwrap(); + let _res = pager.allocate_page().unwrap(); } - header_accessor::set_page_size(&pager, page_size).unwrap(); + pager + .io + .block(|| { + pager.with_header_mut(|header| header.page_size = PageSize::new(page_size).unwrap()) + }) + .unwrap(); pager } @@ -8337,17 +8368,20 @@ mod tests { let drop_fn = Rc::new(|_buf| {}); #[allow(clippy::arc_with_non_send_sync)] let buf = Arc::new(RefCell::new(Buffer::allocate( - header_accessor::get_page_size(&pager)? as usize, + pager + .io + .block(|| pager.with_header(|header| header.page_size))? + .get() as usize, drop_fn, ))); let c = Completion::new_write(|_| {}); #[allow(clippy::arc_with_non_send_sync)] - let c = pager + let _c = pager .db_file .write_page(current_page as usize, buf.clone(), c)?; pager.io.run_once()?; - let (page, c) = cursor.read_page(current_page as usize)?; + let (page, _c) = cursor.read_page(current_page as usize)?; while page.get().is_locked() { cursor.pager.io.run_once()?; } @@ -8379,23 +8413,38 @@ mod tests { payload_size: large_payload.len() as u64, }); - let initial_freelist_pages = header_accessor::get_freelist_pages(&pager)?; + let initial_freelist_pages = pager + .io + .block(|| pager.with_header(|header| header.freelist_pages))? + .get(); // Clear overflow pages let clear_result = cursor.clear_overflow_pages(&leaf_cell)?; match clear_result { IOResult::Done(_) => { + let (freelist_pages, freelist_trunk_page) = pager + .io + .block(|| { + pager.with_header(|header| { + ( + header.freelist_pages.get(), + header.freelist_trunk_page.get(), + ) + }) + }) + .unwrap(); + // Verify proper number of pages were added to freelist assert_eq!( - header_accessor::get_freelist_pages(&pager)?, + freelist_pages, initial_freelist_pages + 3, "Expected 3 pages to be added to freelist" ); // If this is first trunk page - let trunk_page_id = header_accessor::get_freelist_trunk_page(&pager)?; + let trunk_page_id = freelist_trunk_page; if trunk_page_id > 0 { // Verify trunk page structure - let (trunk_page, c) = cursor.read_page(trunk_page_id as usize)?; + let (trunk_page, _c) = cursor.read_page(trunk_page_id as usize)?; if let Some(contents) = trunk_page.get().get().contents.as_ref() { // Read number of leaf pages in trunk let n_leaf = contents.read_u32(4); @@ -8436,23 +8485,33 @@ mod tests { payload_size: small_payload.len() as u64, }); - let initial_freelist_pages = header_accessor::get_freelist_pages(&pager)?; + let initial_freelist_pages = pager + .io + .block(|| pager.with_header(|header| header.freelist_pages))? + .get() as usize; // Try to clear non-existent overflow pages let clear_result = cursor.clear_overflow_pages(&leaf_cell)?; match clear_result { IOResult::Done(_) => { + let (freelist_pages, freelist_trunk_page) = pager.io.block(|| { + pager.with_header(|header| { + ( + header.freelist_pages.get(), + header.freelist_trunk_page.get(), + ) + }) + })?; + // Verify freelist was not modified assert_eq!( - header_accessor::get_freelist_pages(&pager)?, - initial_freelist_pages, + freelist_pages as usize, initial_freelist_pages, "Freelist should not change when no overflow pages exist" ); // Verify trunk page wasn't created assert_eq!( - header_accessor::get_freelist_trunk_page(&pager)?, - 0, + freelist_trunk_page, 0, "No trunk page should be created when no overflow pages exist" ); } @@ -8532,18 +8591,28 @@ mod tests { // Verify structure before destruction assert_eq!( - header_accessor::get_database_size(&pager)?, + pager + .io + .block(|| pager.with_header(|header| header.database_size))? + .get(), 4, // We should have pages 1-4 "Database should have 4 pages total" ); // Track freelist state before destruction - let initial_free_pages = header_accessor::get_freelist_pages(&pager)?; + let initial_free_pages = pager + .io + .block(|| pager.with_header(|header| header.freelist_pages))? + .get(); assert_eq!(initial_free_pages, 0, "should start with no free pages"); run_until_done(|| cursor.btree_destroy(), pager.deref())?; - let pages_freed = header_accessor::get_freelist_pages(&pager)? - initial_free_pages; + let pages_freed = pager + .io + .block(|| pager.with_header(|header| header.freelist_pages))? + .get() + - initial_free_pages; assert_eq!(pages_freed, 3, "should free 3 pages (root + 2 leaves)"); Ok(()) @@ -9356,7 +9425,7 @@ mod tests { ); } let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page, num_columns); - cursor.move_to_root().unwrap(); + let _c = cursor.move_to_root().unwrap(); for i in 0..iterations { let has_next = run_until_done(|| cursor.next(), pager.deref()).unwrap(); if !has_next { diff --git a/core/storage/buffer_pool.rs b/core/storage/buffer_pool.rs index 1ded1bca4..4d7beae2b 100644 --- a/core/storage/buffer_pool.rs +++ b/core/storage/buffer_pool.rs @@ -3,18 +3,18 @@ use parking_lot::Mutex; use std::pin::Pin; use std::sync::atomic::{AtomicUsize, Ordering}; +use super::sqlite3_ondisk::PageSize; + pub struct BufferPool { pub free_buffers: Mutex>, page_size: AtomicUsize, } -const DEFAULT_PAGE_SIZE: usize = 4096; - impl BufferPool { pub fn new(page_size: Option) -> Self { Self { free_buffers: Mutex::new(Vec::new()), - page_size: AtomicUsize::new(page_size.unwrap_or(DEFAULT_PAGE_SIZE)), + page_size: AtomicUsize::new(page_size.unwrap_or(PageSize::DEFAULT as usize)), } } diff --git a/core/storage/database.rs b/core/storage/database.rs index fd2555b59..671e4a0da 100644 --- a/core/storage/database.rs +++ b/core/storage/database.rs @@ -16,6 +16,13 @@ pub trait DatabaseStorage: Send + Sync { buffer: Arc>, c: Completion, ) -> Result; + fn write_pages( + &self, + first_page_idx: usize, + page_size: usize, + buffers: Vec>>, + c: Completion, + ) -> Result; fn sync(&self, c: Completion) -> Result; fn size(&self) -> Result; fn truncate(&self, len: usize, c: Completion) -> Result; @@ -61,6 +68,22 @@ impl DatabaseStorage for DatabaseFile { self.file.pwrite(pos, buffer, c) } + fn write_pages( + &self, + page_idx: usize, + page_size: usize, + buffers: Vec>>, + c: Completion, + ) -> Result { + assert!(page_idx > 0); + assert!(page_size >= 512); + assert!(page_size <= 65536); + assert_eq!(page_size & (page_size - 1), 0); + let pos = (page_idx - 1) * page_size; + let c = self.file.pwritev(pos, buffers, c)?; + Ok(c) + } + #[instrument(skip_all, level = Level::DEBUG)] fn sync(&self, c: Completion) -> Result { self.file.sync(c) @@ -84,61 +107,3 @@ impl DatabaseFile { Self { file } } } - -pub struct FileMemoryStorage { - file: Arc, -} - -unsafe impl Send for FileMemoryStorage {} -unsafe impl Sync for FileMemoryStorage {} - -impl DatabaseStorage for FileMemoryStorage { - #[instrument(skip_all, level = Level::DEBUG)] - fn read_page(&self, page_idx: usize, c: Completion) -> Result { - let r = c.as_read(); - let size = r.buf().len(); - assert!(page_idx > 0); - if !(512..=65536).contains(&size) || size & (size - 1) != 0 { - return Err(LimboError::NotADB); - } - let pos = (page_idx - 1) * size; - self.file.pread(pos, c) - } - - #[instrument(skip_all, level = Level::DEBUG)] - fn write_page( - &self, - page_idx: usize, - buffer: Arc>, - c: Completion, - ) -> Result { - let buffer_size = buffer.borrow().len(); - assert!(buffer_size >= 512); - assert!(buffer_size <= 65536); - assert_eq!(buffer_size & (buffer_size - 1), 0); - let pos = (page_idx - 1) * buffer_size; - self.file.pwrite(pos, buffer, c) - } - - #[instrument(skip_all, level = Level::DEBUG)] - fn sync(&self, c: Completion) -> Result { - self.file.sync(c) - } - - #[instrument(skip_all, level = Level::DEBUG)] - fn size(&self) -> Result { - self.file.size() - } - - #[instrument(skip_all, level = Level::INFO)] - fn truncate(&self, len: usize, c: Completion) -> Result { - let c = self.file.truncate(len, c)?; - Ok(c) - } -} - -impl FileMemoryStorage { - pub fn new(file: Arc) -> Self { - Self { file } - } -} diff --git a/core/storage/header_accessor.rs b/core/storage/header_accessor.rs index 592d4f570..2fc2e7778 100644 --- a/core/storage/header_accessor.rs +++ b/core/storage/header_accessor.rs @@ -1,230 +1,75 @@ -use crate::storage::sqlite3_ondisk::MAX_PAGE_SIZE; +use super::sqlite3_ondisk::{DatabaseHeader, PageContent}; use crate::turso_assert; use crate::{ - storage::{ - self, - pager::{PageRef, Pager}, - sqlite3_ondisk::DATABASE_HEADER_PAGE_ID, - }, + storage::pager::{PageRef, Pager}, types::IOResult, LimboError, Result, }; +use std::cell::{Ref, RefMut}; -// const HEADER_OFFSET_MAGIC: usize = 0; -const HEADER_OFFSET_PAGE_SIZE: usize = 16; -const HEADER_OFFSET_WRITE_VERSION: usize = 18; -const HEADER_OFFSET_READ_VERSION: usize = 19; -const HEADER_OFFSET_RESERVED_SPACE: usize = 20; -const HEADER_OFFSET_MAX_EMBED_FRAC: usize = 21; -const HEADER_OFFSET_MIN_EMBED_FRAC: usize = 22; -const HEADER_OFFSET_MIN_LEAF_FRAC: usize = 23; -const HEADER_OFFSET_CHANGE_COUNTER: usize = 24; -const HEADER_OFFSET_DATABASE_SIZE: usize = 28; -const HEADER_OFFSET_FREELIST_TRUNK_PAGE: usize = 32; -const HEADER_OFFSET_FREELIST_PAGES: usize = 36; -const HEADER_OFFSET_SCHEMA_COOKIE: usize = 40; -const HEADER_OFFSET_SCHEMA_FORMAT: usize = 44; -const HEADER_OFFSET_DEFAULT_PAGE_CACHE_SIZE: usize = 48; -const HEADER_OFFSET_VACUUM_MODE_LARGEST_ROOT_PAGE: usize = 52; -const HEADER_OFFSET_TEXT_ENCODING: usize = 56; -const HEADER_OFFSET_USER_VERSION: usize = 60; -const HEADER_OFFSET_INCREMENTAL_VACUUM_ENABLED: usize = 64; -const HEADER_OFFSET_APPLICATION_ID: usize = 68; -//const HEADER_OFFSET_RESERVED_FOR_EXPANSION: usize = 72; -const HEADER_OFFSET_VERSION_VALID_FOR: usize = 92; -const HEADER_OFFSET_VERSION_NUMBER: usize = 96; +pub struct HeaderRef(PageRef); -// Helper to get a read-only reference to the header page. -fn get_header_page(pager: &Pager) -> Result> { - if !pager.db_state.is_initialized() { - return Err(LimboError::InternalError( - "Database is empty, header does not exist - page 1 should've been allocated before this".to_string(), - )); - } - let (page, c) = pager.read_page(DATABASE_HEADER_PAGE_ID)?; - if page.is_locked() { - return Ok(IOResult::IO); - } - Ok(IOResult::Done(page)) -} - -// Helper to get a writable reference to the header page and mark it dirty. -fn get_header_page_for_write(pager: &Pager) -> Result> { - if !pager.db_state.is_initialized() { - // This should not be called on an empty DB for writing, as page 1 is allocated on first transaction. - return Err(LimboError::InternalError( - "Cannot write to header of an empty database - page 1 should've been allocated before this".to_string(), - )); - } - let (page, c) = pager.read_page(DATABASE_HEADER_PAGE_ID)?; - if page.is_locked() { - return Ok(IOResult::IO); - } - turso_assert!( - page.get().id == DATABASE_HEADER_PAGE_ID, - "page must have number 1" - ); - pager.add_dirty(&page); - Ok(IOResult::Done(page)) -} - -/// Helper function to run async header accessors until completion -fn run_header_accessor_until_done(pager: &Pager, mut accessor: F) -> Result -where - F: FnMut() -> Result>, -{ - loop { - match accessor()? { - IOResult::Done(value) => return Ok(value), - IOResult::IO => { - pager.io.run_once()?; - } +impl HeaderRef { + pub fn from_pager(pager: &Pager) -> Result> { + if !pager.db_state.is_initialized() { + return Err(LimboError::InternalError( + "Database is empty, header does not exist - page 1 should've been allocated before this".to_string() + )); } - } -} -/// Helper macro to implement getters and setters for header fields. -/// For example, `impl_header_field_accessor!(page_size, u16, HEADER_OFFSET_PAGE_SIZE);` -/// will generate the following functions: -/// - `pub fn get_page_size(pager: &Pager) -> Result` (sync) -/// - `pub fn get_page_size_async(pager: &Pager) -> Result>` (async) -/// - `pub fn set_page_size(pager: &Pager, value: u16) -> Result<()>` (sync) -/// - `pub fn set_page_size_async(pager: &Pager, value: u16) -> Result>` (async) -/// -/// The macro takes three required arguments: -/// - `$field_name`: The name of the field to implement. -/// - `$type`: The type of the field. -/// - `$offset`: The offset of the field in the header page. -/// -/// And a fourth optional argument: -/// - `$ifzero`: A value to return if the field is 0. -/// -/// The macro will generate both sync and async versions of the functions. -/// -macro_rules! impl_header_field_accessor { - ($field_name:ident, $type:ty, $offset:expr $(, $ifzero:expr)?) => { - paste::paste! { - // Async version - #[allow(dead_code)] - pub fn [](pager: &Pager) -> Result> { - if !pager.db_state.is_initialized() { - return Err(LimboError::InternalError(format!("Database is empty, header does not exist - page 1 should've been allocated before this"))); - } - let page = match get_header_page(pager)? { - IOResult::Done(page) => page, - IOResult::IO => return Ok(IOResult::IO), - }; - let page_inner = page.get(); - let page_content = page_inner.contents.as_ref().unwrap(); - let buf = page_content.buffer.borrow(); - let buf_slice = buf.as_slice(); - let mut bytes = [0; std::mem::size_of::<$type>()]; - bytes.copy_from_slice(&buf_slice[$offset..$offset + std::mem::size_of::<$type>()]); - let value = <$type>::from_be_bytes(bytes); - $( - if value == 0 { - return Ok(IOResult::Done($ifzero)); - } - )? - Ok(IOResult::Done(value)) - } - - // Sync version - #[allow(dead_code)] - pub fn [](pager: &Pager) -> Result<$type> { - run_header_accessor_until_done(pager, || [](pager)) - } - - // Async setter - #[allow(dead_code)] - pub fn [](pager: &Pager, value: $type) -> Result> { - let page = match get_header_page_for_write(pager)? { - IOResult::Done(page) => page, - IOResult::IO => return Ok(IOResult::IO), - }; - let page_inner = page.get(); - let page_content = page_inner.contents.as_ref().unwrap(); - let mut buf = page_content.buffer.borrow_mut(); - let buf_slice = buf.as_mut_slice(); - buf_slice[$offset..$offset + std::mem::size_of::<$type>()].copy_from_slice(&value.to_be_bytes()); - turso_assert!(page.get().id == 1, "page must have number 1"); - pager.add_dirty(&page); - Ok(IOResult::Done(())) - } - - // Sync setter - #[allow(dead_code)] - pub fn [](pager: &Pager, value: $type) -> Result<()> { - run_header_accessor_until_done(pager, || [](pager, value)) - } + let (page, _c) = pager.read_page(DatabaseHeader::PAGE_ID)?; + if page.is_locked() { + return Ok(IOResult::IO); } - }; -} -// impl_header_field_accessor!(magic, [u8; 16], HEADER_OFFSET_MAGIC); -impl_header_field_accessor!(page_size_u16, u16, HEADER_OFFSET_PAGE_SIZE); -impl_header_field_accessor!(write_version, u8, HEADER_OFFSET_WRITE_VERSION); -impl_header_field_accessor!(read_version, u8, HEADER_OFFSET_READ_VERSION); -impl_header_field_accessor!(reserved_space, u8, HEADER_OFFSET_RESERVED_SPACE); -impl_header_field_accessor!(max_embed_frac, u8, HEADER_OFFSET_MAX_EMBED_FRAC); -impl_header_field_accessor!(min_embed_frac, u8, HEADER_OFFSET_MIN_EMBED_FRAC); -impl_header_field_accessor!(min_leaf_frac, u8, HEADER_OFFSET_MIN_LEAF_FRAC); -impl_header_field_accessor!(change_counter, u32, HEADER_OFFSET_CHANGE_COUNTER); -impl_header_field_accessor!(database_size, u32, HEADER_OFFSET_DATABASE_SIZE); -impl_header_field_accessor!(freelist_trunk_page, u32, HEADER_OFFSET_FREELIST_TRUNK_PAGE); -impl_header_field_accessor!(freelist_pages, u32, HEADER_OFFSET_FREELIST_PAGES); -impl_header_field_accessor!(schema_cookie, u32, HEADER_OFFSET_SCHEMA_COOKIE); -impl_header_field_accessor!(schema_format, u32, HEADER_OFFSET_SCHEMA_FORMAT); -impl_header_field_accessor!( - default_page_cache_size, - i32, - HEADER_OFFSET_DEFAULT_PAGE_CACHE_SIZE, - storage::sqlite3_ondisk::DEFAULT_CACHE_SIZE -); -impl_header_field_accessor!( - vacuum_mode_largest_root_page, - u32, - HEADER_OFFSET_VACUUM_MODE_LARGEST_ROOT_PAGE -); -impl_header_field_accessor!(text_encoding, u32, HEADER_OFFSET_TEXT_ENCODING); -impl_header_field_accessor!(user_version, i32, HEADER_OFFSET_USER_VERSION); -impl_header_field_accessor!( - incremental_vacuum_enabled, - u32, - HEADER_OFFSET_INCREMENTAL_VACUUM_ENABLED -); -impl_header_field_accessor!(application_id, i32, HEADER_OFFSET_APPLICATION_ID); -//impl_header_field_accessor!(reserved_for_expansion, [u8; 20], HEADER_OFFSET_RESERVED_FOR_EXPANSION); -impl_header_field_accessor!(version_valid_for, u32, HEADER_OFFSET_VERSION_VALID_FOR); -impl_header_field_accessor!(version_number, u32, HEADER_OFFSET_VERSION_NUMBER); + turso_assert!( + page.get().id == DatabaseHeader::PAGE_ID, + "incorrect header page id" + ); -pub fn get_page_size(pager: &Pager) -> Result { - let size = get_page_size_u16(pager)?; - if size == 1 { - return Ok(MAX_PAGE_SIZE); + Ok(IOResult::Done(Self(page))) + } + + pub fn borrow(&self) -> Ref<'_, DatabaseHeader> { + // TODO: Instead of erasing mutability, implement `get_mut_contents` and return a shared reference. + let content: &PageContent = self.0.get_contents(); + Ref::map(content.buffer.borrow(), |buffer| { + bytemuck::from_bytes::(&buffer.as_slice()[0..DatabaseHeader::SIZE]) + }) } - Ok(size as u32) } -#[allow(dead_code)] -pub fn set_page_size(pager: &Pager, value: u32) -> Result<()> { - let page_size = if value == MAX_PAGE_SIZE { - 1 - } else { - value as u16 - }; - set_page_size_u16(pager, page_size) -} +pub struct HeaderRefMut(PageRef); -#[allow(dead_code)] -pub fn get_page_size_async(pager: &Pager) -> Result> { - match get_page_size_u16_async(pager)? { - IOResult::Done(size) => { - if size == 1 { - return Ok(IOResult::Done(MAX_PAGE_SIZE)); - } - Ok(IOResult::Done(size as u32)) +impl HeaderRefMut { + pub fn from_pager(pager: &Pager) -> Result> { + if !pager.db_state.is_initialized() { + return Err(LimboError::InternalError( + "Database is empty, header does not exist - page 1 should've been allocated before this".to_string(), + )); } - IOResult::IO => Ok(IOResult::IO), + + let (page, _c) = pager.read_page(DatabaseHeader::PAGE_ID)?; + if page.is_locked() { + return Ok(IOResult::IO); + } + + turso_assert!( + page.get().id == DatabaseHeader::PAGE_ID, + "incorrect header page id" + ); + + pager.add_dirty(&page); + + Ok(IOResult::Done(Self(page))) + } + + pub fn borrow_mut(&self) -> RefMut<'_, DatabaseHeader> { + let content = self.0.get_contents(); + RefMut::map(content.buffer.borrow_mut(), |buffer| { + bytemuck::from_bytes_mut::( + &mut buffer.as_mut_slice()[0..DatabaseHeader::SIZE], + ) + }) } } diff --git a/core/storage/mod.rs b/core/storage/mod.rs index a3f396287..c62a2a9df 100644 --- a/core/storage/mod.rs +++ b/core/storage/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod page_cache; #[allow(clippy::arc_with_non_send_sync)] pub(crate) mod pager; pub(crate) mod sqlite3_ondisk; +mod state_machines; #[allow(clippy::arc_with_non_send_sync)] pub(crate) mod wal; diff --git a/core/storage/pager.rs b/core/storage/pager.rs index a31094f19..15c7ff733 100644 --- a/core/storage/pager.rs +++ b/core/storage/pager.rs @@ -2,9 +2,8 @@ use crate::result::LimboResult; use crate::storage::btree::BTreePageInner; use crate::storage::buffer_pool::BufferPool; use crate::storage::database::DatabaseStorage; -use crate::storage::header_accessor; use crate::storage::sqlite3_ondisk::{ - self, parse_wal_frame_header, DatabaseHeader, PageContent, PageType, DEFAULT_PAGE_SIZE, + self, parse_wal_frame_header, DatabaseHeader, PageContent, PageSize, PageType, }; use crate::storage::wal::{CheckpointResult, Wal}; use crate::types::{IOResult, WalInsertInfo}; @@ -21,8 +20,9 @@ use std::sync::{Arc, Mutex}; use tracing::{instrument, trace, Level}; use super::btree::{btree_init_page, BTreePage}; +use super::header_accessor::{HeaderRef, HeaderRefMut}; use super::page_cache::{CacheError, CacheResizeResult, DumbLruPageCache, PageCacheKey}; -use super::sqlite3_ondisk::{begin_write_btree_page, DATABASE_HEADER_SIZE}; +use super::sqlite3_ondisk::begin_write_btree_page; use super::wal::CheckpointMode; #[cfg(not(feature = "omit_autovacuum"))] @@ -318,7 +318,8 @@ pub struct Pager { /// Source of the database pages. pub db_file: Arc, /// The write-ahead log (WAL) for the database. - pub(crate) wal: Rc>, + /// in-memory databases, ephemeral tables and ephemeral indexes do not have a WAL. + pub(crate) wal: Option>>, /// A page cache for the database. page_cache: Arc>, /// Buffer pool for temporary data storage. @@ -346,7 +347,7 @@ pub struct Pager { /// Cache page_size and reserved_space at Pager init and reuse for subsequent /// `usable_space` calls. TODO: Invalidate reserved_space when we add the functionality /// to change it. - page_size: Cell>, + pub(crate) page_size: Cell>, reserved_space: OnceCell, free_page_state: RefCell, } @@ -410,7 +411,7 @@ enum FreePageState { impl Pager { pub fn new( db_file: Arc, - wal: Rc>, + wal: Option>>, io: Arc, page_cache: Arc>, buffer_pool: Arc, @@ -456,7 +457,7 @@ impl Pager { } pub fn set_wal(&mut self, wal: Rc>) { - self.wal = wal; + self.wal = Some(wal); } pub fn get_auto_vacuum_mode(&self) -> AutoVacuumMode { @@ -473,10 +474,8 @@ impl Pager { #[cfg(not(feature = "omit_autovacuum"))] pub fn ptrmap_get(&self, target_page_num: u32) -> Result>> { tracing::trace!("ptrmap_get(page_idx = {})", target_page_num); - let configured_page_size = match header_accessor::get_page_size_async(self)? { - IOResult::Done(size) => size as usize, - IOResult::IO => return Ok(IOResult::IO), - }; + let configured_page_size = + return_if_io!(self.with_header(|header| header.page_size)).get() as usize; if target_page_num < FIRST_PTRMAP_PAGE_NO || is_ptrmap_page(target_page_num, configured_page_size) @@ -493,7 +492,7 @@ impl Pager { ptrmap_pg_no ); - let (ptrmap_page, c) = self.read_page(ptrmap_pg_no as usize)?; + let (ptrmap_page, _c) = self.read_page(ptrmap_pg_no as usize)?; if ptrmap_page.is_locked() { return Ok(IOResult::IO); } @@ -559,10 +558,7 @@ impl Pager { parent_page_no ); - let page_size = match header_accessor::get_page_size_async(self)? { - IOResult::Done(size) => size as usize, - IOResult::IO => return Ok(IOResult::IO), - }; + let page_size = return_if_io!(self.with_header(|header| header.page_size)).get() as usize; if db_page_no_to_update < FIRST_PTRMAP_PAGE_NO || is_ptrmap_page(db_page_no_to_update, page_size) @@ -584,7 +580,7 @@ impl Pager { offset_in_ptrmap_page ); - let (ptrmap_page, c) = self.read_page(ptrmap_pg_no as usize)?; + let (ptrmap_page, _c) = self.read_page(ptrmap_pg_no as usize)?; if ptrmap_page.is_locked() { return Ok(IOResult::IO); } @@ -658,21 +654,19 @@ impl Pager { Ok(IOResult::Done(page.get().get().id as u32)) } AutoVacuumMode::Full => { - let mut root_page_num = - match header_accessor::get_vacuum_mode_largest_root_page_async(self)? { - IOResult::Done(value) => value, - IOResult::IO => return Ok(IOResult::IO), - }; + let (mut root_page_num, page_size) = + return_if_io!(self.with_header(|header| { + ( + header.vacuum_mode_largest_root_page.get(), + header.page_size.get(), + ) + })); + assert!(root_page_num > 0); // Largest root page number cannot be 0 because that is set to 1 when creating the database with autovacuum enabled root_page_num += 1; assert!(root_page_num >= FIRST_PTRMAP_PAGE_NO); // can never be less than 2 because we have already incremented - let page_size = match header_accessor::get_page_size_async(self)? { - IOResult::Done(size) => size as usize, - IOResult::IO => return Ok(IOResult::IO), - }; - - while is_ptrmap_page(root_page_num, page_size) { + while is_ptrmap_page(root_page_num, page_size as usize) { root_page_num += 1; } assert!(root_page_num >= 3); // the very first root page is page 3 @@ -745,14 +739,18 @@ impl Pager { /// The usable size of a page might be an odd number. However, the usable size is not allowed to be less than 480. /// In other words, if the page size is 512, then the reserved space size cannot exceed 32. pub fn usable_space(&self) -> usize { - let page_size = *self - .page_size - .get() - .get_or_insert_with(|| header_accessor::get_page_size(self).unwrap()); + let page_size = *self.page_size.get().get_or_insert_with(|| { + self.io + .block(|| self.with_header(|header| header.page_size)) + .unwrap_or_default() + .get() + }); - let reserved_space = *self - .reserved_space - .get_or_init(|| header_accessor::get_reserved_space(self).unwrap()); + let reserved_space = *self.reserved_space.get_or_init(|| { + self.io + .block(|| self.with_header(|header| header.reserved_space)) + .unwrap_or_default() + }); (page_size as usize) - (reserved_space as usize) } @@ -766,7 +764,10 @@ impl Pager { #[inline(always)] #[instrument(skip_all, level = Level::DEBUG)] pub fn begin_read_tx(&self) -> Result { - let (result, changed) = self.wal.borrow_mut().begin_read_tx()?; + let Some(wal) = self.wal.as_ref() else { + return Ok(LimboResult::Ok); + }; + let (result, changed) = wal.borrow_mut().begin_read_tx()?; if changed { // Someone else changed the database -> assume our page cache is invalid (this is default SQLite behavior, we can probably do better with more granular invalidation) self.clear_page_cache(); @@ -805,7 +806,10 @@ impl Pager { IOResult::Done(_) => {} IOResult::IO => return Ok(IOResult::IO), } - Ok(IOResult::Done(self.wal.borrow_mut().begin_write_tx()?)) + let Some(wal) = self.wal.as_ref() else { + return Ok(IOResult::Done(LimboResult::Ok)); + }; + Ok(IOResult::Done(wal.borrow_mut().begin_write_tx()?)) } #[instrument(skip_all, level = Level::DEBUG)] @@ -817,23 +821,28 @@ impl Pager { wal_checkpoint_disabled: bool, ) -> Result> { tracing::trace!("end_tx(rollback={})", rollback); + let Some(wal) = self.wal.as_ref() else { + // TODO: Unsure what the semantics of "end_tx" is for in-memory databases, ephemeral tables and ephemeral indexes. + return Ok(IOResult::Done(PagerCommitResult::Rollback)); + }; if rollback { - if matches!( + let is_write = matches!( connection.transaction_state.get(), TransactionState::Write { .. } - ) { - self.wal.borrow().end_write_tx(); + ); + if is_write { + wal.borrow().end_write_tx(); } - self.wal.borrow().end_read_tx(); - self.rollback(schema_did_change, connection)?; + wal.borrow().end_read_tx(); + self.rollback(schema_did_change, connection, is_write)?; return Ok(IOResult::Done(PagerCommitResult::Rollback)); } let commit_status = self.commit_dirty_pages(wal_checkpoint_disabled)?; match commit_status { IOResult::IO => Ok(IOResult::IO), IOResult::Done(_) => { - self.wal.borrow().end_write_tx(); - self.wal.borrow().end_read_tx(); + wal.borrow().end_write_tx(); + wal.borrow().end_read_tx(); if schema_did_change { let schema = connection.schema.borrow().clone(); @@ -846,7 +855,10 @@ impl Pager { #[instrument(skip_all, level = Level::DEBUG)] pub fn end_read_tx(&self) -> Result<()> { - self.wal.borrow().end_read_tx(); + let Some(wal) = self.wal.as_ref() else { + return Ok(()); + }; + wal.borrow().end_read_tx(); Ok(()) } @@ -864,35 +876,46 @@ impl Pager { let page = Arc::new(Page::new(page_idx)); page.set_locked(); - if let Some(frame_id) = self.wal.borrow().find_frame(page_idx as u64)? { - let c = - self.wal - .borrow() - .read_frame(frame_id, page.clone(), self.buffer_pool.clone())?; - page.set_uptodate(); + let Some(wal) = self.wal.as_ref() else { + let c = self.begin_read_disk_page(page_idx, page.clone())?; + self.cache_insert(page_idx, page.clone(), &mut page_cache)?; + return Ok((page, c)); + }; + + if let Some(frame_id) = wal.borrow().find_frame(page_idx as u64)? { + let c = wal + .borrow() + .read_frame(frame_id, page.clone(), self.buffer_pool.clone())?; + { + page.set_uptodate(); + } // TODO(pere) should probably first insert to page cache, and if successful, // read frame or page - match page_cache.insert(page_key, page.clone()) { - Ok(_) => {} - Err(CacheError::Full) => return Err(LimboError::CacheFull), - Err(CacheError::KeyExists) => { - unreachable!("Page should not exist in cache after get() miss") - } - Err(e) => { - return Err(LimboError::InternalError(format!( - "Failed to insert page into cache: {e:?}" - ))) - } - } + self.cache_insert(page_idx, page.clone(), &mut page_cache)?; return Ok((page, c)); } - let c = sqlite3_ondisk::begin_read_page( + let c = self.begin_read_disk_page(page_idx, page.clone())?; + self.cache_insert(page_idx, page.clone(), &mut page_cache)?; + Ok((page, c)) + } + + fn begin_read_disk_page(&self, page_idx: usize, page: PageRef) -> Result { + sqlite3_ondisk::begin_read_page( self.db_file.clone(), self.buffer_pool.clone(), - page.clone(), + page, page_idx, - )?; + ) + } + + fn cache_insert( + &self, + page_idx: usize, + page: PageRef, + page_cache: &mut DumbLruPageCache, + ) -> Result<()> { + let page_key = PageCacheKey::new(page_idx); match page_cache.insert(page_key, page.clone()) { Ok(_) => {} Err(CacheError::Full) => return Err(LimboError::CacheFull), @@ -905,7 +928,7 @@ impl Pager { ))) } } - Ok((page, c)) + Ok(()) } // Get a page from the cache, if it exists. @@ -930,13 +953,25 @@ impl Pager { } pub fn wal_frame_count(&self) -> Result { - Ok(self.wal.borrow().get_max_frame_in_wal()) + let Some(wal) = self.wal.as_ref() else { + return Err(LimboError::InternalError( + "wal_frame_count() called on database without WAL".to_string(), + )); + }; + Ok(wal.borrow().get_max_frame_in_wal()) } /// Flush all dirty pages to disk. /// Unlike commit_dirty_pages, this function does not commit, checkpoint now sync the WAL/Database. #[instrument(skip_all, level = Level::INFO)] pub fn cacheflush(&self) -> Result> { + let Some(wal) = self.wal.as_ref() else { + // TODO: when ephemeral table spills to disk, it should cacheflush pages directly to the temporary database file. + // This handling is not yet implemented, but it should be when spilling is implemented. + return Err(LimboError::InternalError( + "cacheflush() called on database without WAL".to_string(), + )); + }; let state = self.flush_info.borrow().state; trace!(?state); match state { @@ -975,7 +1010,7 @@ impl Pager { page }; - let c = self.wal.borrow_mut().append_frame( + let _c = wal.borrow_mut().append_frame( page.clone(), 0, self.flush_info.borrow().in_flight_writes.clone(), @@ -1034,6 +1069,11 @@ impl Pager { &self, wal_checkpoint_disabled: bool, ) -> Result> { + let Some(wal) = self.wal.as_ref() else { + return Err(LimboError::InternalError( + "commit_dirty_pages() called on database without WAL".to_string(), + )); + }; let mut checkpoint_result = CheckpointResult::default(); let res = loop { let state = self.commit_info.borrow().state; @@ -1080,14 +1120,17 @@ impl Pager { }; let db_size = { - let db_size = header_accessor::get_database_size(self)?; + let db_size = self + .io + .block(|| self.with_header(|header| header.database_size))? + .get(); if is_last_frame { db_size } else { 0 } }; - let c = self.wal.borrow_mut().append_frame( + let _c = wal.borrow_mut().append_frame( page.clone(), db_size, self.commit_info.borrow().in_flight_writes.clone(), @@ -1141,9 +1184,9 @@ impl Pager { } } CommitState::SyncWal => { - return_if_io!(self.wal.borrow_mut().sync()); + return_if_io!(wal.borrow_mut().sync()); - if wal_checkpoint_disabled || !self.wal.borrow().should_checkpoint() { + if wal_checkpoint_disabled || !wal.borrow().should_checkpoint() { self.commit_info.borrow_mut().state = CommitState::Start; break PagerCommitResult::WalWritten; } @@ -1154,7 +1197,8 @@ impl Pager { self.commit_info.borrow_mut().state = CommitState::SyncDbFile; } CommitState::SyncDbFile => { - let c = sqlite3_ondisk::begin_sync(self.db_file.clone(), self.syncing.clone())?; + let _c = + sqlite3_ondisk::begin_sync(self.db_file.clone(), self.syncing.clone())?; self.commit_info.borrow_mut().state = CommitState::WaitSyncDbFile; } CommitState::WaitSyncDbFile => { @@ -1168,19 +1212,29 @@ impl Pager { } }; // We should only signal that we finished appenind frames after wal sync to avoid inconsistencies when sync fails - self.wal.borrow_mut().finish_append_frames_commit()?; + wal.borrow_mut().finish_append_frames_commit()?; Ok(IOResult::Done(res)) } #[instrument(skip_all, level = Level::DEBUG)] pub fn wal_get_frame(&self, frame_no: u32, frame: &mut [u8]) -> Result { - let wal = self.wal.borrow(); + let Some(wal) = self.wal.as_ref() else { + return Err(LimboError::InternalError( + "wal_get_frame() called on database without WAL".to_string(), + )); + }; + let wal = wal.borrow(); wal.read_frame_raw(frame_no.into(), frame) } #[instrument(skip_all, level = Level::DEBUG)] pub fn wal_insert_frame(&self, frame_no: u32, frame: &[u8]) -> Result { - let mut wal = self.wal.borrow_mut(); + let Some(wal) = self.wal.as_ref() else { + return Err(LimboError::InternalError( + "wal_insert_frame() called on database without WAL".to_string(), + )); + }; + let mut wal = wal.borrow_mut(); let (header, raw_page) = parse_wal_frame_header(frame); wal.write_frame_raw( self.buffer_pool.clone(), @@ -1215,6 +1269,11 @@ impl Pager { #[instrument(skip_all, level = Level::DEBUG, name = "pager_checkpoint",)] pub fn checkpoint(&self) -> Result> { + let Some(wal) = self.wal.as_ref() else { + return Err(LimboError::InternalError( + "checkpoint() called on database without WAL".to_string(), + )); + }; let mut checkpoint_result = CheckpointResult::default(); loop { let state = *self.checkpoint_state.borrow(); @@ -1222,11 +1281,10 @@ impl Pager { match state { CheckpointState::Checkpoint => { let in_flight = self.checkpoint_inflight.clone(); - match self.wal.borrow_mut().checkpoint( - self, - in_flight, - CheckpointMode::Passive, - )? { + match wal + .borrow_mut() + .checkpoint(self, in_flight, CheckpointMode::Passive)? + { IOResult::IO => return Ok(IOResult::IO), IOResult::Done(res) => { checkpoint_result = res; @@ -1235,7 +1293,8 @@ impl Pager { }; } CheckpointState::SyncDbFile => { - let c = sqlite3_ondisk::begin_sync(self.db_file.clone(), self.syncing.clone())?; + let _c = + sqlite3_ondisk::begin_sync(self.db_file.clone(), self.syncing.clone())?; self.checkpoint_state .replace(CheckpointState::WaitSyncDbFile); } @@ -1274,7 +1333,12 @@ impl Pager { pub fn checkpoint_shutdown(&self, wal_checkpoint_disabled: bool) -> Result<()> { let mut _attempts = 0; { - let mut wal = self.wal.borrow_mut(); + let Some(wal) = self.wal.as_ref() else { + return Err(LimboError::InternalError( + "checkpoint_shutdown() called on database without WAL".to_string(), + )); + }; + let mut wal = wal.borrow_mut(); // fsync the wal syncronously before beginning checkpoint while let Ok(IOResult::IO) = wal.sync() { // TODO: for now forget about timeouts as they fail regularly in SIM @@ -1299,22 +1363,29 @@ impl Pager { wal_checkpoint_disabled: bool, mode: CheckpointMode, ) -> Result { + let Some(wal) = self.wal.as_ref() else { + return Err(LimboError::InternalError( + "wal_checkpoint() called on database without WAL".to_string(), + )); + }; if wal_checkpoint_disabled { return Ok(CheckpointResult::default()); } - let counter = Rc::new(RefCell::new(0)); + let write_counter = Rc::new(RefCell::new(0)); let mut checkpoint_result = self.io.block(|| { - self.wal - .borrow_mut() - .checkpoint(self, counter.clone(), mode) + wal.borrow_mut() + .checkpoint(self, write_counter.clone(), mode) })?; if checkpoint_result.everything_backfilled() && checkpoint_result.num_checkpointed_frames != 0 { - let db_size = header_accessor::get_database_size(self)?; - let page_size = self.page_size.get().unwrap_or(DEFAULT_PAGE_SIZE); + let db_size = self + .io + .block(|| self.with_header(|header| header.database_size))? + .get(); + let page_size = self.page_size.get().unwrap_or(PageSize::DEFAULT as u32); let expected = (db_size * page_size) as u64; if expected < self.db_file.size()? { self.io.wait_for_completion(self.db_file.truncate( @@ -1354,18 +1425,21 @@ impl Pager { const TRUNK_PAGE_NEXT_PAGE_OFFSET: usize = 0; // Offset to next trunk page pointer const TRUNK_PAGE_LEAF_COUNT_OFFSET: usize = 4; // Offset to leaf count + let header_ref = self.io.block(|| HeaderRefMut::from_pager(self))?; + let mut header = header_ref.borrow_mut(); + let mut state = self.free_page_state.borrow_mut(); tracing::debug!(?state); loop { match &mut *state { FreePageState::Start => { - if page_id < 2 || page_id > header_accessor::get_database_size(self)? as usize { + if page_id < 2 || page_id > header.database_size.get() as usize { return Err(LimboError::Corrupt(format!( "Invalid page number {page_id} for free operation" ))); } - let (page, c) = match page.clone() { + let (page, _c) = match page.clone() { Some(page) => { assert_eq!( page.get().id, @@ -1385,12 +1459,9 @@ impl Pager { (page, Some(c)) } }; - header_accessor::set_freelist_pages( - self, - header_accessor::get_freelist_pages(self)? + 1, - )?; + header.freelist_pages = (header.freelist_pages.get() + 1).into(); - let trunk_page_id = header_accessor::get_freelist_trunk_page(self)?; + let trunk_page_id = header.freelist_trunk_page.get(); if trunk_page_id != 0 { *state = FreePageState::AddToTrunk { @@ -1402,10 +1473,10 @@ impl Pager { } } FreePageState::AddToTrunk { page, trunk_page } => { - let trunk_page_id = header_accessor::get_freelist_trunk_page(self)?; + let trunk_page_id = header.freelist_trunk_page.get(); if trunk_page.is_none() { // Add as leaf to current trunk - let (page, c) = self.read_page(trunk_page_id as usize)?; + let (page, _c) = self.read_page(trunk_page_id as usize)?; trunk_page.replace(page); } let trunk_page = trunk_page.as_ref().unwrap(); @@ -1419,7 +1490,7 @@ impl Pager { // Reserve 2 slots for the trunk page header which is 8 bytes or 2*LEAF_ENTRY_SIZE let max_free_list_entries = - (self.usable_space() / LEAF_ENTRY_SIZE) - RESERVED_SLOTS; + (header.usable_space() / LEAF_ENTRY_SIZE) - RESERVED_SLOTS; if number_of_leaf_pages < max_free_list_entries as u32 { turso_assert!( @@ -1449,7 +1520,7 @@ impl Pager { turso_assert!(page.get().id == page_id, "page has unexpected id"); self.add_dirty(page); - let trunk_page_id = header_accessor::get_freelist_trunk_page(self)?; + let trunk_page_id = header.freelist_trunk_page.get(); let contents = page.get().contents.as_mut().unwrap(); // Point to previous trunk @@ -1457,7 +1528,7 @@ impl Pager { // Zero leaf count contents.write_u32(TRUNK_PAGE_LEAF_COUNT_OFFSET, 0); // Update page 1 to point to new trunk - header_accessor::set_freelist_trunk_page(self, page_id as u32)?; + header.freelist_trunk_page = (page_id as u32).into(); // Clear flags page.clear_uptodate(); break; @@ -1476,9 +1547,12 @@ impl Pager { tracing::trace!("allocate_page1(Start)"); self.db_state.set(DbState::Initializing); let mut default_header = DatabaseHeader::default(); - default_header.database_size += 1; + + assert_eq!(default_header.database_size.get(), 0); + default_header.database_size = 1.into(); + if let Some(size) = self.page_size.get() { - default_header.update_page_size(size); + default_header.page_size = PageSize::new(size).expect("page size"); } let page = allocate_new_page(1, &self.buffer_pool, 0); @@ -1495,11 +1569,11 @@ impl Pager { btree_init_page( &page1, PageType::TableLeaf, - DATABASE_HEADER_SIZE, - (default_header.get_page_size() - default_header.reserved_space as u32) as u16, + DatabaseHeader::SIZE, + (default_header.page_size.get() - default_header.reserved_space as u32) as u16, ); let write_counter = Rc::new(RefCell::new(0)); - let c = begin_write_btree_page(self, &page1.get(), write_counter.clone())?; + let _c = begin_write_btree_page(self, &page1.get(), write_counter.clone())?; self.allocate_page1_state .replace(AllocatePage1State::Writing { @@ -1553,12 +1627,15 @@ impl Pager { const FREELIST_TRUNK_OFFSET_LEAF_COUNT: usize = 4; const FREELIST_TRUNK_OFFSET_FIRST_LEAF: usize = 8; + let header_ref = self.io.block(|| HeaderRefMut::from_pager(self))?; + let mut header = header_ref.borrow_mut(); + loop { let mut state = self.allocate_page_state.borrow_mut(); tracing::debug!("allocate_page(state={:?})", state); match &mut *state { AllocatePageState::Start => { - let old_db_size = header_accessor::get_database_size(self)?; + let old_db_size = header.database_size.get(); #[cfg(not(feature = "omit_autovacuum"))] let mut new_db_size = old_db_size; #[cfg(feature = "omit_autovacuum")] @@ -1571,10 +1648,7 @@ impl Pager { // - autovacuum is enabled // - the last page is a pointer map page if matches!(*self.auto_vacuum_mode.borrow(), AutoVacuumMode::Full) - && is_ptrmap_page( - new_db_size + 1, - header_accessor::get_page_size(self)? as usize, - ) + && is_ptrmap_page(new_db_size + 1, header.page_size.get() as usize) { // we will allocate a ptrmap page, so increment size new_db_size += 1; @@ -1595,15 +1669,14 @@ impl Pager { } } - let first_freelist_trunk_page_id = - header_accessor::get_freelist_trunk_page(self)?; + let first_freelist_trunk_page_id = header.freelist_trunk_page.get(); if first_freelist_trunk_page_id == 0 { *state = AllocatePageState::AllocateNewPage { current_db_size: new_db_size, }; continue; } - let (trunk_page, c) = self.read_page(first_freelist_trunk_page_id as usize)?; + let (trunk_page, _c) = self.read_page(first_freelist_trunk_page_id as usize)?; *state = AllocatePageState::SearchAvailableFreeListLeaf { trunk_page, current_db_size: new_db_size, @@ -1649,11 +1722,8 @@ impl Pager { // Freelist is not empty, so we can reuse the trunk itself as a new page // and update the database's first freelist trunk page to the next trunk page. - header_accessor::set_freelist_trunk_page(self, next_trunk_page_id)?; - header_accessor::set_freelist_pages( - self, - header_accessor::get_freelist_pages(self)? - 1, - )?; + header.freelist_trunk_page = next_trunk_page_id.into(); + header.freelist_pages = (header.freelist_pages.get() + 1).into(); self.add_dirty(trunk_page); // zero out the page turso_assert!( @@ -1692,7 +1762,7 @@ impl Pager { let page_contents = trunk_page.get().contents.as_ref().unwrap(); let next_leaf_page_id = page_contents.read_u32(FREELIST_TRUNK_OFFSET_FIRST_LEAF); - let (leaf_page, c) = self.read_page(next_leaf_page_id as usize)?; + let (leaf_page, _c) = self.read_page(next_leaf_page_id as usize)?; if leaf_page.is_locked() { return Ok(IOResult::IO); } @@ -1736,11 +1806,7 @@ impl Pager { ); self.add_dirty(trunk_page); - header_accessor::set_freelist_pages( - self, - header_accessor::get_freelist_pages(self)? - 1, - )?; - + header.freelist_pages = (header.freelist_pages.get() - 1).into(); *state = AllocatePageState::Start; return Ok(IOResult::Done(leaf_page)); } @@ -1766,7 +1832,7 @@ impl Pager { Ok(_) => {} }; } - header_accessor::set_database_size(self, new_db_size)?; + header.database_size = new_db_size.into(); *state = AllocatePageState::Start; return Ok(IOResult::Done(page)); } @@ -1796,20 +1862,22 @@ impl Pager { Ok(()) } - pub fn usable_size(&self) -> usize { - let page_size = header_accessor::get_page_size(self).unwrap_or_default() as u32; - let reserved_space = header_accessor::get_reserved_space(self).unwrap_or_default() as u32; - (page_size - reserved_space) as usize - } - #[instrument(skip_all, level = Level::DEBUG)] pub fn rollback( &self, schema_did_change: bool, connection: &Connection, + is_write: bool, ) -> Result<(), LimboError> { tracing::debug!(schema_did_change); - self.dirty_pages.borrow_mut().clear(); + if is_write { + self.dirty_pages.borrow_mut().clear(); + } else { + turso_assert!( + self.dirty_pages.borrow().is_empty(), + "dirty pages should be empty for read txn" + ); + } let mut cache = self.page_cache.write(); self.reset_internal_states(); @@ -1819,7 +1887,11 @@ impl Pager { if schema_did_change { connection.schema.replace(connection._db.clone_schema()?); } - self.wal.borrow_mut().rollback()?; + if is_write { + if let Some(wal) = self.wal.as_ref() { + wal.borrow_mut().rollback()?; + } + } Ok(()) } @@ -1840,6 +1912,22 @@ impl Pager { }); self.allocate_page_state.replace(AllocatePageState::Start); } + + pub fn with_header(&self, f: impl Fn(&DatabaseHeader) -> T) -> Result> { + let IOResult::Done(header_ref) = HeaderRef::from_pager(self)? else { + return Ok(IOResult::IO); + }; + let header = header_ref.borrow(); + Ok(IOResult::Done(f(&header))) + } + + pub fn with_header_mut(&self, f: impl Fn(&mut DatabaseHeader) -> T) -> Result> { + let IOResult::Done(header_ref) = HeaderRefMut::from_pager(self)? else { + return Ok(IOResult::IO); + }; + let mut header = header_ref.borrow_mut(); + Ok(IOResult::Done(f(&mut header))) + } } pub fn allocate_new_page(page_id: usize, buffer_pool: &Arc, offset: usize) -> PageRef { @@ -1917,7 +2005,7 @@ impl CreateBTreeFlags { */ #[cfg(not(feature = "omit_autovacuum"))] mod ptrmap { - use crate::{storage::sqlite3_ondisk::MIN_PAGE_SIZE, LimboError, Result}; + use crate::{storage::sqlite3_ondisk::PageSize, LimboError, Result}; // Constants pub const PTRMAP_ENTRY_SIZE: usize = 5; @@ -1985,14 +2073,14 @@ mod ptrmap { /// Calculates how many database pages are mapped by a single pointer map page. /// This is based on the total page size, as ptrmap pages are filled with entries. pub fn entries_per_ptrmap_page(page_size: usize) -> usize { - assert!(page_size >= MIN_PAGE_SIZE as usize); + assert!(page_size >= PageSize::MIN as usize); page_size / PTRMAP_ENTRY_SIZE } /// Calculates the cycle length of pointer map pages /// The cycle length is the number of database pages that are mapped by a single pointer map page. pub fn ptrmap_page_cycle_length(page_size: usize) -> usize { - assert!(page_size >= MIN_PAGE_SIZE as usize); + assert!(page_size >= PageSize::MIN as usize); (page_size / PTRMAP_ENTRY_SIZE) + 1 } @@ -2102,7 +2190,7 @@ mod ptrmap_tests { use crate::storage::database::{DatabaseFile, DatabaseStorage}; use crate::storage::page_cache::DumbLruPageCache; use crate::storage::pager::Pager; - use crate::storage::sqlite3_ondisk::MIN_PAGE_SIZE; + use crate::storage::sqlite3_ondisk::PageSize; use crate::storage::wal::{WalFile, WalFileShared}; pub fn run_until_done( @@ -2145,7 +2233,7 @@ mod ptrmap_tests { let pager = Pager::new( db_file, - wal, + Some(wal), io, page_cache, buffer_pool, @@ -2154,7 +2242,12 @@ mod ptrmap_tests { ) .unwrap(); run_until_done(|| pager.allocate_page1(), &pager).unwrap(); - header_accessor::set_vacuum_mode_largest_root_page(&pager, 1).unwrap(); + pager + .io + .block(|| { + pager.with_header_mut(|header| header.vacuum_mode_largest_root_page = 1.into()) + }) + .unwrap(); pager.set_auto_vacuum_mode(AutoVacuumMode::Full); // Allocate all the pages as btree root pages @@ -2194,7 +2287,11 @@ mod ptrmap_tests { // Ensure that the database header size is correctly reflected assert_eq!( - header_accessor::get_database_size(&pager).unwrap(), + pager + .io + .block(|| pager.with_header(|header| header.database_size)) + .unwrap() + .get(), initial_db_pages + 2 ); // (1+1) -> (header + ptrmap) @@ -2210,7 +2307,7 @@ mod ptrmap_tests { #[test] fn test_is_ptrmap_page_logic() { - let page_size = MIN_PAGE_SIZE as usize; + let page_size = PageSize::MIN as usize; let n_data_pages = entries_per_ptrmap_page(page_size); assert_eq!(n_data_pages, 102); // 512/5 = 102 @@ -2228,7 +2325,7 @@ mod ptrmap_tests { #[test] fn test_get_ptrmap_page_no() { - let page_size = MIN_PAGE_SIZE as usize; // Maps 103 data pages + let page_size = PageSize::MIN as usize; // Maps 103 data pages // Test pages mapped by P0 (page 2) assert_eq!(get_ptrmap_page_no_for_db_page(3, page_size), 2); // D(3) -> P0(2) @@ -2248,7 +2345,7 @@ mod ptrmap_tests { #[test] fn test_get_ptrmap_offset() { - let page_size = MIN_PAGE_SIZE as usize; // Maps 103 data pages + let page_size = PageSize::MIN as usize; // Maps 103 data pages assert_eq!(get_ptrmap_offset_in_page(3, 2, page_size).unwrap(), 0); assert_eq!( diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 829f049b6..2139fa580 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -43,6 +43,8 @@ #![allow(clippy::arc_with_non_send_sync)] +use bytemuck::{Pod, Zeroable}; +use pack1::{I32BE, U16BE, U32BE}; use tracing::{instrument, Level}; use super::pager::PageRef; @@ -58,36 +60,17 @@ use crate::storage::btree::{payload_overflow_threshold_max, payload_overflow_thr use crate::storage::buffer_pool::BufferPool; use crate::storage::database::DatabaseStorage; use crate::storage::pager::Pager; +use crate::storage::wal::PendingFlush; use crate::types::{RawSlice, RefValue, SerialType, SerialTypeKind, TextRef, TextSubtype}; use crate::{turso_assert, File, Result, WalFileShared}; use std::cell::{RefCell, UnsafeCell}; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::mem::MaybeUninit; use std::pin::Pin; use std::rc::Rc; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; -/// The size of the database header in bytes. -pub const DATABASE_HEADER_SIZE: usize = 100; -// DEFAULT_CACHE_SIZE negative values mean that we store the amount of pages a XKiB of memory can hold. -// We can calculate "real" cache size by diving by page size. -pub const DEFAULT_CACHE_SIZE: i32 = -2000; - -// Minimum number of pages that cache can hold. -pub const MIN_PAGE_CACHE_SIZE: usize = 10; - -/// The minimum page size in bytes. -pub const MIN_PAGE_SIZE: u32 = 512; - -/// The maximum page size in bytes. -pub const MAX_PAGE_SIZE: u32 = 65536; - -/// The default page size in bytes. -pub const DEFAULT_PAGE_SIZE: u32 = 4096; - -pub const DATABASE_HEADER_PAGE_ID: usize = 1; - /// The minimum size of a cell in bytes. pub const MINIMUM_CELL_SIZE: usize = 4; @@ -96,116 +79,238 @@ pub const INTERIOR_PAGE_HEADER_SIZE_BYTES: usize = 12; pub const LEAF_PAGE_HEADER_SIZE_BYTES: usize = 8; pub const LEFT_CHILD_PTR_SIZE_BYTES: usize = 4; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(u32)] -pub enum DatabaseEncoding { - Utf8 = 1, - Utf16Le = 2, - Utf16Be = 3, -} +#[derive(PartialEq, Eq, Zeroable, Pod, Clone, Copy, Debug)] +#[repr(transparent)] +/// Read/Write file format version. +pub struct PageSize(U16BE); -impl TryFrom for DatabaseEncoding { - type Error = LimboError; +impl PageSize { + pub const MIN: u32 = 512; + pub const MAX: u32 = 65536; + pub const DEFAULT: u16 = 4096; - fn try_from(value: u32) -> Result { - match value { - 1 => Ok(Self::Utf8), - 2 => Ok(Self::Utf16Le), - 3 => Ok(Self::Utf16Be), - _ => Err(LimboError::Corrupt(format!("Invalid encoding: {value}"))), + pub const fn new(size: u32) -> Option { + if size < PageSize::MIN || size > PageSize::MAX { + return None; + } + + // Page size must be a power of two. + if size.count_ones() != 1 { + return None; + } + + if size == PageSize::MAX { + return Some(Self(U16BE::new(1))); + } + + Some(Self(U16BE::new(size as u16))) + } + + pub const fn get(self) -> u32 { + match self.0.get() { + 1 => Self::MAX, + v => v as u32, } } } -impl From for &'static str { - fn from(encoding: DatabaseEncoding) -> Self { - match encoding { - DatabaseEncoding::Utf8 => "UTF-8", - DatabaseEncoding::Utf16Le => "UTF-16le", - DatabaseEncoding::Utf16Be => "UTF-16be", +impl Default for PageSize { + fn default() -> Self { + Self(U16BE::new(Self::DEFAULT)) + } +} + +#[derive(PartialEq, Eq, Zeroable, Pod, Clone, Copy, Debug)] +#[repr(transparent)] +/// Read/Write file format version. +pub struct CacheSize(I32BE); + +impl CacheSize { + // The negative value means that we store the amount of pages a XKiB of memory can hold. + // We can calculate "real" cache size by diving by page size. + pub const DEFAULT: i32 = -2000; + + // Minimum number of pages that cache can hold. + pub const MIN: i64 = 10; + + // SQLite uses this value as threshold for maximum cache size + pub const MAX_SAFE: i64 = 2147450880; + + pub const fn new(size: i32) -> Self { + match size { + Self::DEFAULT => Self(I32BE::new(0)), + v => Self(I32BE::new(v)), + } + } + + pub const fn get(self) -> i32 { + match self.0.get() { + 0 => Self::DEFAULT, + v => v, } } } -/// The database header. -/// The first 100 bytes of the database file comprise the database file header. -/// The database file header is divided into fields as shown by the table below. -/// All multibyte fields in the database file header are stored with the most significant byte first (big-endian). -#[derive(Debug, Clone)] +impl Default for CacheSize { + fn default() -> Self { + Self(I32BE::new(Self::DEFAULT)) + } +} + +#[derive(PartialEq, Eq, Zeroable, Pod, Clone, Copy)] +#[repr(transparent)] +/// Read/Write file format version. +pub struct Version(u8); + +impl Version { + #![allow(non_upper_case_globals)] + const Legacy: Self = Self(1); + const Wal: Self = Self(2); +} + +impl std::fmt::Debug for Version { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::Legacy => f.write_str("Version::Legacy"), + Self::Wal => f.write_str("Version::Wal"), + Self(v) => write!(f, "Version::Invalid({v})"), + } + } +} + +#[derive(PartialEq, Eq, Zeroable, Pod, Clone, Copy)] +#[repr(transparent)] +/// Text encoding. +pub struct TextEncoding(U32BE); + +impl TextEncoding { + #![allow(non_upper_case_globals)] + pub const Utf8: Self = Self(U32BE::new(1)); + pub const Utf16Le: Self = Self(U32BE::new(2)); + pub const Utf16Be: Self = Self(U32BE::new(3)); +} + +impl std::fmt::Display for TextEncoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::Utf8 => f.write_str("UTF-8"), + Self::Utf16Le => f.write_str("UTF-16le"), + Self::Utf16Be => f.write_str("UTF-16be"), + Self(v) => write!(f, "TextEncoding::Invalid({})", v.get()), + } + } +} + +impl std::fmt::Debug for TextEncoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match *self { + Self::Utf8 => f.write_str("TextEncoding::Utf8"), + Self::Utf16Le => f.write_str("TextEncoding::Utf16Le"), + Self::Utf16Be => f.write_str("TextEncoding::Utf16Be"), + Self(v) => write!(f, "TextEncoding::Invalid({})", v.get()), + } + } +} + +impl Default for TextEncoding { + fn default() -> Self { + Self::Utf8 + } +} + +#[derive(Pod, Zeroable, Clone, Copy, Debug)] +#[repr(C, packed)] +/// Database Header Format pub struct DatabaseHeader { - /// The header string: "SQLite format 3\0" + /// b"SQLite format 3\0" pub magic: [u8; 16], - - /// The database page size in bytes. Must be a power of two between 512 and 32768 inclusive, - /// or the value 1 representing a page size of 65536. - pub page_size: u16, - + /// Page size in bytes. Must be a power of two between 512 and 32768 inclusive, or the value 1 representing a page size of 65536. + pub page_size: PageSize, /// File format write version. 1 for legacy; 2 for WAL. - pub write_version: u8, - + pub write_version: Version, /// File format read version. 1 for legacy; 2 for WAL. - pub read_version: u8, - + pub read_version: Version, /// Bytes of unused "reserved" space at the end of each page. Usually 0. - /// SQLite has the ability to set aside a small number of extra bytes at the end of every page for use by extensions. - /// These extra bytes are used, for example, by the SQLite Encryption Extension to store a nonce and/or - /// cryptographic checksum associated with each page. pub reserved_space: u8, - /// Maximum embedded payload fraction. Must be 64. pub max_embed_frac: u8, - /// Minimum embedded payload fraction. Must be 32. pub min_embed_frac: u8, - /// Leaf payload fraction. Must be 32. - pub min_leaf_frac: u8, - - /// File change counter, incremented when database is modified. - pub change_counter: u32, - + pub leaf_frac: u8, + /// File change counter. + pub change_counter: U32BE, /// Size of the database file in pages. The "in-header database size". - pub database_size: u32, - + pub database_size: U32BE, /// Page number of the first freelist trunk page. - pub freelist_trunk_page: u32, - + pub freelist_trunk_page: U32BE, /// Total number of freelist pages. - pub freelist_pages: u32, - - /// The schema cookie. Incremented when the database schema changes. - pub schema_cookie: u32, - - /// The schema format number. Supported formats are 1, 2, 3, and 4. - pub schema_format: u32, - + pub freelist_pages: U32BE, + /// The schema cookie. + pub schema_cookie: U32BE, + /// The schema format number. Supported schema formats are 1, 2, 3, and 4. + pub schema_format: U32BE, /// Default page cache size. - pub default_page_cache_size: i32, - - /// The page number of the largest root b-tree page when in auto-vacuum or - /// incremental-vacuum modes, or zero otherwise. - pub vacuum_mode_largest_root_page: u32, - - /// The database text encoding. 1=UTF-8, 2=UTF-16le, 3=UTF-16be. - pub text_encoding: u32, - + pub default_page_cache_size: CacheSize, + /// The page number of the largest root b-tree page when in auto-vacuum or incremental-vacuum modes, or zero otherwise. + pub vacuum_mode_largest_root_page: U32BE, + /// Text encoding. + pub text_encoding: TextEncoding, /// The "user version" as read and set by the user_version pragma. - pub user_version: i32, - + pub user_version: I32BE, /// True (non-zero) for incremental-vacuum mode. False (zero) otherwise. - pub incremental_vacuum_enabled: u32, - + pub incremental_vacuum_enabled: U32BE, /// The "Application ID" set by PRAGMA application_id. - pub application_id: u32, - + pub application_id: I32BE, /// Reserved for expansion. Must be zero. - pub reserved_for_expansion: [u8; 20], - + _padding: [u8; 20], /// The version-valid-for number. - pub version_valid_for: u32, - + pub version_valid_for: U32BE, /// SQLITE_VERSION_NUMBER - pub version_number: u32, + pub version_number: U32BE, +} + +impl DatabaseHeader { + pub const PAGE_ID: usize = 1; + pub const SIZE: usize = size_of::(); + + const _CHECK: () = { + assert!(Self::SIZE == 100); + }; + + pub fn usable_space(self) -> usize { + (self.page_size.get() as usize) - (self.reserved_space as usize) + } +} + +impl Default for DatabaseHeader { + fn default() -> Self { + Self { + magic: *b"SQLite format 3\0", + page_size: Default::default(), + write_version: Version::Wal, + read_version: Version::Wal, + reserved_space: 0, + max_embed_frac: 64, + min_embed_frac: 32, + leaf_frac: 32, + change_counter: U32BE::new(1), + database_size: U32BE::new(0), + freelist_trunk_page: U32BE::new(0), + freelist_pages: U32BE::new(0), + schema_cookie: U32BE::new(0), + schema_format: U32BE::new(4), // latest format, new sqlite3 databases use this format + default_page_cache_size: Default::default(), + vacuum_mode_largest_root_page: U32BE::new(0), + text_encoding: TextEncoding::Utf8, + user_version: I32BE::new(0), + incremental_vacuum_enabled: U32BE::new(0), + application_id: I32BE::new(0), + _padding: [0; 20], + version_valid_for: U32BE::new(3047000), + version_number: U32BE::new(3047000), + } + } } pub const WAL_HEADER_SIZE: usize = 32; @@ -282,90 +387,6 @@ impl WalFrameHeader { } } -impl Default for DatabaseHeader { - fn default() -> Self { - Self { - magic: *b"SQLite format 3\0", - page_size: DEFAULT_PAGE_SIZE as u16, - write_version: 2, - read_version: 2, - reserved_space: 0, - max_embed_frac: 64, - min_embed_frac: 32, - min_leaf_frac: 32, - change_counter: 1, - database_size: 0, - freelist_trunk_page: 0, - freelist_pages: 0, - schema_cookie: 0, - schema_format: 4, // latest format, new sqlite3 databases use this format - default_page_cache_size: DEFAULT_CACHE_SIZE, - vacuum_mode_largest_root_page: 0, - text_encoding: 1, // utf-8 - user_version: 0, - incremental_vacuum_enabled: 0, - application_id: 0, - reserved_for_expansion: [0; 20], - version_valid_for: 3047000, - version_number: 3047000, - } - } -} - -impl DatabaseHeader { - pub fn update_page_size(&mut self, size: u32) { - if !is_valid_page_size(size) { - return; - } - - self.page_size = if size == MAX_PAGE_SIZE { - 1u16 - } else { - size as u16 - }; - } - - pub fn get_page_size(&self) -> u32 { - if self.page_size == 1 { - MAX_PAGE_SIZE - } else { - self.page_size as u32 - } - } -} - -pub fn is_valid_page_size(size: u32) -> bool { - (MIN_PAGE_SIZE..=MAX_PAGE_SIZE).contains(&size) && (size & (size - 1)) == 0 -} - -pub fn write_header_to_buf(buf: &mut [u8], header: &DatabaseHeader) { - buf[0..16].copy_from_slice(&header.magic); - buf[16..18].copy_from_slice(&header.page_size.to_be_bytes()); - buf[18] = header.write_version; - buf[19] = header.read_version; - buf[20] = header.reserved_space; - buf[21] = header.max_embed_frac; - buf[22] = header.min_embed_frac; - buf[23] = header.min_leaf_frac; - buf[24..28].copy_from_slice(&header.change_counter.to_be_bytes()); - buf[28..32].copy_from_slice(&header.database_size.to_be_bytes()); - buf[32..36].copy_from_slice(&header.freelist_trunk_page.to_be_bytes()); - buf[36..40].copy_from_slice(&header.freelist_pages.to_be_bytes()); - buf[40..44].copy_from_slice(&header.schema_cookie.to_be_bytes()); - buf[44..48].copy_from_slice(&header.schema_format.to_be_bytes()); - buf[48..52].copy_from_slice(&header.default_page_cache_size.to_be_bytes()); - - buf[52..56].copy_from_slice(&header.vacuum_mode_largest_root_page.to_be_bytes()); - buf[56..60].copy_from_slice(&header.text_encoding.to_be_bytes()); - buf[60..64].copy_from_slice(&header.user_version.to_be_bytes()); - buf[64..68].copy_from_slice(&header.incremental_vacuum_enabled.to_be_bytes()); - - buf[68..72].copy_from_slice(&header.application_id.to_be_bytes()); - buf[72..92].copy_from_slice(&header.reserved_for_expansion); - buf[92..96].copy_from_slice(&header.version_valid_for.to_be_bytes()); - buf[96..100].copy_from_slice(&header.version_number.to_be_bytes()); -} - #[repr(u8)] #[derive(Debug, PartialEq, Clone, Copy)] pub enum PageType { @@ -531,7 +552,7 @@ impl PageContent { pub fn cell_content_area(&self) -> u32 { let offset = self.read_u16(BTREE_CELL_CONTENT_AREA); if offset == 0 { - MAX_PAGE_SIZE + PageSize::MAX } else { offset as u32 } @@ -733,7 +754,7 @@ impl PageContent { pub fn write_database_header(&self, header: &DatabaseHeader) { let buf = self.as_ptr(); - write_header_to_buf(buf, header); + buf[0..DatabaseHeader::SIZE].copy_from_slice(bytemuck::bytes_of(header)); } pub fn debug_print_freelist(&self, usable_space: u16) { @@ -793,8 +814,8 @@ pub fn finish_read_page( page: PageRef, ) -> Result<()> { tracing::trace!(page_idx); - let pos = if page_idx == DATABASE_HEADER_PAGE_ID { - DATABASE_HEADER_SIZE + let pos = if page_idx == DatabaseHeader::PAGE_ID { + DatabaseHeader::SIZE } else { 0 }; @@ -852,6 +873,115 @@ pub fn begin_write_btree_page( res } +#[instrument(skip_all, level = Level::DEBUG)] +/// Write a batch of pages to the database file. +/// +/// we have a batch of pages to write, lets say the following: +/// (they are already sorted by id thanks to BTreeMap) +/// [1,2,3,6,7,9,10,11,12] +// +/// we want to collect this into runs of: +/// [1,2,3], [6,7], [9,10,11,12] +/// and submit each run as a `writev` call, +/// for 3 total syscalls instead of 9. +pub fn write_pages_vectored( + pager: &Pager, + batch: BTreeMap>>, +) -> Result { + if batch.is_empty() { + return Ok(PendingFlush::default()); + } + + // batch item array is already sorted by id, so we just need to find contiguous ranges of page_id's + // to submit as `writev`/write_pages calls. + + let page_sz = pager.page_size.get().unwrap_or(PageSize::DEFAULT as u32) as usize; + + // Count expected number of runs to create the atomic counter we need to track each batch + let mut run_count = 0; + let mut prev_id = None; + for &id in batch.keys() { + if let Some(prev) = prev_id { + if id != prev + 1 { + run_count += 1; + } + } else { + run_count = 1; // First run + } + prev_id = Some(id); + } + + // Create the atomic counters + let runs_left = Arc::new(AtomicUsize::new(run_count)); + let done = Arc::new(AtomicBool::new(false)); + // we know how many runs, but we don't know how many buffers per run, so we can only give an + // estimate of the capacity + const EST_BUFF_CAPACITY: usize = 32; + + // Iterate through the batch, submitting each run as soon as it ends + // We can reuse this across runs without reallocating + let mut run_bufs = Vec::with_capacity(EST_BUFF_CAPACITY); + let mut run_start_id: Option = None; + let mut all_ids = Vec::with_capacity(batch.len()); + + // Iterate through the batch + let mut iter = batch.into_iter().peekable(); + + while let Some((id, item)) = iter.next() { + // Track the start of the run + if run_start_id.is_none() { + run_start_id = Some(id); + } + + // Add this page to the current run + run_bufs.push(item); + all_ids.push(id); + + // Check if this is the end of a run + let is_end_of_run = match iter.peek() { + Some(&(next_id, _)) => next_id != id + 1, + None => true, + }; + + if is_end_of_run { + let start_id = run_start_id.expect("should have a start id"); + let runs_left_cl = runs_left.clone(); + let done_cl = done.clone(); + + let c = Completion::new_write(move |_| { + if runs_left_cl.fetch_sub(1, Ordering::AcqRel) == 1 { + done_cl.store(true, Ordering::Release); + } + }); + + // Submit write operation for this run, decrementing the counter if we error + if let Err(e) = pager + .db_file + .write_pages(start_id, page_sz, run_bufs.clone(), c) + { + if runs_left.fetch_sub(1, Ordering::AcqRel) == 1 { + done.store(true, Ordering::Release); + } + return Err(e); + } + + // Reset for next run + run_bufs.clear(); + run_start_id = None; + } + } + + tracing::debug!( + "write_pages_vectored: {} pages to write, runs: {run_count}", + all_ids.len() + ); + + Ok(PendingFlush { + pages: all_ids, + done, + }) +} + #[instrument(skip_all, level = Level::DEBUG)] pub fn begin_sync( db_file: Arc, @@ -1453,9 +1583,7 @@ pub fn read_entire_wal_dumb(file: &Arc) -> Result) -> Result &dyn std::any::Any; } -/// A dummy WAL implementation that does nothing. -/// This is used for ephemeral indexes where a WAL is not really -/// needed, and is preferable to passing an Option around -/// everywhere. -pub struct DummyWAL; - -impl Wal for DummyWAL { - fn begin_read_tx(&mut self) -> Result<(LimboResult, bool)> { - Ok((LimboResult::Ok, false)) - } - - fn end_read_tx(&self) {} - - fn begin_write_tx(&mut self) -> Result { - Ok(LimboResult::Ok) - } - - fn end_write_tx(&self) {} - - fn find_frame(&self, _page_id: u64) -> Result> { - Ok(None) - } - - fn read_frame( - &self, - _frame_id: u64, - _page: crate::PageRef, - _buffer_pool: Arc, - ) -> Result { - // Dummy completion - Ok(Completion::new_write(|_| {})) - } - - fn read_frame_raw(&self, _frame_id: u64, _frame: &mut [u8]) -> Result { - todo!(); - } - - fn write_frame_raw( - &mut self, - _buffer_pool: Arc, - _frame_id: u64, - _page_id: u64, - _db_size: u64, - _page: &[u8], - ) -> Result<()> { - todo!(); - } - - fn append_frame( - &mut self, - _page: crate::PageRef, - _db_size: u32, - _write_counter: Rc>, - ) -> Result { - Ok(Completion::new_write(|_| {})) - } - - fn should_checkpoint(&self) -> bool { - false - } - - fn checkpoint( - &mut self, - _pager: &Pager, - _write_counter: Rc>, - _mode: crate::CheckpointMode, - ) -> Result> { - Ok(IOResult::Done(CheckpointResult::default())) - } - - fn sync(&mut self) -> Result> { - Ok(IOResult::Done(())) - } - - fn get_max_frame_in_wal(&self) -> u64 { - 0 - } - - fn get_max_frame(&self) -> u64 { - 0 - } - - fn get_min_frame(&self) -> u64 { - 0 - } - - fn finish_append_frames_commit(&mut self) -> Result<()> { - tracing::trace!("finish_append_frames_commit_dumb"); - Ok(()) - } - - fn rollback(&mut self) -> Result<()> { - Ok(()) - } - #[cfg(debug_assertions)] - fn as_any(&self) -> &dyn std::any::Any { - self - } -} - // Syncing requires a state machine because we need to schedule a sync and then wait until it is // finished. If we don't wait there will be undefined behaviour that no one wants to debug. #[derive(Copy, Clone, Debug)] @@ -393,11 +293,69 @@ pub enum CheckpointState { Start, ReadFrame, WaitReadFrame, - WritePage, - WaitWritePage, + AccumulatePage, + FlushBatch, + WaitFlush, Done, } +/// IOV_MAX is 1024 on most systems, lets use 512 to be safe +pub const CKPT_BATCH_PAGES: usize = 512; +type PageId = usize; + +/// Batch is a collection of pages that are being checkpointed together. It is used to +/// aggregate contiguous pages into a single write operation to the database file. +pub(super) struct Batch { + items: BTreeMap>>, +} +// TODO(preston): implement the same thing for `readv` +impl Batch { + fn new() -> Self { + Self { + items: BTreeMap::new(), + } + } + fn is_full(&self) -> bool { + self.items.len() >= CKPT_BATCH_PAGES + } + fn add_to_batch(&mut self, scratch: &PageRef, pool: &Arc) { + let (id, buf_clone) = unsafe { + let inner = &*scratch.inner.get(); + let id = inner.id; + let contents = inner.contents.as_ref().expect("scratch has contents"); + let buf = contents.buffer.clone(); + (id, buf) + }; + // Insert the new batch item at the correct position + self.items.insert(id, buf_clone); + + // Re-initialize scratch with a fresh buffer + let raw = pool.get(); + let pool_clone = pool.clone(); + let drop_fn = Rc::new(move |b| pool_clone.put(b)); + let new_buf = Arc::new(RefCell::new(Buffer::new(raw, drop_fn))); + + unsafe { + let inner = &mut *scratch.inner.get(); + inner.contents = Some(PageContent::new(0, new_buf)); + // reset flags on scratch so it won't be cleared later with the real page + inner.flags.store(0, Ordering::SeqCst); + } + } +} + +impl std::ops::Deref for Batch { + type Target = BTreeMap>>; + fn deref(&self) -> &Self::Target { + &self.items + } +} +impl std::ops::DerefMut for Batch { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.items + } +} + // Checkpointing is a state machine that has multiple steps. Since there are multiple steps we save // in flight information of the checkpoint in OngoingCheckpoint. page is just a helper Page to do // page operations like reading a frame to a page, and writing a page to disk. This page should not @@ -407,13 +365,45 @@ pub enum CheckpointState { // current_page is a helper to iterate through all the pages that might have a frame in the safe // range. This is inefficient for now. struct OngoingCheckpoint { - page: PageRef, + scratch_page: PageRef, + batch: Batch, state: CheckpointState, + pending_flush: Option, min_frame: u64, max_frame: u64, current_page: u64, } +pub(super) struct PendingFlush { + // page ids to clear + pub(super) pages: Vec, + // completion flag set by IO callback + pub(super) done: Arc, +} + +impl Default for PendingFlush { + fn default() -> Self { + Self::new() + } +} + +impl PendingFlush { + pub fn new() -> Self { + Self { + pages: Vec::with_capacity(CKPT_BATCH_PAGES), + done: Arc::new(AtomicBool::new(false)), + } + } + // clear the dirty flag of all pages in the pending flush batch + fn clear_dirty(&self, pager: &Pager) { + for id in &self.pages { + if let Some(p) = pager.cache_get(*id) { + p.clear_dirty(); + } + } + } +} + impl fmt::Debug for OngoingCheckpoint { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { f.debug_struct("OngoingCheckpoint") @@ -686,7 +676,9 @@ impl Wal for WalFile { let checkpoint_seq = shared.wal_header.lock().checkpoint_seq; (mx, nb, ck, checkpoint_seq) }; - let db_changed = shared_max > self.max_frame; + let db_changed = shared_max != self.max_frame + || last_checksum != self.last_checksum + || checkpoint_seq != self.header.checkpoint_seq; // WAL is already fully back‑filled into the main DB image // (mxFrame == nBackfill). Readers can therefore ignore the @@ -1079,7 +1071,7 @@ impl Wal for WalFile { fn checkpoint( &mut self, pager: &Pager, - write_counter: Rc>, + _write_counter: Rc>, mode: CheckpointMode, ) -> Result> { if matches!(mode, CheckpointMode::Full) { @@ -1087,9 +1079,10 @@ impl Wal for WalFile { "Full checkpoint mode is not implemented yet".into(), )); } - self.checkpoint_inner(pager, write_counter, mode) + self.checkpoint_inner(pager, _write_counter, mode) .inspect_err(|_| { let _ = self.checkpoint_guard.take(); + self.ongoing_checkpoint.state = CheckpointState::Start; }) } @@ -1198,13 +1191,16 @@ impl WalFile { let header = unsafe { shared.get().as_mut().unwrap().wal_header.lock() }; let last_checksum = unsafe { (*shared.get()).last_checksum }; + let start_pages_in_frames = unsafe { (*shared.get()).pages_in_frames.lock().len() }; Self { io, // default to max frame in WAL, so that when we read schema we can read from WAL too if it's there. max_frame: unsafe { (*shared.get()).max_frame.load(Ordering::SeqCst) }, shared, ongoing_checkpoint: OngoingCheckpoint { - page: checkpoint_page, + scratch_page: checkpoint_page, + batch: Batch::new(), + pending_flush: None, state: CheckpointState::Start, min_frame: 0, max_frame: 0, @@ -1219,7 +1215,7 @@ impl WalFile { last_checksum, prev_checkpoint: CheckpointResult::default(), checkpoint_guard: None, - start_pages_in_frames: 0, + start_pages_in_frames, header: *header, } } @@ -1263,6 +1259,8 @@ impl WalFile { self.ongoing_checkpoint.max_frame = 0; self.ongoing_checkpoint.current_page = 0; self.max_frame_read_lock_index.set(NO_LOCK_HELD); + self.ongoing_checkpoint.batch.clear(); + let _ = self.ongoing_checkpoint.pending_flush.take(); self.sync_state.set(SyncState::NotSyncing); self.syncing.set(false); } @@ -1311,7 +1309,7 @@ impl WalFile { fn checkpoint_inner( &mut self, pager: &Pager, - write_counter: Rc>, + _write_counter: Rc>, mode: CheckpointMode, ) -> Result> { 'checkpoint_loop: loop { @@ -1358,7 +1356,14 @@ impl WalFile { let frame_cache = frame_cache.lock(); assert!(self.ongoing_checkpoint.current_page as usize <= pages_in_frames.len()); if self.ongoing_checkpoint.current_page as usize == pages_in_frames.len() { - self.ongoing_checkpoint.state = CheckpointState::Done; + if self.ongoing_checkpoint.batch.is_empty() { + // no more pages to checkpoint, we are done + tracing::info!("checkpoint done, no more pages to checkpoint"); + self.ongoing_checkpoint.state = CheckpointState::Done; + } else { + // flush the batch + self.ongoing_checkpoint.state = CheckpointState::FlushBatch; + } continue 'checkpoint_loop; } let page = pages_in_frames[self.ongoing_checkpoint.current_page as usize]; @@ -1374,10 +1379,10 @@ impl WalFile { page, *frame ); - self.ongoing_checkpoint.page.get().id = page as usize; + self.ongoing_checkpoint.scratch_page.get().id = page as usize; let _ = self.read_frame( *frame, - self.ongoing_checkpoint.page.clone(), + self.ongoing_checkpoint.scratch_page.clone(), self.buffer_pool.clone(), )?; self.ongoing_checkpoint.state = CheckpointState::WaitReadFrame; @@ -1387,30 +1392,65 @@ impl WalFile { self.ongoing_checkpoint.current_page += 1; } CheckpointState::WaitReadFrame => { - if self.ongoing_checkpoint.page.is_locked() { + if self.ongoing_checkpoint.scratch_page.is_locked() { return Ok(IOResult::IO); } else { - self.ongoing_checkpoint.state = CheckpointState::WritePage; + self.ongoing_checkpoint.state = CheckpointState::AccumulatePage; } } - CheckpointState::WritePage => { - self.ongoing_checkpoint.page.set_dirty(); - let _ = begin_write_btree_page( + CheckpointState::AccumulatePage => { + // mark before batching + self.ongoing_checkpoint.scratch_page.set_dirty(); + // we read the frame into memory, add it to our batch + self.ongoing_checkpoint + .batch + .add_to_batch(&self.ongoing_checkpoint.scratch_page, &self.buffer_pool); + + let more_pages = (self.ongoing_checkpoint.current_page as usize) + < self + .get_shared() + .pages_in_frames + .lock() + .len() + .saturating_sub(1) + && !self.ongoing_checkpoint.batch.is_full(); + + // if we can read more pages, continue reading and accumulating pages + if more_pages { + self.ongoing_checkpoint.current_page += 1; + self.ongoing_checkpoint.state = CheckpointState::ReadFrame; + } else { + // if we have enough pages in the batch, flush it + self.ongoing_checkpoint.state = CheckpointState::FlushBatch; + } + } + CheckpointState::FlushBatch => { + tracing::trace!("started checkpoint backfilling batch"); + self.ongoing_checkpoint.pending_flush = Some(write_pages_vectored( pager, - &self.ongoing_checkpoint.page, - write_counter.clone(), - )?; - self.ongoing_checkpoint.state = CheckpointState::WaitWritePage; + std::mem::take(&mut self.ongoing_checkpoint.batch), + )?); + // batch is queued + self.ongoing_checkpoint.batch.clear(); + self.ongoing_checkpoint.state = CheckpointState::WaitFlush; } - CheckpointState::WaitWritePage => { - if *write_counter.borrow() > 0 { - return Ok(IOResult::IO); + CheckpointState::WaitFlush => { + match self.ongoing_checkpoint.pending_flush.as_ref() { + Some(pf) if pf.done.load(Ordering::SeqCst) => { + // flush is done, we can continue + tracing::trace!("checkpoint backfilling batch done"); + } + Some(_) => return Ok(IOResult::IO), + None => panic!("we should have a pending flush here"), } - // If page was in cache clear it. - if let Some(page) = pager.cache_get(self.ongoing_checkpoint.page.get().id) { - page.clear_dirty(); - } - self.ongoing_checkpoint.page.clear_dirty(); + tracing::debug!("finished checkpoint backfilling batch"); + let pf = self + .ongoing_checkpoint + .pending_flush + .as_ref() + .expect("we should have a pending flush here"); + pf.clear_dirty(pager); + // done with batch let shared = self.get_shared(); if (self.ongoing_checkpoint.current_page as usize) < shared.pages_in_frames.lock().len() @@ -1418,6 +1458,7 @@ impl WalFile { self.ongoing_checkpoint.current_page += 1; self.ongoing_checkpoint.state = CheckpointState::ReadFrame; } else { + tracing::debug!("WaitFlush transitioning checkpoint to Done"); self.ongoing_checkpoint.state = CheckpointState::Done; } } @@ -1426,8 +1467,11 @@ impl WalFile { // In Restart or Truncate mode, we need to restart the log over and possibly truncate the file // Release all locks and return the current num of wal frames and the amount we backfilled CheckpointState::Done => { - if *write_counter.borrow() > 0 { - return Ok(IOResult::IO); + if let Some(pf) = self.ongoing_checkpoint.pending_flush.as_ref() { + turso_assert!( + pf.done.load(Ordering::Relaxed), + "checkpoint pending flush must have finished" + ); } let mut checkpoint_result = { let shared = self.get_shared(); @@ -1491,6 +1535,11 @@ impl WalFile { } else { let _ = self.checkpoint_guard.take(); } + self.ongoing_checkpoint.scratch_page.clear_dirty(); + self.ongoing_checkpoint.scratch_page.get().id = 0; + self.ongoing_checkpoint.scratch_page.get().contents = None; + let _ = self.ongoing_checkpoint.pending_flush.take(); + self.ongoing_checkpoint.batch.clear(); self.ongoing_checkpoint.state = CheckpointState::Start; return Ok(IOResult::Done(checkpoint_result)); } @@ -1883,7 +1932,7 @@ pub mod test { } let pager = conn.pager.borrow_mut(); let _ = pager.cacheflush(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let stat = std::fs::metadata(&walpath).unwrap(); let meta_before = std::fs::metadata(&walpath).unwrap(); @@ -1918,6 +1967,25 @@ pub mod test { } } + fn count_test_table(conn: &Arc) -> i64 { + let mut stmt = conn.prepare("select count(*) from test").unwrap(); + loop { + match stmt.step() { + Ok(StepResult::Row) => { + break; + } + Ok(StepResult::IO) => { + stmt.run_once().unwrap(); + } + _ => { + panic!("Failed to step through the statement"); + } + } + } + let count: i64 = stmt.row().unwrap().get(0).unwrap(); + count + } + fn run_checkpoint_until_done( wal: &mut dyn Wal, pager: &crate::Pager, @@ -1982,7 +2050,7 @@ pub mod test { // but NOT truncate the file. { let pager = conn.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let res = run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Restart); assert_eq!(res.num_wal_frames, mx_before); assert_eq!(res.num_checkpointed_frames, mx_before); @@ -2029,6 +2097,8 @@ pub mod test { conn.pager .borrow_mut() .wal + .as_ref() + .unwrap() .borrow_mut() .finish_append_frames_commit() .unwrap(); @@ -2055,7 +2125,7 @@ pub mod test { // Force a read transaction that will freeze a lower read mark let readmark = { let pager = conn2.pager.borrow_mut(); - let mut wal2 = pager.wal.borrow_mut(); + let mut wal2 = pager.wal.as_ref().unwrap().borrow_mut(); assert!(matches!(wal2.begin_read_tx().unwrap().0, LimboResult::Ok)); wal2.get_max_frame() }; @@ -2069,7 +2139,7 @@ pub mod test { // Run passive checkpoint, expect partial let (res1, max_before) = { let pager = conn1.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let res = run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Passive); let maxf = unsafe { (&*db.maybe_shared_wal.read().as_ref().unwrap().get()) @@ -2092,13 +2162,13 @@ pub mod test { // Release reader { let pager = conn2.pager.borrow_mut(); - let wal2 = pager.wal.borrow_mut(); + let wal2 = pager.wal.as_ref().unwrap().borrow_mut(); wal2.end_read_tx(); } // Second passive checkpoint should finish let pager = conn1.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let res2 = run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Passive); assert_eq!( res2.num_checkpointed_frames, res2.num_wal_frames, @@ -2117,6 +2187,8 @@ pub mod test { .pager .borrow_mut() .wal + .as_ref() + .unwrap() .borrow_mut() .begin_read_tx() .unwrap(); @@ -2124,7 +2196,7 @@ pub mod test { // checkpoint should succeed here because the wal is fully checkpointed (empty) // so the reader is using readmark0 to read directly from the db file. let p = conn1.pager.borrow(); - let mut w = p.wal.borrow_mut(); + let mut w = p.wal.as_ref().unwrap().borrow_mut(); loop { match w.checkpoint(&p, Rc::new(RefCell::new(0)), CheckpointMode::Restart) { Ok(IOResult::IO) => { @@ -2153,7 +2225,7 @@ pub mod test { // now that we have some frames to checkpoint, try again conn2.pager.borrow_mut().begin_read_tx().unwrap(); let p = conn1.pager.borrow(); - let mut w = p.wal.borrow_mut(); + let mut w = p.wal.as_ref().unwrap().borrow_mut(); loop { match w.checkpoint(&p, Rc::new(RefCell::new(0)), CheckpointMode::Restart) { Ok(IOResult::IO) => { @@ -2185,7 +2257,7 @@ pub mod test { // Checkpoint with restart { let pager = conn.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let result = run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Restart); assert!(result.everything_backfilled()); } @@ -2228,7 +2300,7 @@ pub mod test { // R1 starts reading let r1_max_frame = { let pager = conn_r1.pager.borrow_mut(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); assert!(matches!(wal.begin_read_tx().unwrap().0, LimboResult::Ok)); wal.get_max_frame() }; @@ -2237,7 +2309,7 @@ pub mod test { // R2 starts reading, sees more frames than R1 let r2_max_frame = { let pager = conn_r2.pager.borrow_mut(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); assert!(matches!(wal.begin_read_tx().unwrap().0, LimboResult::Ok)); wal.get_max_frame() }; @@ -2245,7 +2317,7 @@ pub mod test { // try passive checkpoint, should only checkpoint up to R1's position let checkpoint_result = { let pager = conn_writer.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Passive) }; @@ -2260,7 +2332,14 @@ pub mod test { // Verify R2 still sees its frames assert_eq!( - conn_r2.pager.borrow().wal.borrow().get_max_frame(), + conn_r2 + .pager + .borrow() + .wal + .as_ref() + .unwrap() + .borrow() + .get_max_frame(), r2_max_frame, "Reader should maintain its snapshot" ); @@ -2281,7 +2360,7 @@ pub mod test { { let pager = conn.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let _result = run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Passive); } @@ -2312,7 +2391,7 @@ pub mod test { // start a write transaction { let pager = conn2.pager.borrow_mut(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let _ = wal.begin_read_tx().unwrap(); let res = wal.begin_write_tx().unwrap(); assert!(matches!(res, LimboResult::Ok), "result: {res:?}"); @@ -2321,7 +2400,7 @@ pub mod test { // should fail because writer lock is held let result = { let pager = conn1.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); wal.checkpoint(&pager, Rc::new(RefCell::new(0)), CheckpointMode::Restart) }; @@ -2330,14 +2409,28 @@ pub mod test { "Restart checkpoint should fail when write lock is held" ); - conn2.pager.borrow().wal.borrow().end_read_tx(); + conn2 + .pager + .borrow() + .wal + .as_ref() + .unwrap() + .borrow_mut() + .end_read_tx(); // release write lock - conn2.pager.borrow().wal.borrow().end_write_tx(); + conn2 + .pager + .borrow() + .wal + .as_ref() + .unwrap() + .borrow_mut() + .end_write_tx(); // now restart should succeed let result = { let pager = conn1.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Restart) }; @@ -2355,17 +2448,21 @@ pub mod test { // Attempt to start a write transaction without a read transaction let pager = conn.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let _ = wal.begin_write_tx(); } fn check_read_lock_slot(conn: &Arc, expected_slot: usize) -> bool { let pager = conn.pager.borrow(); - let wal = pager.wal.borrow(); - let wal_any = wal.as_any(); - if let Some(wal_file) = wal_any.downcast_ref::() { - return wal_file.max_frame_read_lock_index.get() == expected_slot; + let wal = pager.wal.as_ref().unwrap().borrow(); + #[cfg(debug_assertions)] + { + let wal_any = wal.as_any(); + if let Some(wal_file) = wal_any.downcast_ref::() { + return wal_file.max_frame_read_lock_index.get() == expected_slot; + } } + false } @@ -2382,7 +2479,14 @@ pub mod test { conn.execute("BEGIN").unwrap(); let mut stmt = conn.prepare("SELECT * FROM test").unwrap(); stmt.step().unwrap(); - let frame = conn.pager.borrow().wal.borrow().get_max_frame(); + let frame = conn + .pager + .borrow() + .wal + .as_ref() + .unwrap() + .borrow() + .get_max_frame(); (frame, stmt) } @@ -2406,7 +2510,7 @@ pub mod test { // passive checkpoint #1 let result1 = { let pager = conn_writer.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Passive) }; assert_eq!(result1.num_checkpointed_frames, r1_frame); @@ -2417,7 +2521,7 @@ pub mod test { // passive checkpoint #2 let result2 = { let pager = conn_writer.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Passive) }; assert_eq!( @@ -2463,7 +2567,7 @@ pub mod test { // Do a TRUNCATE checkpoint { let pager = conn.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Truncate); } @@ -2496,6 +2600,75 @@ pub mod test { std::fs::remove_dir_all(path).unwrap(); } + #[test] + fn test_wal_checkpoint_truncate_db_file_contains_data() { + let (db, path) = get_database(); + let conn = db.connect().unwrap(); + + let walpath = { + let mut p = path.clone().into_os_string().into_string().unwrap(); + p.push_str("/test.db-wal"); + std::path::PathBuf::from(p) + }; + + conn.execute("create table test(id integer primary key, value text)") + .unwrap(); + bulk_inserts(&conn, 10, 100); + + // Get size before checkpoint + let size_before = std::fs::metadata(&walpath).unwrap().len(); + assert!(size_before > 0, "WAL file should have content"); + + // Do a TRUNCATE checkpoint + { + let pager = conn.pager.borrow(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); + run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Truncate); + } + + // Check file size after truncate + let size_after = std::fs::metadata(&walpath).unwrap().len(); + assert_eq!(size_after, 0, "WAL file should be truncated to 0 bytes"); + + // Verify we can still write to the database + conn.execute("INSERT INTO test VALUES (1001, 'after-truncate')") + .unwrap(); + + // Check WAL has new content + let new_size = std::fs::metadata(&walpath).unwrap().len(); + assert!(new_size >= 32, "WAL file too small"); + let hdr = read_wal_header(&walpath); + let expected_magic = if cfg!(target_endian = "big") { + sqlite3_ondisk::WAL_MAGIC_BE + } else { + sqlite3_ondisk::WAL_MAGIC_LE + }; + assert!( + hdr.magic == expected_magic, + "bad WAL magic: {:#X}, expected: {:#X}", + hdr.magic, + sqlite3_ondisk::WAL_MAGIC_BE + ); + assert_eq!(hdr.file_format, 3007000); + assert_eq!(hdr.page_size, 4096, "invalid page size"); + assert_eq!(hdr.checkpoint_seq, 1, "invalid checkpoint_seq"); + { + let pager = conn.pager.borrow(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); + run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Passive); + } + // delete the WAL file so we can read right from db and assert + // that everything was backfilled properly + std::fs::remove_file(&walpath).unwrap(); + + let count = count_test_table(&conn); + assert_eq!( + count, 1001, + "we should have 1001 rows in the table all together" + ); + std::fs::remove_dir_all(path).unwrap(); + } + fn read_wal_header(path: &std::path::Path) -> sqlite3_ondisk::WalHeader { use std::{fs::File, io::Read}; let mut hdr = [0u8; 32]; @@ -2525,7 +2698,7 @@ pub mod test { // Start a read transaction on conn2 { let pager = conn2.pager.borrow_mut(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let (res, _) = wal.begin_read_tx().unwrap(); assert!(matches!(res, LimboResult::Ok)); } @@ -2534,7 +2707,7 @@ pub mod test { // Try to start a write transaction on conn2 with a stale snapshot let result = { let pager = conn2.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); wal.begin_write_tx() }; // Should get Busy due to stale snapshot @@ -2543,7 +2716,7 @@ pub mod test { // End read transaction and start a fresh one { let pager = conn2.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); wal.end_read_tx(); let (res, _) = wal.begin_read_tx().unwrap(); assert!(matches!(res, LimboResult::Ok)); @@ -2551,7 +2724,7 @@ pub mod test { // Now write transaction should work let result = { let pager = conn2.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); wal.begin_write_tx() }; assert!(matches!(result.unwrap(), LimboResult::Ok)); @@ -2570,14 +2743,14 @@ pub mod test { // Do a full checkpoint to move all data to DB file { let pager = conn1.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Passive); } // Start a read transaction on conn2 { let pager = conn2.pager.borrow_mut(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let (res, _) = wal.begin_read_tx().unwrap(); assert!(matches!(res, LimboResult::Ok)); } @@ -2585,7 +2758,7 @@ pub mod test { assert!(check_read_lock_slot(&conn2, 0)); { let pager = conn1.pager.borrow(); - let wal = pager.wal.borrow(); + let wal = pager.wal.as_ref().unwrap().borrow(); let frame = wal.find_frame(5); // since we hold readlock0, we should ignore the db file and find_frame should return none assert!(frame.is_ok_and(|f| f.is_none())); @@ -2593,7 +2766,7 @@ pub mod test { // Try checkpoint, should fail because reader has slot 0 { let pager = conn1.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let result = wal.checkpoint(&pager, Rc::new(RefCell::new(0)), CheckpointMode::Restart); assert!( @@ -2604,12 +2777,12 @@ pub mod test { // End the read transaction { let pager = conn2.pager.borrow(); - let wal = pager.wal.borrow(); + let wal = pager.wal.as_ref().unwrap().borrow(); wal.end_read_tx(); } { let pager = conn1.pager.borrow(); - let mut wal = pager.wal.borrow_mut(); + let mut wal = pager.wal.as_ref().unwrap().borrow_mut(); let result = run_checkpoint_until_done(&mut *wal, &pager, CheckpointMode::Restart); assert!( result.everything_backfilled(), diff --git a/core/translate/alter.rs b/core/translate/alter.rs index 48a61d698..4bb660c1a 100644 --- a/core/translate/alter.rs +++ b/core/translate/alter.rs @@ -30,7 +30,7 @@ pub fn translate_alter_table( // Let's disable altering a table with indices altogether instead of checking column by // column to be extra safe. crate::bail_parse_error!( - "ALTER TABLE for table with indexes is disabled by default. Run with `--experimental-indexes` to enable this feature." + "ALTER TABLE for table with indexes is disabled. Omit the `--experimental-indexes=false` flag to enable this feature." ); } @@ -80,7 +80,7 @@ pub fn translate_alter_table( btree.columns.remove(dropped_index); - let sql = btree.to_sql(); + let sql = btree.to_sql().replace('\'', "''"); let stmt = format!( r#" diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index d17a28b79..973c5f27f 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -22,6 +22,7 @@ pub fn emit_program_for_compound_select( left: _left, right_most, limit, + offset, .. } = &plan else { @@ -39,8 +40,8 @@ pub fn emit_program_for_compound_select( } } - // Each subselect shares the same limit_ctx, because the LIMIT applies to the entire compound select, - // not just a single subselect. + // Each subselect shares the same limit_ctx and offset, because the LIMIT, OFFSET applies to + // the entire compound select, not just a single subselect. let limit_ctx = limit.map(|limit| { let reg = program.alloc_register(); program.emit_insn(Insn::Integer { @@ -49,6 +50,22 @@ pub fn emit_program_for_compound_select( }); LimitCtx::new_shared(reg) }); + let offset_reg = offset.map(|offset| { + let reg = program.alloc_register(); + program.emit_insn(Insn::Integer { + value: offset as i64, + dest: reg, + }); + + let combined_reg = program.alloc_register(); + program.emit_insn(Insn::OffsetLimit { + offset_reg: reg, + combined_reg, + limit_reg: limit_ctx.unwrap().reg_limit, + }); + + reg + }); // When a compound SELECT is part of a query that yields results to a coroutine (e.g. within an INSERT clause), // we must allocate registers for the result columns to be yielded. Each subselect will then yield to @@ -67,6 +84,7 @@ pub fn emit_program_for_compound_select( schema, syms, limit_ctx, + offset_reg, yield_reg, reg_result_cols_start, )?; @@ -80,12 +98,14 @@ pub fn emit_program_for_compound_select( // Emits bytecode for a compound SELECT statement. This function processes the rightmost part of // the compound SELECT and handles the left parts recursively based on the compound operator type. +#[allow(clippy::too_many_arguments)] fn emit_compound_select( program: &mut ProgramBuilder, plan: Plan, schema: &Schema, syms: &SymbolTable, limit_ctx: Option, + offset_reg: Option, yield_reg: Option, reg_result_cols_start: Option, ) -> crate::Result<()> { @@ -130,6 +150,7 @@ fn emit_compound_select( schema, syms, limit_ctx, + offset_reg, yield_reg, reg_result_cols_start, )?; @@ -144,6 +165,10 @@ fn emit_compound_select( right_most.limit = limit; right_most_ctx.limit_ctx = Some(limit_ctx); } + if offset_reg.is_some() { + right_most.offset = offset; + right_most_ctx.reg_offset = offset_reg; + } emit_query(program, &mut right_most, &mut right_most_ctx)?; program.preassign_label_to_next_insn(label_next_select); } @@ -176,6 +201,7 @@ fn emit_compound_select( schema, syms, None, + None, yield_reg, reg_result_cols_start, )?; @@ -193,6 +219,7 @@ fn emit_compound_select( dedupe_index.0, dedupe_index.1.as_ref(), limit_ctx, + offset_reg, yield_reg, ); } @@ -225,6 +252,7 @@ fn emit_compound_select( schema, syms, None, + None, yield_reg, reg_result_cols_start, )?; @@ -244,6 +272,7 @@ fn emit_compound_select( right_cursor_id, target_cursor_id, limit_ctx, + offset_reg, yield_reg, ); } @@ -276,6 +305,7 @@ fn emit_compound_select( schema, syms, None, + None, yield_reg, reg_result_cols_start, )?; @@ -287,7 +317,7 @@ fn emit_compound_select( emit_query(program, &mut right_most, &mut right_most_ctx)?; if new_index { read_deduplicated_union_or_except_rows( - program, cursor_id, &index, limit_ctx, yield_reg, + program, cursor_id, &index, limit_ctx, offset_reg, yield_reg, ); } } @@ -297,6 +327,10 @@ fn emit_compound_select( right_most_ctx.limit_ctx = Some(limit_ctx); right_most.limit = limit; } + if offset_reg.is_some() { + right_most.offset = offset; + right_most_ctx.reg_offset = offset_reg; + } emit_query(program, &mut right_most, &mut right_most_ctx)?; } } @@ -351,6 +385,7 @@ fn read_deduplicated_union_or_except_rows( dedupe_cursor_id: usize, dedupe_index: &Index, limit_ctx: Option, + offset_reg: Option, yield_reg: Option, ) { let label_close = program.allocate_label(); @@ -362,6 +397,13 @@ fn read_deduplicated_union_or_except_rows( pc_if_empty: label_dedupe_next, }); program.preassign_label_to_next_insn(label_dedupe_loop_start); + if let Some(reg) = offset_reg { + program.emit_insn(Insn::IfPos { + reg, + target_pc: label_dedupe_next, + decrement_by: 1, + }); + } for col_idx in 0..dedupe_index.columns.len() { let start_reg = if let Some(yield_reg) = yield_reg { // Need to reuse the yield_reg for the column being emitted @@ -406,6 +448,7 @@ fn read_deduplicated_union_or_except_rows( } // Emits the bytecode for Reading rows from the intersection of two cursors. +#[allow(clippy::too_many_arguments)] fn read_intersect_rows( program: &mut ProgramBuilder, left_cursor_id: usize, @@ -413,6 +456,7 @@ fn read_intersect_rows( right_cursor_id: usize, target_cursor: Option, limit_ctx: Option, + offset_reg: Option, yield_reg: Option, ) { let label_close = program.allocate_label(); @@ -435,6 +479,13 @@ fn read_intersect_rows( record_reg: row_content_reg, num_regs: 0, }); + if let Some(reg) = offset_reg { + program.emit_insn(Insn::IfPos { + reg, + target_pc: label_next, + decrement_by: 1, + }); + } let column_count = index.columns.len(); let cols_start_reg = if let Some(yield_reg) = yield_reg { yield_reg + 1 diff --git a/core/translate/delete.rs b/core/translate/delete.rs index 02bb345b8..24490f455 100644 --- a/core/translate/delete.rs +++ b/core/translate/delete.rs @@ -25,7 +25,7 @@ pub fn translate_delete( // Let's disable altering a table with indices altogether instead of checking column by // column to be extra safe. crate::bail_parse_error!( - "DELETE for table with indexes is disabled by default. Run with `--experimental-indexes` to enable this feature." + "DELETE for table with indexes is disabled. Omit the `--experimental-indexes=false` flag to enable this feature." ); } diff --git a/core/translate/display.rs b/core/translate/display.rs index 73878f771..21344c025 100644 --- a/core/translate/display.rs +++ b/core/translate/display.rs @@ -557,7 +557,7 @@ impl ToTokens for UpdatePlan { .unwrap(); ast::Set { - col_names: ast::DistinctNames::single(ast::Name::from_str(col_name)), + col_names: ast::Names::single(ast::Name::from_str(col_name)), expr: set_expr.clone(), } }), diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index dcb11fc05..6a2d90661 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -266,7 +266,7 @@ pub fn emit_query<'a>( t_ctx: &mut TranslateCtx<'a>, ) -> Result { if !plan.values.is_empty() { - let reg_result_cols_start = emit_values(program, plan, &t_ctx.resolver, t_ctx.limit_ctx)?; + let reg_result_cols_start = emit_values(program, plan, t_ctx)?; return Ok(reg_result_cols_start); } @@ -825,7 +825,6 @@ fn emit_update_insns( }); // Check if rowid was provided (through INTEGER PRIMARY KEY as a rowid alias) - let rowid_alias_index = table_ref.columns().iter().position(|c| c.is_rowid_alias); let has_user_provided_rowid = if let Some(index) = rowid_alias_index { diff --git a/core/translate/expr.rs b/core/translate/expr.rs index 2d7cdc26b..e90a54fc5 100644 --- a/core/translate/expr.rs +++ b/core/translate/expr.rs @@ -8,7 +8,7 @@ use super::plan::TableReferences; use crate::function::JsonFunc; use crate::function::{Func, FuncCtx, MathFuncArity, ScalarFunc, VectorFunc}; use crate::functions::datetime; -use crate::schema::{Affinity, Table, Type}; +use crate::schema::{affinity, Affinity, Table, Type}; use crate::util::{exprs_are_equivalent, parse_numeric_literal}; use crate::vdbe::builder::CursorKey; use crate::vdbe::{ @@ -141,6 +141,138 @@ macro_rules! expect_arguments_even { }}; } +/// Core implementation of IN expression logic that can be used in both conditional and expression contexts. +/// This follows SQLite's approach where a single core function handles all InList cases. +/// +/// This is extracted from the original conditional implementation to be reusable. +/// The logic exactly matches the original conditional InList implementation. +#[instrument(skip(program, referenced_tables, resolver), level = Level::DEBUG)] +fn translate_in_list( + program: &mut ProgramBuilder, + referenced_tables: Option<&TableReferences>, + lhs: &ast::Expr, + rhs: &Option>, + not: bool, + condition_metadata: ConditionMetadata, + resolver: &Resolver, +) -> Result<()> { + // lhs is e.g. a column reference + // rhs is an Option> + // If rhs is None, it means the IN expression is always false, i.e. tbl.id IN (). + // If rhs is Some, it means the IN expression has a list of values to compare against, e.g. tbl.id IN (1, 2, 3). + // + // The IN expression is equivalent to a series of OR expressions. + // For example, `a IN (1, 2, 3)` is equivalent to `a = 1 OR a = 2 OR a = 3`. + // The NOT IN expression is equivalent to a series of AND expressions. + // For example, `a NOT IN (1, 2, 3)` is equivalent to `a != 1 AND a != 2 AND a != 3`. + // + // SQLite typically optimizes IN expressions to use a binary search on an ephemeral index if there are many values. + // For now we don't have the plumbing to do that, so we'll just emit a series of comparisons, + // which is what SQLite also does for small lists of values. + // TODO: Let's refactor this later to use a more efficient implementation conditionally based on the number of values. + + if rhs.is_none() { + // If rhs is None, IN expressions are always false and NOT IN expressions are always true. + if not { + // On a trivially true NOT IN () expression we can only jump to the 'jump_target_when_true' label if 'jump_if_condition_is_true'; otherwise me must fall through. + // This is because in a more complex condition we might need to evaluate the rest of the condition. + // Note that we are already breaking up our WHERE clauses into a series of terms at "AND" boundaries, so right now we won't be running into cases where jumping on true would be incorrect, + // but once we have e.g. parenthesization and more complex conditions, not having this 'if' here would introduce a bug. + if condition_metadata.jump_if_condition_is_true { + program.emit_insn(Insn::Goto { + target_pc: condition_metadata.jump_target_when_true, + }); + } + } else { + program.emit_insn(Insn::Goto { + target_pc: condition_metadata.jump_target_when_false, + }); + } + return Ok(()); + } + + // The left hand side only needs to be evaluated once we have a list of values to compare against. + let lhs_reg = program.alloc_register(); + let _ = translate_expr(program, referenced_tables, lhs, lhs_reg, resolver)?; + + let rhs = rhs.as_ref().unwrap(); + + // The difference between a local jump and an "upper level" jump is that for example in this case: + // WHERE foo IN (1,2,3) OR bar = 5, + // we can immediately jump to the 'jump_target_when_true' label of the ENTIRE CONDITION if foo = 1, foo = 2, or foo = 3 without evaluating the bar = 5 condition. + // This is why in Binary-OR expressions we set jump_if_condition_is_true to true for the first condition. + // However, in this example: + // WHERE foo IN (1,2,3) AND bar = 5, + // we can't jump to the 'jump_target_when_true' label of the entire condition foo = 1, foo = 2, or foo = 3, because we still need to evaluate the bar = 5 condition later. + // This is why in that case we just jump over the rest of the IN conditions in this "local" branch which evaluates the IN condition. + let jump_target_when_true = if condition_metadata.jump_if_condition_is_true { + condition_metadata.jump_target_when_true + } else { + program.allocate_label() + }; + + if !not { + // If it's an IN expression, we need to jump to the 'jump_target_when_true' label if any of the conditions are true. + for (i, expr) in rhs.iter().enumerate() { + let rhs_reg = program.alloc_register(); + let last_condition = i == rhs.len() - 1; + let _ = translate_expr(program, referenced_tables, expr, rhs_reg, resolver)?; + // If this is not the last condition, we need to jump to the 'jump_target_when_true' label if the condition is true. + if !last_condition { + program.emit_insn(Insn::Eq { + lhs: lhs_reg, + rhs: rhs_reg, + target_pc: jump_target_when_true, + flags: CmpInsFlags::default(), + collation: program.curr_collation(), + }); + } else { + // If this is the last condition, we need to jump to the 'jump_target_when_false' label if there is no match. + program.emit_insn(Insn::Ne { + lhs: lhs_reg, + rhs: rhs_reg, + target_pc: condition_metadata.jump_target_when_false, + flags: CmpInsFlags::default().jump_if_null(), + collation: program.curr_collation(), + }); + } + } + // If we got here, then the last condition was a match, so we jump to the 'jump_target_when_true' label if 'jump_if_condition_is_true'. + // If not, we can just fall through without emitting an unnecessary instruction. + if condition_metadata.jump_if_condition_is_true { + program.emit_insn(Insn::Goto { + target_pc: condition_metadata.jump_target_when_true, + }); + } + } else { + // If it's a NOT IN expression, we need to jump to the 'jump_target_when_false' label if any of the conditions are true. + for expr in rhs.iter() { + let rhs_reg = program.alloc_register(); + let _ = translate_expr(program, referenced_tables, expr, rhs_reg, resolver)?; + program.emit_insn(Insn::Eq { + lhs: lhs_reg, + rhs: rhs_reg, + target_pc: condition_metadata.jump_target_when_false, + flags: CmpInsFlags::default().jump_if_null(), + collation: program.curr_collation(), + }); + } + // If we got here, then none of the conditions were a match, so we jump to the 'jump_target_when_true' label if 'jump_if_condition_is_true'. + // If not, we can just fall through without emitting an unnecessary instruction. + if condition_metadata.jump_if_condition_is_true { + program.emit_insn(Insn::Goto { + target_pc: condition_metadata.jump_target_when_true, + }); + } + } + + if !condition_metadata.jump_if_condition_is_true { + program.preassign_label_to_next_insn(jump_target_when_true); + } + + Ok(()) +} + #[instrument(skip(program, referenced_tables, expr, resolver), level = Level::DEBUG)] pub fn translate_condition_expr( program: &mut ProgramBuilder, @@ -219,121 +351,15 @@ pub fn translate_condition_expr( emit_cond_jump(program, condition_metadata, reg); } ast::Expr::InList { lhs, not, rhs } => { - // lhs is e.g. a column reference - // rhs is an Option> - // If rhs is None, it means the IN expression is always false, i.e. tbl.id IN (). - // If rhs is Some, it means the IN expression has a list of values to compare against, e.g. tbl.id IN (1, 2, 3). - // - // The IN expression is equivalent to a series of OR expressions. - // For example, `a IN (1, 2, 3)` is equivalent to `a = 1 OR a = 2 OR a = 3`. - // The NOT IN expression is equivalent to a series of AND expressions. - // For example, `a NOT IN (1, 2, 3)` is equivalent to `a != 1 AND a != 2 AND a != 3`. - // - // SQLite typically optimizes IN expressions to use a binary search on an ephemeral index if there are many values. - // For now we don't have the plumbing to do that, so we'll just emit a series of comparisons, - // which is what SQLite also does for small lists of values. - // TODO: Let's refactor this later to use a more efficient implementation conditionally based on the number of values. - - if rhs.is_none() { - // If rhs is None, IN expressions are always false and NOT IN expressions are always true. - if *not { - // On a trivially true NOT IN () expression we can only jump to the 'jump_target_when_true' label if 'jump_if_condition_is_true'; otherwise me must fall through. - // This is because in a more complex condition we might need to evaluate the rest of the condition. - // Note that we are already breaking up our WHERE clauses into a series of terms at "AND" boundaries, so right now we won't be running into cases where jumping on true would be incorrect, - // but once we have e.g. parenthesization and more complex conditions, not having this 'if' here would introduce a bug. - if condition_metadata.jump_if_condition_is_true { - program.emit_insn(Insn::Goto { - target_pc: condition_metadata.jump_target_when_true, - }); - } - } else { - program.emit_insn(Insn::Goto { - target_pc: condition_metadata.jump_target_when_false, - }); - } - return Ok(()); - } - - // The left hand side only needs to be evaluated once we have a list of values to compare against. - let lhs_reg = program.alloc_register(); - let _ = translate_expr(program, Some(referenced_tables), lhs, lhs_reg, resolver)?; - - let rhs = rhs.as_ref().unwrap(); - - // The difference between a local jump and an "upper level" jump is that for example in this case: - // WHERE foo IN (1,2,3) OR bar = 5, - // we can immediately jump to the 'jump_target_when_true' label of the ENTIRE CONDITION if foo = 1, foo = 2, or foo = 3 without evaluating the bar = 5 condition. - // This is why in Binary-OR expressions we set jump_if_condition_is_true to true for the first condition. - // However, in this example: - // WHERE foo IN (1,2,3) AND bar = 5, - // we can't jump to the 'jump_target_when_true' label of the entire condition foo = 1, foo = 2, or foo = 3, because we still need to evaluate the bar = 5 condition later. - // This is why in that case we just jump over the rest of the IN conditions in this "local" branch which evaluates the IN condition. - let jump_target_when_true = if condition_metadata.jump_if_condition_is_true { - condition_metadata.jump_target_when_true - } else { - program.allocate_label() - }; - - if !*not { - // If it's an IN expression, we need to jump to the 'jump_target_when_true' label if any of the conditions are true. - for (i, expr) in rhs.iter().enumerate() { - let rhs_reg = program.alloc_register(); - let last_condition = i == rhs.len() - 1; - let _ = - translate_expr(program, Some(referenced_tables), expr, rhs_reg, resolver)?; - // If this is not the last condition, we need to jump to the 'jump_target_when_true' label if the condition is true. - if !last_condition { - program.emit_insn(Insn::Eq { - lhs: lhs_reg, - rhs: rhs_reg, - target_pc: jump_target_when_true, - flags: CmpInsFlags::default(), - collation: program.curr_collation(), - }); - } else { - // If this is the last condition, we need to jump to the 'jump_target_when_false' label if there is no match. - program.emit_insn(Insn::Ne { - lhs: lhs_reg, - rhs: rhs_reg, - target_pc: condition_metadata.jump_target_when_false, - flags: CmpInsFlags::default().jump_if_null(), - collation: program.curr_collation(), - }); - } - } - // If we got here, then the last condition was a match, so we jump to the 'jump_target_when_true' label if 'jump_if_condition_is_true'. - // If not, we can just fall through without emitting an unnecessary instruction. - if condition_metadata.jump_if_condition_is_true { - program.emit_insn(Insn::Goto { - target_pc: condition_metadata.jump_target_when_true, - }); - } - } else { - // If it's a NOT IN expression, we need to jump to the 'jump_target_when_false' label if any of the conditions are true. - for expr in rhs.iter() { - let rhs_reg = program.alloc_register(); - let _ = - translate_expr(program, Some(referenced_tables), expr, rhs_reg, resolver)?; - program.emit_insn(Insn::Eq { - lhs: lhs_reg, - rhs: rhs_reg, - target_pc: condition_metadata.jump_target_when_false, - flags: CmpInsFlags::default().jump_if_null(), - collation: program.curr_collation(), - }); - } - // If we got here, then none of the conditions were a match, so we jump to the 'jump_target_when_true' label if 'jump_if_condition_is_true'. - // If not, we can just fall through without emitting an unnecessary instruction. - if condition_metadata.jump_if_condition_is_true { - program.emit_insn(Insn::Goto { - target_pc: condition_metadata.jump_target_when_true, - }); - } - } - - if !condition_metadata.jump_if_condition_is_true { - program.preassign_label_to_next_insn(jump_target_when_true); - } + translate_in_list( + program, + Some(referenced_tables), + lhs, + rhs, + *not, + condition_metadata, + resolver, + )?; } ast::Expr::Like { not, .. } => { let cur_reg = program.alloc_register(); @@ -651,24 +677,11 @@ pub fn translate_expr( } ast::Expr::Cast { expr, type_name } => { let type_name = type_name.as_ref().unwrap(); // TODO: why is this optional? - let reg_expr = program.alloc_registers(2); - translate_expr(program, referenced_tables, expr, reg_expr, resolver)?; - program.emit_insn(Insn::String8 { - // we make a comparison against uppercase static strs in the affinity() function, - // so we need to make sure we're comparing against the uppercase version, - // and it's better to do this once instead of every time we check affinity - value: type_name.name.to_uppercase(), - dest: reg_expr + 1, - }); - program.mark_last_insn_constant(); - program.emit_insn(Insn::Function { - constant_mask: 0, - start_reg: reg_expr, - dest: target_register, - func: FuncCtx { - func: Func::Scalar(ScalarFunc::Cast), - arg_count: 2, - }, + translate_expr(program, referenced_tables, expr, target_register, resolver)?; + let type_affinity = affinity(&type_name.name.to_uppercase()); + program.emit_insn(Insn::Cast { + reg: target_register, + affinity: type_affinity, }); Ok(target_register) } @@ -2060,7 +2073,61 @@ pub fn translate_expr( } Ok(target_register) } - ast::Expr::InList { .. } => todo!(), + ast::Expr::InList { lhs, rhs, not } => { + // Following SQLite's approach: use the same core logic as conditional InList, + // but wrap it with appropriate expression context handling + let result_reg = target_register; + + // Set result to NULL initially (matches SQLite behavior) + program.emit_insn(Insn::Null { + dest: result_reg, + dest_end: None, + }); + + let dest_if_false = program.allocate_label(); + let label_integer_conversion = program.allocate_label(); + + // Call the core InList logic with expression-appropriate condition metadata + translate_in_list( + program, + referenced_tables, + lhs, + rhs, + *not, + ConditionMetadata { + jump_if_condition_is_true: false, + jump_target_when_true: label_integer_conversion, // will be resolved below + jump_target_when_false: dest_if_false, + }, + resolver, + )?; + + // condition true: set result to 1 + program.emit_insn(Insn::Integer { + value: 1, + dest: result_reg, + }); + program.emit_insn(Insn::Goto { + target_pc: label_integer_conversion, + }); + + // False path: set result to 0 + program.resolve_label(dest_if_false, program.offset()); + program.emit_insn(Insn::Integer { + value: 0, + dest: result_reg, + }); + + program.resolve_label(label_integer_conversion, program.offset()); + + // Force integer conversion with AddImm 0 + program.emit_insn(Insn::AddImm { + register: result_reg, + value: 0, + }); + + Ok(result_reg) + } ast::Expr::InSelect { .. } => todo!(), ast::Expr::InTable { .. } => todo!(), ast::Expr::IsNull(expr) => { diff --git a/core/translate/insert.rs b/core/translate/insert.rs index acbf10649..6448d8d21 100644 --- a/core/translate/insert.rs +++ b/core/translate/insert.rs @@ -67,7 +67,7 @@ pub fn translate_insert( // Let's disable altering a table with indices altogether instead of checking column by // column to be extra safe. crate::bail_parse_error!( - "INSERT to table with indexes is disabled by default. Run with `--experimental-indexes` to enable this feature." + "INSERT to table with indexes is disabled. Omit the `--experimental-indexes=false` flag to enable this feature." ); } let table_name = &tbl_name.name; diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index 3fb9ec21a..a2ab74331 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -113,7 +113,7 @@ pub fn emit_order_by( }); program.preassign_label_to_next_insn(sort_loop_start_label); - emit_offset(program, plan, sort_loop_next_label, t_ctx.reg_offset)?; + emit_offset(program, plan, sort_loop_next_label, t_ctx.reg_offset); program.emit_insn(Insn::SorterData { cursor_id: sort_cursor, diff --git a/core/translate/pragma.rs b/core/translate/pragma.rs index 4c3c64d58..a9f1b3729 100644 --- a/core/translate/pragma.rs +++ b/core/translate/pragma.rs @@ -10,18 +10,17 @@ use turso_sqlite3_parser::ast::{PragmaName, QualifiedName}; use crate::pragma::pragma_for; use crate::schema::Schema; use crate::storage::pager::AutoVacuumMode; -use crate::storage::sqlite3_ondisk::{DatabaseEncoding, MIN_PAGE_CACHE_SIZE}; +use crate::storage::sqlite3_ondisk::CacheSize; use crate::storage::wal::CheckpointMode; use crate::translate::schema::translate_create_table; -use crate::util::{normalize_ident, parse_signed_number, parse_string}; +use crate::util::{normalize_ident, parse_signed_number, parse_string, IOExt as _}; use crate::vdbe::builder::{ProgramBuilder, ProgramBuilderOpts}; use crate::vdbe::insn::{Cookie, Insn}; -use crate::{bail_parse_error, storage, CaptureDataChangesMode, LimboError, Value}; +use crate::{bail_parse_error, CaptureDataChangesMode, LimboError, Value}; use std::str::FromStr; use strum::IntoEnumIterator; use super::integrity_check::translate_integrity_check; -use crate::storage::header_accessor; use crate::storage::pager::Pager; use crate::translate::emitter::TransactionMode; @@ -311,15 +310,12 @@ fn query_pragma( Ok((program, TransactionMode::None)) } PragmaName::Encoding => { - let encoding: &str = if !pager.db_state.is_initialized() { - DatabaseEncoding::Utf8 - } else { - let encoding: DatabaseEncoding = - header_accessor::get_text_encoding(&pager)?.try_into()?; - encoding - } - .into(); - program.emit_string8(encoding.into(), register); + let encoding = pager + .io + .block(|| pager.with_header(|header| header.text_encoding)) + .unwrap_or_default() + .to_string(); + program.emit_string8(encoding, register); program.emit_result_row(register, 1); program.add_pragma_result_column(pragma.to_string()); Ok((program, TransactionMode::None)) @@ -433,7 +429,10 @@ fn query_pragma( } PragmaName::PageSize => { program.emit_int( - header_accessor::get_page_size(&pager).unwrap_or(connection.get_page_size()) as i64, + pager + .io + .block(|| pager.with_header(|header| header.page_size.get())) + .unwrap_or(connection.get_page_size()) as i64, register, ); program.emit_result_row(register, 1); @@ -484,7 +483,11 @@ fn update_auto_vacuum_mode( largest_root_page_number: u32, pager: Rc, ) -> crate::Result<()> { - header_accessor::set_vacuum_mode_largest_root_page(&pager, largest_root_page_number)?; + pager.io.block(|| { + pager.with_header_mut(|header| { + header.vacuum_mode_largest_root_page = largest_root_page_number.into() + }) + })?; pager.set_auto_vacuum_mode(auto_vacuum_mode); Ok(()) } @@ -498,8 +501,11 @@ fn update_cache_size( let mut cache_size = if cache_size_unformatted < 0 { let kb = cache_size_unformatted.abs().saturating_mul(1024); - let page_size = header_accessor::get_page_size(&pager) - .unwrap_or(storage::sqlite3_ondisk::DEFAULT_PAGE_SIZE) as i64; + let page_size = pager + .io + .block(|| pager.with_header(|header| header.page_size)) + .unwrap_or_default() + .get() as i64; if page_size == 0 { return Err(LimboError::InternalError( "Page size cannot be zero".to_string(), @@ -510,10 +516,7 @@ fn update_cache_size( value }; - // SQLite uses this value as threshold for maximum cache size - const MAX_SAFE_CACHE_SIZE: i64 = 2147450880; - - if cache_size > MAX_SAFE_CACHE_SIZE { + if cache_size > CacheSize::MAX_SAFE { cache_size = 0; cache_size_unformatted = 0; } @@ -523,19 +526,17 @@ fn update_cache_size( cache_size_unformatted = 0; } - let cache_size_usize = cache_size as usize; - - let final_cache_size = if cache_size_usize < MIN_PAGE_CACHE_SIZE { - cache_size_unformatted = MIN_PAGE_CACHE_SIZE as i64; - MIN_PAGE_CACHE_SIZE + let final_cache_size = if cache_size < CacheSize::MIN { + cache_size_unformatted = CacheSize::MIN; + CacheSize::MIN } else { - cache_size_usize + cache_size }; connection.set_cache_size(cache_size_unformatted as i32); pager - .change_page_cache_size(final_cache_size) + .change_page_cache_size(final_cache_size as usize) .map_err(|e| LimboError::InternalError(format!("Failed to update page cache size: {e}")))?; Ok(()) diff --git a/core/translate/result_row.rs b/core/translate/result_row.rs index 4a7b78890..f2b722988 100644 --- a/core/translate/result_row.rs +++ b/core/translate/result_row.rs @@ -30,7 +30,7 @@ pub fn emit_select_result( limit_ctx: Option, ) -> Result<()> { if let (Some(jump_to), Some(_)) = (offset_jump_to, label_on_limit_reached) { - emit_offset(program, plan, jump_to, reg_offset)?; + emit_offset(program, plan, jump_to, reg_offset); } let start_reg = reg_result_cols_start; @@ -163,7 +163,7 @@ pub fn emit_offset( plan: &SelectPlan, jump_to: BranchOffset, reg_offset: Option, -) -> Result<()> { +) { match plan.offset { Some(offset) if offset > 0 => { program.add_comment(program.offset(), "OFFSET"); @@ -175,5 +175,4 @@ pub fn emit_offset( } _ => {} } - Ok(()) } diff --git a/core/translate/schema.rs b/core/translate/schema.rs index 9ee88125d..ef3739644 100644 --- a/core/translate/schema.rs +++ b/core/translate/schema.rs @@ -626,7 +626,7 @@ pub fn translate_drop_table( ) -> Result { if !schema.indexes_enabled() && schema.table_has_indexes(&tbl_name.name.to_string()) { bail_parse_error!( - "DROP TABLE with indexes on the table is disabled by default. Run with `--experimental-indexes` to enable this feature." + "DROP TABLE with indexes on the table is disabled by default. Omit the `--experimental-indexes=false` flag to enable this feature." ); } let opts = ProgramBuilderOpts { diff --git a/core/translate/select.rs b/core/translate/select.rs index bec600890..0c4888d45 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -154,10 +154,6 @@ pub fn prepare_select_plan( } let (limit, offset) = select.limit.map_or(Ok((None, None)), |l| parse_limit(&l))?; - // FIXME: handle OFFSET for compound selects - if offset.is_some_and(|o| o > 0) { - crate::bail_parse_error!("OFFSET is not supported for compound SELECTs yet"); - } // FIXME: handle ORDER BY for compound selects if select.order_by.is_some() { crate::bail_parse_error!("ORDER BY is not supported for compound SELECTs yet"); diff --git a/core/translate/update.rs b/core/translate/update.rs index 72a0d29b3..285446160 100644 --- a/core/translate/update.rs +++ b/core/translate/update.rs @@ -110,7 +110,7 @@ pub fn prepare_update_plan( // Let's disable altering a table with indices altogether instead of checking column by // column to be extra safe. bail_parse_error!( - "UPDATE table disabled for table with indexes is disabled by default. Run with `--experimental-indexes` to enable this feature." + "UPDATE table disabled for table with indexes is disabled. Omit the `--experimental-indexes=false` flag to enable this feature." ); } let table = match schema.get_table(table_name.as_str()) { @@ -156,18 +156,38 @@ pub fn prepare_update_plan( .collect(); let mut set_clauses = Vec::with_capacity(body.sets.len()); - for set in &mut body.sets { - let ident = normalize_ident(set.col_names[0].as_str()); - let Some(col_index) = column_lookup.get(&ident) else { - bail_parse_error!("no such column: {}", ident); - }; + // Process each SET assignment and map column names to expressions + // e.g the statement `SET x = 1, y = 2, z = 3` has 3 set assigments + for set in &mut body.sets { bind_column_references(&mut set.expr, &mut table_references, None, connection)?; - if let Some(idx) = set_clauses.iter().position(|(idx, _)| *idx == *col_index) { - set_clauses[idx].1 = set.expr.clone(); - } else { - set_clauses.push((*col_index, set.expr.clone())); + let values = match &set.expr { + Expr::Parenthesized(vals) => vals.clone(), + expr => vec![expr.clone()], + }; + + if set.col_names.len() != values.len() { + bail_parse_error!( + "{} columns assigned {} values", + set.col_names.len(), + values.len() + ); + } + + // Map each column to its corresponding expression + for (col_name, expr) in set.col_names.iter().zip(values.iter()) { + let ident = normalize_ident(col_name.as_str()); + let col_index = match column_lookup.get(&ident) { + Some(idx) => idx, + None => bail_parse_error!("no such column: {}", ident), + }; + + // Update existing entry or add new one + match set_clauses.iter_mut().find(|(idx, _)| idx == col_index) { + Some((_, existing_expr)) => *existing_expr = expr.clone(), + None => set_clauses.push((*col_index, expr.clone())), + } } } diff --git a/core/translate/values.rs b/core/translate/values.rs index 73a33d5eb..8315290e6 100644 --- a/core/translate/values.rs +++ b/core/translate/values.rs @@ -1,6 +1,7 @@ -use crate::translate::emitter::{LimitCtx, Resolver}; +use crate::translate::emitter::{Resolver, TranslateCtx}; use crate::translate::expr::{translate_expr_no_constant_opt, NoConstantOptReason}; use crate::translate::plan::{QueryDestination, SelectPlan}; +use crate::translate::result_row::emit_offset; use crate::vdbe::builder::ProgramBuilder; use crate::vdbe::insn::{IdxInsertFlags, Insn}; use crate::vdbe::BranchOffset; @@ -9,22 +10,19 @@ use crate::Result; pub fn emit_values( program: &mut ProgramBuilder, plan: &SelectPlan, - resolver: &Resolver, - limit_ctx: Option, + t_ctx: &TranslateCtx, ) -> Result { if plan.values.len() == 1 { - let start_reg = emit_values_when_single_row(program, plan, resolver, limit_ctx)?; + let start_reg = emit_values_when_single_row(program, plan, t_ctx)?; return Ok(start_reg); } let reg_result_cols_start = match plan.query_destination { - QueryDestination::ResultRows => emit_toplevel_values(program, plan, resolver, limit_ctx)?, + QueryDestination::ResultRows => emit_toplevel_values(program, plan, t_ctx)?, QueryDestination::CoroutineYield { yield_reg, .. } => { - emit_values_in_subquery(program, plan, resolver, yield_reg)? - } - QueryDestination::EphemeralIndex { .. } => { - emit_toplevel_values(program, plan, resolver, limit_ctx)? + emit_values_in_subquery(program, plan, &t_ctx.resolver, yield_reg)? } + QueryDestination::EphemeralIndex { .. } => emit_toplevel_values(program, plan, t_ctx)?, QueryDestination::EphemeralTable { .. } => unreachable!(), }; Ok(reg_result_cols_start) @@ -33,9 +31,10 @@ pub fn emit_values( fn emit_values_when_single_row( program: &mut ProgramBuilder, plan: &SelectPlan, - resolver: &Resolver, - limit_ctx: Option, + t_ctx: &TranslateCtx, ) -> Result { + let end_label = program.allocate_label(); + emit_offset(program, plan, end_label, t_ctx.reg_offset); let first_row = &plan.values[0]; let row_len = first_row.len(); let start_reg = program.alloc_registers(row_len); @@ -45,12 +44,11 @@ fn emit_values_when_single_row( None, v, start_reg + i, - resolver, + &t_ctx.resolver, NoConstantOptReason::RegisterReuse, )?; } - let end_label = program.allocate_label(); - emit_values_to_destination(program, plan, start_reg, row_len, limit_ctx, end_label); + emit_values_to_destination(program, plan, t_ctx, start_reg, row_len, end_label); program.preassign_label_to_next_insn(end_label); Ok(start_reg) } @@ -58,8 +56,7 @@ fn emit_values_when_single_row( fn emit_toplevel_values( program: &mut ProgramBuilder, plan: &SelectPlan, - resolver: &Resolver, - limit_ctx: Option, + t_ctx: &TranslateCtx, ) -> Result { let yield_reg = program.alloc_register(); let definition_label = program.allocate_label(); @@ -71,7 +68,7 @@ fn emit_toplevel_values( }); program.preassign_label_to_next_insn(start_offset_label); - let start_reg = emit_values_in_subquery(program, plan, resolver, yield_reg)?; + let start_reg = emit_values_in_subquery(program, plan, &t_ctx.resolver, yield_reg)?; program.emit_insn(Insn::EndCoroutine { yield_reg }); program.preassign_label_to_next_insn(definition_label); @@ -82,12 +79,15 @@ fn emit_toplevel_values( start_offset: start_offset_label, }); let end_label = program.allocate_label(); - let goto_label = program.allocate_label(); - program.preassign_label_to_next_insn(goto_label); + let yield_label = program.allocate_label(); + program.preassign_label_to_next_insn(yield_label); program.emit_insn(Insn::Yield { yield_reg, end_offset: end_label, }); + + let goto_label = program.allocate_label(); + emit_offset(program, plan, goto_label, t_ctx.reg_offset); let row_len = plan.values[0].len(); let copy_start_reg = program.alloc_registers(row_len); for i in 0..row_len { @@ -98,10 +98,11 @@ fn emit_toplevel_values( }); } - emit_values_to_destination(program, plan, copy_start_reg, row_len, limit_ctx, end_label); + emit_values_to_destination(program, plan, t_ctx, copy_start_reg, row_len, end_label); + program.preassign_label_to_next_insn(goto_label); program.emit_insn(Insn::Goto { - target_pc: goto_label, + target_pc: yield_label, }); program.preassign_label_to_next_insn(end_label); @@ -139,9 +140,9 @@ fn emit_values_in_subquery( fn emit_values_to_destination( program: &mut ProgramBuilder, plan: &SelectPlan, + t_ctx: &TranslateCtx, start_reg: usize, row_len: usize, - limit_ctx: Option, end_label: BranchOffset, ) { match &plan.query_destination { @@ -150,7 +151,7 @@ fn emit_values_to_destination( start_reg, count: row_len, }); - if let Some(limit_ctx) = limit_ctx { + if let Some(limit_ctx) = t_ctx.limit_ctx { program.emit_insn(Insn::DecrJumpZero { reg: limit_ctx.reg_limit, target_pc: end_label, diff --git a/core/types.rs b/core/types.rs index 2adfd30de..b3f118fb6 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1156,6 +1156,12 @@ impl ImmutableRecord { Err(_) => None, } } + + pub fn column_count(&self) -> usize { + let mut cursor = RecordCursor::new(); + cursor.parse_full_header(self).unwrap(); + cursor.offsets.len() + } } /// A cursor for lazily parsing SQLite record format data. diff --git a/core/util.rs b/core/util.rs index 961288afe..5534759d1 100644 --- a/core/util.rs +++ b/core/util.rs @@ -1,5 +1,4 @@ #![allow(unused)] -use crate::storage::header_accessor::get_schema_cookie; use crate::translate::expr::WalkControl; use crate::types::IOResult; use crate::{ diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index fa784de1b..0ae41e388 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -2,18 +2,16 @@ use crate::function::AlterTableFunc; use crate::numeric::{NullableInteger, Numeric}; use crate::storage::btree::{integrity_check, IntegrityCheckError, IntegrityCheckState}; -use crate::storage::database::FileMemoryStorage; +use crate::storage::database::DatabaseFile; use crate::storage::page_cache::DumbLruPageCache; use crate::storage::pager::{AtomicDbState, CreateBTreeFlags, DbState}; use crate::storage::sqlite3_ondisk::read_varint; -use crate::storage::wal::DummyWAL; -use crate::storage::{self, header_accessor}; use crate::translate::collate::CollationSeq; use crate::types::{ compare_immutable, compare_records_generic, Extendable, ImmutableRecord, RawSlice, SeekResult, Text, TextRef, TextSubtype, }; -use crate::util::normalize_ident; +use crate::util::{normalize_ident, IOExt as _}; use crate::vdbe::insn::InsertFlags; use crate::vdbe::registers_to_ref_values; use crate::vector::{vector_concat, vector_slice}; @@ -29,8 +27,8 @@ use crate::{ }, printf::exec_printf, }, - IO, }; +use std::env::temp_dir; use std::ops::DerefMut; use std::{ borrow::BorrowMut, @@ -115,7 +113,7 @@ pub type InsnFunction = fn( &mut ProgramState, &Insn, &Rc, - Option<&Rc>, + Option<&Arc>, ) -> Result; pub enum InsnFunctionStepResult { @@ -144,7 +142,7 @@ pub fn op_init( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Init { target_pc } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -159,7 +157,7 @@ pub fn op_add( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Add { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -178,7 +176,7 @@ pub fn op_subtract( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Subtract { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -197,7 +195,7 @@ pub fn op_multiply( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Multiply { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -216,7 +214,7 @@ pub fn op_divide( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Divide { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -235,7 +233,7 @@ pub fn op_drop_index( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::DropIndex { index, db: _ } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -252,7 +250,7 @@ pub fn op_remainder( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Remainder { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -271,7 +269,7 @@ pub fn op_bit_and( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::BitAnd { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -290,7 +288,7 @@ pub fn op_bit_or( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::BitOr { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -309,7 +307,7 @@ pub fn op_bit_not( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::BitNot { reg, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -325,7 +323,7 @@ pub fn op_checkpoint( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Checkpoint { database: _, @@ -335,6 +333,13 @@ pub fn op_checkpoint( else { unreachable!("unexpected Insn {:?}", insn) }; + if !program.connection.auto_commit.get() { + // TODO: sqlite returns "Runtime error: database table is locked (6)" when a table is in use + // when a checkpoint is attempted. We don't have table locks, so return TableLocked for any + // attempt to checkpoint in an interactive transaction. This does not end the transaction, + // however. + return Err(LimboError::TableLocked); + } let result = program.connection.checkpoint(*checkpoint_mode); match result { Ok(CheckpointResult { @@ -363,7 +368,7 @@ pub fn op_null( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { match insn { Insn::Null { dest, dest_end } | Insn::BeginSubrtn { dest, dest_end } => { @@ -386,7 +391,7 @@ pub fn op_null_row( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::NullRow { cursor_id } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -405,7 +410,7 @@ pub fn op_compare( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Compare { start_reg_a, @@ -451,7 +456,7 @@ pub fn op_jump( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Jump { target_pc_lt, @@ -484,7 +489,7 @@ pub fn op_move( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Move { source_reg, @@ -512,7 +517,7 @@ pub fn op_if_pos( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IfPos { reg, @@ -547,7 +552,7 @@ pub fn op_not_null( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::NotNull { reg, target_pc } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -634,7 +639,7 @@ pub fn op_comparison( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let (lhs, rhs, target_pc, flags, collation, op) = match insn { Insn::Eq { @@ -841,7 +846,7 @@ pub fn op_if( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::If { reg, @@ -868,7 +873,7 @@ pub fn op_if_not( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IfNot { reg, @@ -895,7 +900,7 @@ pub fn op_open_read( state: &mut ProgramState, insn: &Insn, _pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::OpenRead { cursor_id, @@ -914,7 +919,7 @@ pub fn op_open_read( let table_id = *root_page as u64; let mv_store = mv_store.unwrap().clone(); let mv_cursor = Rc::new(RefCell::new( - MvCursor::new(mv_store.clone(), tx_id, table_id).unwrap(), + MvCursor::new(mv_store, tx_id, table_id, pager.clone()).unwrap(), )); Some(mv_cursor) } @@ -967,7 +972,7 @@ pub fn op_vopen( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::VOpen { cursor_id } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -992,7 +997,7 @@ pub fn op_vcreate( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::VCreate { module_name, @@ -1033,7 +1038,7 @@ pub fn op_vfilter( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::VFilter { cursor_id, @@ -1073,7 +1078,7 @@ pub fn op_vcolumn( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::VColumn { cursor_id, @@ -1098,7 +1103,7 @@ pub fn op_vupdate( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::VUpdate { cursor_id, @@ -1162,7 +1167,7 @@ pub fn op_vnext( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::VNext { cursor_id, @@ -1189,7 +1194,7 @@ pub fn op_vdestroy( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::VDestroy { db, table_name } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -1213,7 +1218,7 @@ pub fn op_open_pseudo( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::OpenPseudo { cursor_id, @@ -1240,7 +1245,7 @@ pub fn op_rewind( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Rewind { cursor_id, @@ -1269,7 +1274,7 @@ pub fn op_last( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Last { cursor_id, @@ -1390,7 +1395,7 @@ pub fn op_column( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Column { cursor_id, @@ -1685,7 +1690,7 @@ pub fn op_type_check( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::TypeCheck { start_reg, @@ -1747,7 +1752,7 @@ pub fn op_make_record( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::MakeRecord { start_reg, @@ -1769,7 +1774,7 @@ pub fn op_result_row( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::ResultRow { start_reg, count } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -1788,7 +1793,7 @@ pub fn op_next( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Next { cursor_id, @@ -1819,7 +1824,7 @@ pub fn op_prev( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Prev { cursor_id, @@ -1849,7 +1854,7 @@ pub fn halt( program: &Program, state: &mut ProgramState, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, err_code: usize, description: &str, ) -> Result { @@ -1889,7 +1894,7 @@ pub fn op_halt( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Halt { err_code, @@ -1940,7 +1945,7 @@ pub fn op_halt_if_null( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::HaltIfNull { target_reg, @@ -1963,7 +1968,7 @@ pub fn op_transaction( state: &mut ProgramState, insn: &Insn, _pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Transaction { db, write } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -1979,7 +1984,7 @@ pub fn op_transaction( if state.mv_tx_id.is_none() { // We allocate the first page lazily in the first transaction. return_if_io!(pager.maybe_allocate_page1()); - let tx_id = mv_store.begin_tx(); + let tx_id = mv_store.begin_tx(pager.clone()); conn.mv_transactions.borrow_mut().push(tx_id); state.mv_tx_id = Some(tx_id); } @@ -2061,7 +2066,7 @@ pub fn op_auto_commit( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::AutoCommit { auto_commit, @@ -2115,7 +2120,7 @@ pub fn op_goto( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Goto { target_pc } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2130,7 +2135,7 @@ pub fn op_gosub( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Gosub { target_pc, @@ -2150,7 +2155,7 @@ pub fn op_return( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Return { return_reg, @@ -2180,7 +2185,7 @@ pub fn op_integer( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Integer { value, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2195,7 +2200,7 @@ pub fn op_real( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Real { value, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2210,7 +2215,7 @@ pub fn op_real_affinity( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::RealAffinity { register } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2227,7 +2232,7 @@ pub fn op_string8( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::String8 { value, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2242,7 +2247,7 @@ pub fn op_blob( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Blob { value, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2257,7 +2262,7 @@ pub fn op_row_data( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::RowData { cursor_id, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2288,7 +2293,7 @@ pub fn op_row_id( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::RowId { cursor_id, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2353,7 +2358,7 @@ pub fn op_idx_row_id( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IdxRowId { cursor_id, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -2375,7 +2380,7 @@ pub fn op_seek_rowid( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::SeekRowid { cursor_id, @@ -2431,7 +2436,7 @@ pub fn op_deferred_seek( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::DeferredSeek { index_cursor_id, @@ -2473,7 +2478,7 @@ pub fn op_seek( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let (cursor_id, is_index, record_source, target_pc) = match insn { Insn::SeekGE { @@ -2579,7 +2584,7 @@ pub fn seek_internal( program: &Program, state: &mut ProgramState, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, record_source: RecordSource, cursor_id: usize, is_index: bool, @@ -2590,7 +2595,7 @@ pub fn seek_internal( program: &Program, state: &mut ProgramState, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, record_source: RecordSource, cursor_id: usize, is_index: bool, @@ -2856,7 +2861,7 @@ pub fn op_idx_ge( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IdxGE { cursor_id, @@ -2907,7 +2912,7 @@ pub fn op_seek_end( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { if let Insn::SeekEnd { cursor_id } = *insn { let mut cursor = state.get_cursor(cursor_id); @@ -2926,7 +2931,7 @@ pub fn op_idx_le( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IdxLE { cursor_id, @@ -2977,7 +2982,7 @@ pub fn op_idx_gt( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IdxGT { cursor_id, @@ -3028,7 +3033,7 @@ pub fn op_idx_lt( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IdxLT { cursor_id, @@ -3079,7 +3084,7 @@ pub fn op_decr_jump_zero( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::DecrJumpZero { reg, target_pc } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -3132,7 +3137,7 @@ pub fn op_agg_step( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::AggStep { acc_reg, @@ -3437,7 +3442,7 @@ pub fn op_agg_final( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::AggFinal { register, func } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -3577,7 +3582,7 @@ pub fn op_sorter_open( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::SorterOpen { cursor_id, @@ -3590,8 +3595,12 @@ pub fn op_sorter_open( }; let cache_size = program.connection.get_cache_size(); // Set the buffer size threshold to be roughly the same as the limit configured for the page-cache. - let page_size = header_accessor::get_page_size(pager) - .unwrap_or(storage::sqlite3_ondisk::DEFAULT_PAGE_SIZE) as usize; + let page_size = pager + .io + .block(|| pager.with_header(|header| header.page_size)) + .unwrap_or_default() + .get() as usize; + let max_buffer_size_bytes = if cache_size < 0 { (cache_size.abs() * 1024) as usize } else { @@ -3621,7 +3630,7 @@ pub fn op_sorter_data( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::SorterData { cursor_id, @@ -3657,7 +3666,7 @@ pub fn op_sorter_insert( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::SorterInsert { cursor_id, @@ -3684,7 +3693,7 @@ pub fn op_sorter_sort( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::SorterSort { cursor_id, @@ -3717,7 +3726,7 @@ pub fn op_sorter_next( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::SorterNext { cursor_id, @@ -3748,7 +3757,7 @@ pub fn op_function( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Function { constant_mask, @@ -4342,7 +4351,8 @@ pub fn op_function( } } ScalarFunc::SqliteVersion => { - let version_integer: i64 = header_accessor::get_version_number(pager)? as i64; + let version_integer = + return_if_io!(pager.with_header(|header| header.version_number)).get() as i64; let version = execute_sqlite_version(version_integer); state.registers[*dest] = Register::Value(Value::build_text(version)); } @@ -4965,7 +4975,7 @@ pub fn op_init_coroutine( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::InitCoroutine { yield_reg, @@ -4993,7 +5003,7 @@ pub fn op_end_coroutine( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::EndCoroutine { yield_reg } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -5015,7 +5025,7 @@ pub fn op_yield( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Yield { yield_reg, @@ -5058,7 +5068,7 @@ pub fn op_insert( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Insert { cursor: cursor_id, @@ -5139,7 +5149,7 @@ pub fn op_int_64( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Int64 { _p1, @@ -5160,7 +5170,7 @@ pub fn op_delete( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Delete { cursor_id } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -5187,7 +5197,7 @@ pub fn op_idx_delete( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IdxDelete { cursor_id, @@ -5294,7 +5304,7 @@ pub fn op_idx_insert( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IdxInsert { cursor_id, @@ -5442,7 +5452,7 @@ pub fn op_new_rowid( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::NewRowid { cursor, rowid_reg, .. @@ -5573,7 +5583,7 @@ pub fn op_must_be_int( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::MustBeInt { reg } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -5608,7 +5618,7 @@ pub fn op_soft_null( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::SoftNull { reg } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -5627,7 +5637,7 @@ pub fn op_no_conflict( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::NoConflict { cursor_id, @@ -5709,7 +5719,7 @@ pub fn op_not_exists( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::NotExists { cursor, @@ -5737,7 +5747,7 @@ pub fn op_offset_limit( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::OffsetLimit { limit_reg, @@ -5782,7 +5792,7 @@ pub fn op_open_write( state: &mut ProgramState, insn: &Insn, _pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::OpenWrite { cursor_id, @@ -5819,7 +5829,7 @@ pub fn op_open_write( let table_id = root_page; let mv_store = mv_store.unwrap().clone(); let mv_cursor = Rc::new(RefCell::new( - MvCursor::new(mv_store.clone(), tx_id, table_id).unwrap(), + MvCursor::new(mv_store.clone(), tx_id, table_id, pager.clone()).unwrap(), )); Some(mv_cursor) } @@ -5866,7 +5876,7 @@ pub fn op_copy( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Copy { src_reg, @@ -5888,7 +5898,7 @@ pub fn op_create_btree( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::CreateBtree { db, root, flags } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -5915,7 +5925,7 @@ pub fn op_destroy( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Destroy { root, @@ -5944,7 +5954,7 @@ pub fn op_drop_table( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::DropTable { db, table_name, .. } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -5968,7 +5978,7 @@ pub fn op_close( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Close { cursor_id } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -5984,7 +5994,7 @@ pub fn op_is_null( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IsNull { reg, target_pc } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6002,7 +6012,7 @@ pub fn op_page_count( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::PageCount { db, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6011,8 +6021,12 @@ pub fn op_page_count( // TODO: implement temp databases todo!("temp databases not implemented yet"); } - let count = header_accessor::get_database_size(pager).unwrap_or(0); - state.registers[*dest] = Register::Value(Value::Integer(count as i64)); + let count = match pager.with_header(|header| header.database_size.get()) { + Err(_) => 0.into(), + Ok(IOResult::Done(v)) => v.into(), + Ok(IOResult::IO) => return Ok(InsnFunctionStepResult::IO), + }; + state.registers[*dest] = Register::Value(Value::Integer(count)); state.pc += 1; Ok(InsnFunctionStepResult::Step) } @@ -6022,7 +6036,7 @@ pub fn op_parse_schema( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::ParseSchema { db: _, @@ -6062,7 +6076,7 @@ pub fn op_read_cookie( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::ReadCookie { db, dest, cookie } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6071,15 +6085,19 @@ pub fn op_read_cookie( // TODO: implement temp databases todo!("temp databases not implemented yet"); } - let cookie_value = match cookie { - Cookie::ApplicationId => header_accessor::get_application_id(pager).unwrap_or(0) as i64, - Cookie::UserVersion => header_accessor::get_user_version(pager).unwrap_or(0) as i64, - Cookie::SchemaVersion => header_accessor::get_schema_cookie(pager).unwrap_or(0) as i64, - Cookie::LargestRootPageNumber => { - header_accessor::get_vacuum_mode_largest_root_page(pager).unwrap_or(0) as i64 - } + + let cookie_value = match pager.with_header(|header| match cookie { + Cookie::ApplicationId => header.application_id.get().into(), + Cookie::UserVersion => header.user_version.get().into(), + Cookie::SchemaVersion => header.schema_cookie.get().into(), + Cookie::LargestRootPageNumber => header.vacuum_mode_largest_root_page.get().into(), cookie => todo!("{cookie:?} is not yet implement for ReadCookie"), + }) { + Err(_) => 0.into(), + Ok(IOResult::Done(v)) => v, + Ok(IOResult::IO) => return Ok(InsnFunctionStepResult::IO), }; + state.registers[*dest] = Register::Value(Value::Integer(cookie_value)); state.pc += 1; Ok(InsnFunctionStepResult::Step) @@ -6090,7 +6108,7 @@ pub fn op_set_cookie( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::SetCookie { db, @@ -6104,38 +6122,38 @@ pub fn op_set_cookie( if *db > 0 { todo!("temp databases not implemented yet"); } - match cookie { - Cookie::ApplicationId => { - header_accessor::set_application_id(pager, *value)?; - } - Cookie::UserVersion => { - header_accessor::set_user_version(pager, *value)?; - } - Cookie::LargestRootPageNumber => { - header_accessor::set_vacuum_mode_largest_root_page(pager, *value as u32)?; - } - Cookie::IncrementalVacuum => { - header_accessor::set_incremental_vacuum_enabled(pager, *value as u32)?; - } - Cookie::SchemaVersion => { - if mv_store.is_none() { - // we update transaction state to indicate that the schema has changed - match program.connection.transaction_state.get() { - TransactionState::Write { schema_did_change } => { - program.connection.transaction_state.set(TransactionState::Write { schema_did_change: true }); - }, - TransactionState::Read => unreachable!("invalid transaction state for SetCookie: TransactionState::Read, should be write"), - TransactionState::None => unreachable!("invalid transaction state for SetCookie: TransactionState::None, should be write"), - TransactionState::PendingUpgrade => unreachable!("invalid transaction state for SetCookie: TransactionState::PendingUpgrade, should be write"), - } + + return_if_io!(pager.with_header_mut(|header| { + match cookie { + Cookie::ApplicationId => header.application_id = (*value).into(), + Cookie::UserVersion => header.user_version = (*value).into(), + Cookie::LargestRootPageNumber => { + header.vacuum_mode_largest_root_page = (*value as u32).into(); } - program - .connection - .with_schema_mut(|schema| schema.schema_version = *value as u32); - header_accessor::set_schema_cookie(pager, *value as u32)?; - } - cookie => todo!("{cookie:?} is not yet implement for SetCookie"), - } + Cookie::IncrementalVacuum => { + header.incremental_vacuum_enabled = (*value as u32).into() + } + Cookie::SchemaVersion => { + if mv_store.is_none() { + // we update transaction state to indicate that the schema has changed + match program.connection.transaction_state.get() { + TransactionState::Write { schema_did_change } => { + program.connection.transaction_state.set(TransactionState::Write { schema_did_change: true }); + }, + TransactionState::Read => unreachable!("invalid transaction state for SetCookie: TransactionState::Read, should be write"), + TransactionState::None => unreachable!("invalid transaction state for SetCookie: TransactionState::None, should be write"), + TransactionState::PendingUpgrade => unreachable!("invalid transaction state for SetCookie: TransactionState::PendingUpgrade, should be write"), + } + } + program + .connection + .with_schema_mut(|schema| schema.schema_version = *value as u32); + header.schema_cookie = (*value as u32).into(); + } + cookie => todo!("{cookie:?} is not yet implement for SetCookie"), + }; + })); + state.pc += 1; Ok(InsnFunctionStepResult::Step) } @@ -6145,7 +6163,7 @@ pub fn op_shift_right( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::ShiftRight { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6164,7 +6182,7 @@ pub fn op_shift_left( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::ShiftLeft { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6178,12 +6196,43 @@ pub fn op_shift_left( Ok(InsnFunctionStepResult::Step) } +pub fn op_add_imm( + program: &Program, + state: &mut ProgramState, + insn: &Insn, + pager: &Rc, + mv_store: Option<&Arc>, +) -> Result { + let Insn::AddImm { register, value } = insn else { + unreachable!("unexpected Insn {:?}", insn) + }; + + let current = &state.registers[*register]; + let current_value = match current { + Register::Value(val) => val, + Register::Aggregate(_) => &Value::Null, + Register::Record(_) => &Value::Null, + }; + + let int_val = match current_value { + Value::Integer(i) => i + value, + Value::Float(f) => (*f as i64) + value, + Value::Text(s) => s.as_str().parse::().unwrap_or(0) + value, + Value::Blob(_) => *value, // BLOB becomes the added value + Value::Null => *value, // NULL becomes the added value + }; + + state.registers[*register] = Register::Value(Value::Integer(int_val)); + state.pc += 1; + Ok(InsnFunctionStepResult::Step) +} + pub fn op_variable( program: &Program, state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Variable { index, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6198,7 +6247,7 @@ pub fn op_zero_or_null( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::ZeroOrNull { rg1, rg2, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6219,7 +6268,7 @@ pub fn op_not( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Not { reg, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6235,7 +6284,7 @@ pub fn op_concat( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Concat { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6254,7 +6303,7 @@ pub fn op_and( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::And { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6273,7 +6322,7 @@ pub fn op_or( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Or { lhs, rhs, dest } = insn else { unreachable!("unexpected Insn {:?}", insn) @@ -6292,7 +6341,7 @@ pub fn op_noop( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { // Do nothing // Advance the program counter for the next opcode @@ -6310,7 +6359,7 @@ pub fn op_open_ephemeral( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let (cursor_id, is_table) = match insn { Insn::OpenEphemeral { @@ -6324,17 +6373,30 @@ pub fn op_open_ephemeral( OpOpenEphemeralState::Start => { tracing::trace!("Start"); let conn = program.connection.clone(); - let io = conn.pager.borrow().io.get_memory_io(); + let io = conn.pager.borrow().io.clone(); + let rand_num = io.generate_random_number(); + let temp_dir = temp_dir(); + let rand_path = + std::path::Path::new(&temp_dir).join(format!("tursodb-ephemeral-{rand_num}")); + let Some(rand_path_str) = rand_path.to_str() else { + return Err(LimboError::InternalError( + "Failed to convert path to string".to_string(), + )); + }; + let file = io.open_file(rand_path_str, OpenFlags::Create, false)?; + let db_file = Arc::new(DatabaseFile::new(file)); - let file = io.open_file("", OpenFlags::Create, true)?; - let db_file = Arc::new(FileMemoryStorage::new(file)); + let page_size = pager + .io + .block(|| pager.with_header(|header| header.page_size))? + .get(); - let buffer_pool = Arc::new(BufferPool::new(None)); + let buffer_pool = Arc::new(BufferPool::new(Some(page_size as usize))); let page_cache = Arc::new(RwLock::new(DumbLruPageCache::default())); let pager = Rc::new(Pager::new( db_file, - Rc::new(RefCell::new(DummyWAL)), + None, io, page_cache, buffer_pool.clone(), @@ -6342,9 +6404,11 @@ pub fn op_open_ephemeral( Arc::new(Mutex::new(())), )?); - let page_size = header_accessor::get_page_size(&pager) - .unwrap_or(storage::sqlite3_ondisk::DEFAULT_PAGE_SIZE) - as usize; + let page_size = pager + .io + .block(|| pager.with_header(|header| header.page_size)) + .unwrap_or_default() + .get() as usize; buffer_pool.set_page_size(page_size); state.op_open_ephemeral_state = OpOpenEphemeralState::StartingTxn { pager }; @@ -6375,7 +6439,7 @@ pub fn op_open_ephemeral( let table_id = root_page as u64; let mv_store = mv_store.unwrap().clone(); let mv_cursor = Rc::new(RefCell::new( - MvCursor::new(mv_store.clone(), tx_id, table_id).unwrap(), + MvCursor::new(mv_store.clone(), tx_id, table_id, pager.clone()).unwrap(), )); Some(mv_cursor) } @@ -6446,7 +6510,7 @@ pub fn op_once( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Once { target_pc_when_reentered, @@ -6470,7 +6534,7 @@ pub fn op_found( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let (cursor_id, target_pc, record_reg, num_regs) = match insn { Insn::NotFound { @@ -6532,7 +6596,7 @@ pub fn op_affinity( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Affinity { start_reg, @@ -6566,7 +6630,7 @@ pub fn op_count( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::Count { cursor_id, @@ -6603,7 +6667,7 @@ pub fn op_integrity_check( state: &mut ProgramState, insn: &Insn, pager: &Rc, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, ) -> Result { let Insn::IntegrityCk { max_errors, @@ -6651,6 +6715,31 @@ pub fn op_integrity_check( Ok(InsnFunctionStepResult::Step) } +pub fn op_cast( + _program: &Program, + state: &mut ProgramState, + insn: &Insn, + _pager: &Rc, + _mv_store: Option<&Arc>, +) -> Result { + let Insn::Cast { reg, affinity } = insn else { + unreachable!("unexpected Insn {:?}", insn) + }; + + let value = state.registers[*reg].get_owned_value().clone(); + let result = match affinity { + Affinity::Blob => value.exec_cast("BLOB"), + Affinity::Text => value.exec_cast("TEXT"), + Affinity::Numeric => value.exec_cast("NUMERIC"), + Affinity::Integer => value.exec_cast("INTEGER"), + Affinity::Real => value.exec_cast("REAL"), + }; + + state.registers[*reg] = Register::Value(result); + state.pc += 1; + Ok(InsnFunctionStepResult::Step) +} + impl Value { pub fn exec_lower(&self) -> Option { match self { diff --git a/core/vdbe/explain.rs b/core/vdbe/explain.rs index 1578c2d44..76a969698 100644 --- a/core/vdbe/explain.rs +++ b/core/vdbe/explain.rs @@ -207,7 +207,7 @@ pub fn insn_to_str( "IfPos", *reg as i32, target_pc.as_debug_int(), - 0, + *decrement_by as i32, Value::build_text(""), 0, format!( @@ -1358,6 +1358,15 @@ pub fn insn_to_str( 0, format!("r[{dest}]=r[{lhs}] << r[{rhs}]"), ), + Insn::AddImm { register, value } => ( + "AddImm", + *register as i32, + *value as i32, + 0, + Value::build_text(""), + 0, + format!("r[{register}]=r[{register}]+{value}"), + ), Insn::Variable { index, dest } => ( "Variable", usize::from(*index) as i32, @@ -1609,6 +1618,15 @@ pub fn insn_to_str( 0, format!("r[{}] = data", *dest), ), + Insn::Cast { reg, affinity } => ( + "Cast", + *reg as i32, + 0, + 0, + Value::build_text(""), + 0, + format!("affinity(r[{}]={:?})", *reg, affinity), + ), }; format!( "{:<4} {:<17} {:<4} {:<4} {:<4} {:<13} {:<2} {}", diff --git a/core/vdbe/insn.rs b/core/vdbe/insn.rs index 92dc66a15..36be77658 100644 --- a/core/vdbe/insn.rs +++ b/core/vdbe/insn.rs @@ -706,6 +706,12 @@ pub enum Insn { func: FuncCtx, // P4 }, + /// Cast register P1 to affinity P2 and store in register P1 + Cast { + reg: usize, + affinity: Affinity, + }, + InitCoroutine { yield_reg: usize, jump_on_definition: BranchOffset, @@ -871,6 +877,14 @@ pub enum Insn { dest: usize, }, + /// Add immediate value to register and force integer conversion. + /// Add the constant P2 to the value in register P1. The result is always an integer. + /// To force any register to be an integer, just add 0. + AddImm { + register: usize, // P1: target register + value: i64, // P2: immediate value to add + }, + /// Get parameter variable. Variable { index: NonZero, @@ -1075,6 +1089,7 @@ impl Insn { Insn::SorterData { .. } => execute::op_sorter_data, Insn::SorterNext { .. } => execute::op_sorter_next, Insn::Function { .. } => execute::op_function, + Insn::Cast { .. } => execute::op_cast, Insn::InitCoroutine { .. } => execute::op_init_coroutine, Insn::EndCoroutine { .. } => execute::op_end_coroutine, Insn::Yield { .. } => execute::op_yield, @@ -1099,6 +1114,7 @@ impl Insn { Insn::ParseSchema { .. } => execute::op_parse_schema, Insn::ShiftRight { .. } => execute::op_shift_right, Insn::ShiftLeft { .. } => execute::op_shift_left, + Insn::AddImm { .. } => execute::op_add_imm, Insn::Variable { .. } => execute::op_variable, Insn::ZeroOrNull { .. } => execute::op_zero_or_null, Insn::Not { .. } => execute::op_not, diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 5d9d1fb67..8e1ec1f67 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -27,6 +27,7 @@ pub mod sorter; use crate::{ error::LimboError, function::{AggFunc, FuncCtx}, + state_machine::StateTransition, storage::sqlite3_ondisk::SmallVec, translate::plan::TableReferences, types::{IOResult, RawSlice, TextRef}, @@ -390,7 +391,7 @@ impl Program { pub fn step( &self, state: &mut ProgramState, - mv_store: Option>, + mv_store: Option>, pager: Rc, ) -> Result { loop { @@ -432,16 +433,22 @@ impl Program { &self, pager: Rc, program_state: &mut ProgramState, - mv_store: Option<&Rc>, + mv_store: Option<&Arc>, rollback: bool, ) -> Result { if let Some(mv_store) = mv_store { let conn = self.connection.clone(); let auto_commit = conn.auto_commit.get(); if auto_commit { + // FIXME: we don't want to commit stuff from other programs. let mut mv_transactions = conn.mv_transactions.borrow_mut(); for tx_id in mv_transactions.iter() { - mv_store.commit_tx(*tx_id).unwrap(); + let mut state_machine = + mv_store.commit_tx(*tx_id, pager.clone(), &conn).unwrap(); + state_machine + .step(mv_store) + .map_err(|e| LimboError::InternalError(e.to_string()))?; + assert!(state_machine.is_finalized()); } mv_transactions.clear(); } @@ -757,7 +764,10 @@ pub fn handle_program_error( err: &LimboError, ) -> Result<()> { match err { + // Transaction errors, e.g. trying to start a nested transaction, do not cause a rollback. LimboError::TxError(_) => {} + // Table locked errors, e.g. trying to checkpoint in an interactive transaction, do not cause a rollback. + LimboError::TableLocked => {} _ => { let state = connection.transaction_state.get(); if let TransactionState::Write { schema_did_change } = state { diff --git a/core/vdbe/sorter.rs b/core/vdbe/sorter.rs index 72edca61a..746e3e6cd 100644 --- a/core/vdbe/sorter.rs +++ b/core/vdbe/sorter.rs @@ -402,7 +402,7 @@ impl SortedChunk { read_buffer_ref, read_complete, ))); - let c = self.file.pread(self.total_bytes_read.get(), c)?; + let _c = self.file.pread(self.total_bytes_read.get(), c)?; Ok(()) } @@ -448,7 +448,7 @@ impl SortedChunk { }); let c = Completion::new(CompletionType::Write(WriteCompletion::new(write_complete))); - let c = self.file.pwrite(0, buffer_ref, c)?; + let _c = self.file.pwrite(0, buffer_ref, c)?; Ok(()) } } diff --git a/core/vector/vector_types.rs b/core/vector/vector_types.rs index 2ec79ed1d..779f1db86 100644 --- a/core/vector/vector_types.rs +++ b/core/vector/vector_types.rs @@ -25,11 +25,59 @@ pub struct Vector { } impl Vector { + /// # Safety + /// + /// This method is used to reinterpret the underlying `Vec` data + /// as a `&[f32]` slice. This is only valid if: + /// - The buffer is correctly aligned for `f32` + /// - The length of the buffer is exactly `dims * size_of::()` pub fn as_f32_slice(&self) -> &[f32] { - unsafe { std::slice::from_raw_parts(self.data.as_ptr() as *const f32, self.dims) } + if self.dims == 0 { + return &[]; + } + + assert_eq!( + self.data.len(), + self.dims * std::mem::size_of::(), + "data length must equal dims * size_of::()" + ); + + let ptr = self.data.as_ptr(); + let align = std::mem::align_of::(); + assert_eq!( + ptr.align_offset(align), + 0, + "data pointer must be aligned to {align} bytes for f32 access" + ); + + unsafe { std::slice::from_raw_parts(ptr as *const f32, self.dims) } } + /// # Safety + /// + /// This method is used to reinterpret the underlying `Vec` data + /// as a `&[f64]` slice. This is only valid if: + /// - The buffer is correctly aligned for `f64` + /// - The length of the buffer is exactly `dims * size_of::()` pub fn as_f64_slice(&self) -> &[f64] { + if self.dims == 0 { + return &[]; + } + + assert_eq!( + self.data.len(), + self.dims * std::mem::size_of::(), + "data length must equal dims * size_of::()" + ); + + let ptr = self.data.as_ptr(); + let align = std::mem::align_of::(); + assert_eq!( + ptr.align_offset(align), + 0, + "data pointer must be aligned to {align} bytes for f64 access" + ); + unsafe { std::slice::from_raw_parts(self.data.as_ptr() as *const f64, self.dims) } } } @@ -281,11 +329,6 @@ pub fn vector_f64_distance_cos(v1: &Vector, v2: &Vector) -> Result { } pub fn vector_type(blob: &[u8]) -> Result { - if blob.is_empty() { - return Err(LimboError::ConversionError( - "Invalid vector value".to_string(), - )); - } // Even-sized blobs are always float32. if blob.len() % 2 == 0 { return Ok(VectorType::Float32); @@ -706,6 +749,7 @@ mod tests { let v2 = float32_vec_from(&[]); let result = vector_concat(&v1, &v2).unwrap(); assert_eq!(result.dims, 0); + assert_eq!(f32_slice_from_vector(&result), Vec::::new()); } #[test] diff --git a/docs/javascript-api-reference.md b/docs/javascript-api-reference.md new file mode 100644 index 000000000..743d5bc9c --- /dev/null +++ b/docs/javascript-api-reference.md @@ -0,0 +1,151 @@ +# JavaScript API reference + +This document describes the JavaScript API for Turso. The API is implemented in two different packages: + +- **`bindings/javascript`**: Native bindings for the Turso database. +- **`packages/turso-serverless`**: Serverless driver for Turso Cloud databases. + +The API is compatible with the libSQL promise API, which is an asynchronous variant of the `better-sqlite3` API. + +## class Database + +The `Database` class represents a connection that can prepare and execute SQL statements. + +### Methods + +#### new Database(path, [options]) ⇒ Database + +Creates a new database connection. + +| Param | Type | Description | +| ------- | ------------------- | ------------------------- | +| path | string | Path to the database file | + +The `path` parameter points to the SQLite database file to open. If the file pointed to by `path` does not exists, it will be created. +To open an in-memory database, please pass `:memory:` as the `path` parameter. + +The function returns a `Database` object. + +#### prepare(sql) ⇒ Statement + +Prepares a SQL statement for execution. + +| Param | Type | Description | +| ------ | ------------------- | ------------------------------------ | +| sql | string | The SQL statement string to prepare. | + +The function returns a `Statement` object. + +#### transaction(function) ⇒ function + +This function is currently not supported. + +#### pragma(string, [options]) ⇒ results + +This function is currently not supported. + +#### backup(destination, [options]) ⇒ promise + +This function is currently not supported. + +#### serialize([options]) ⇒ Buffer + +This function is currently not supported. + +#### function(name, [options], function) ⇒ this + +This function is currently not supported. + +#### aggregate(name, options) ⇒ this + +This function is currently not supported. + +#### table(name, definition) ⇒ this + +This function is currently not supported. + +#### authorizer(rules) ⇒ this + +This function is currently not supported. + +#### loadExtension(path, [entryPoint]) ⇒ this + +This function is currently not supported. + +#### exec(sql) ⇒ this + +Executes a SQL statement. + +| Param | Type | Description | +| ------ | ------------------- | ------------------------------------ | +| sql | string | The SQL statement string to execute. | + +#### interrupt() ⇒ this + +This function is currently not supported. + +#### close() ⇒ this + +Closes the database connection. + +## class Statement + +### Methods + +#### run([...bindParameters]) ⇒ object + +Executes the SQL statement and returns an info object. + +| Param | Type | Description | +| -------------- | ----------------------------- | ------------------------------------------------ | +| bindParameters | array of objects | The bind parameters for executing the statement. | + +The returned info object contains two properties: `changes` that describes the number of modified rows and `info.lastInsertRowid` that represents the `rowid` of the last inserted row. + +#### get([...bindParameters]) ⇒ row + +Executes the SQL statement and returns the first row. + +| Param | Type | Description | +| -------------- | ----------------------------- | ------------------------------------------------ | +| bindParameters | array of objects | The bind parameters for executing the statement. | + +### all([...bindParameters]) ⇒ array of rows + +Executes the SQL statement and returns an array of the resulting rows. + +| Param | Type | Description | +| -------------- | ----------------------------- | ------------------------------------------------ | +| bindParameters | array of objects | The bind parameters for executing the statement. | + +### iterate([...bindParameters]) ⇒ iterator + +Executes the SQL statement and returns an iterator to the resulting rows. + +| Param | Type | Description | +| -------------- | ----------------------------- | ------------------------------------------------ | +| bindParameters | array of objects | The bind parameters for executing the statement. | + +#### pluck([toggleState]) ⇒ this + +This function is currently not supported. + +#### expand([toggleState]) ⇒ this + +This function is currently not supported. + +#### raw([rawMode]) ⇒ this + +This function is currently not supported. + +#### timed([toggle]) ⇒ this + +This function is currently not supported. + +#### columns() ⇒ array of objects + +This function is currently not supported. + +#### bind([...bindParameters]) ⇒ this + +This function is currently not supported. diff --git a/docs/manual.md b/docs/manual.md index 1885d55ef..97df0f48c 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -331,6 +331,10 @@ Installing the WebAssembly package: npm i @tursodatabase/turso --cpu wasm32 ``` +### API reference + +See [JavaScript API reference](docs/javascript-api-reference.md) for more information. + ### Getting Started To use Turso from JavaScript application, you need to import `Database` type from the `@tursodatabase/turso` package. diff --git a/packages/turso-serverless/package-lock.json b/packages/turso-serverless/package-lock.json index 6708f3b83..4c10ae645 100644 --- a/packages/turso-serverless/package-lock.json +++ b/packages/turso-serverless/package-lock.json @@ -1,12 +1,12 @@ { "name": "@tursodatabase/serverless", - "version": "0.1.1", + "version": "0.1.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@tursodatabase/serverless", - "version": "0.1.1", + "version": "0.1.2", "license": "MIT", "devDependencies": { "@types/node": "^24.0.13", diff --git a/packages/turso-serverless/package.json b/packages/turso-serverless/package.json index cce84f084..9da36eacf 100644 --- a/packages/turso-serverless/package.json +++ b/packages/turso-serverless/package.json @@ -1,6 +1,6 @@ { "name": "@tursodatabase/serverless", - "version": "0.1.1", + "version": "0.1.2", "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/turso-serverless/src/compat.ts b/packages/turso-serverless/src/compat.ts index c5f954389..44523d942 100644 --- a/packages/turso-serverless/src/compat.ts +++ b/packages/turso-serverless/src/compat.ts @@ -307,6 +307,11 @@ class LibSQLClient implements Client { close(): void { this._closed = true; + // Note: The libSQL client interface expects synchronous close, + // but our underlying session needs async close. We'll fire and forget. + this.session.close().catch(error => { + console.error('Error closing session:', error); + }); } } diff --git a/packages/turso-serverless/src/connection.ts b/packages/turso-serverless/src/connection.ts index a7846e8dc..3f73a08bc 100644 --- a/packages/turso-serverless/src/connection.ts +++ b/packages/turso-serverless/src/connection.ts @@ -90,6 +90,15 @@ export class Connection { const sql = `PRAGMA ${pragma}`; return this.session.execute(sql); } + + /** + * Close the connection. + * + * This sends a close request to the server to properly clean up the stream. + */ + async close(): Promise { + await this.session.close(); + } } /** diff --git a/packages/turso-serverless/src/protocol.ts b/packages/turso-serverless/src/protocol.ts index 07a94e96c..1c7654b39 100644 --- a/packages/turso-serverless/src/protocol.ts +++ b/packages/turso-serverless/src/protocol.ts @@ -18,12 +18,17 @@ export interface ExecuteResult { last_insert_rowid?: string; } +export interface NamedArg { + name: string; + value: Value; +} + export interface ExecuteRequest { type: 'execute'; stmt: { sql: string; args: Value[]; - named_args: Value[]; + named_args: NamedArg[]; want_rows: boolean; }; } @@ -32,6 +37,7 @@ export interface BatchStep { stmt: { sql: string; args: Value[]; + named_args?: NamedArg[]; want_rows: boolean; }; condition?: { @@ -52,9 +58,13 @@ export interface SequenceRequest { sql: string; } +export interface CloseRequest { + type: 'close'; +} + export interface PipelineRequest { baton: string | null; - requests: (ExecuteRequest | BatchRequest | SequenceRequest)[]; + requests: (ExecuteRequest | BatchRequest | SequenceRequest | CloseRequest)[]; } export interface PipelineResponse { @@ -63,7 +73,7 @@ export interface PipelineResponse { results: Array<{ type: 'ok' | 'error'; response?: { - type: 'execute' | 'batch' | 'sequence'; + type: 'execute' | 'batch' | 'sequence' | 'close'; result?: ExecuteResult; }; error?: { @@ -182,52 +192,71 @@ export async function executeCursor( const decoder = new TextDecoder(); let buffer = ''; - let isFirstLine = true; - let cursorResponse: CursorResponse; + let cursorResponse: CursorResponse | undefined; + + // First, read until we get the cursor response (first line) + while (!cursorResponse) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + + const newlineIndex = buffer.indexOf('\n'); + if (newlineIndex !== -1) { + const line = buffer.slice(0, newlineIndex).trim(); + buffer = buffer.slice(newlineIndex + 1); + + if (line) { + cursorResponse = JSON.parse(line); + break; + } + } + } + + if (!cursorResponse) { + throw new DatabaseError('No cursor response received'); + } async function* parseEntries(): AsyncGenerator { try { + // Process any remaining data in the buffer + let newlineIndex; + while ((newlineIndex = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, newlineIndex).trim(); + buffer = buffer.slice(newlineIndex + 1); + + if (line) { + yield JSON.parse(line) as CursorEntry; + } + } + + // Continue reading from the stream while (true) { const { done, value } = await reader!.read(); if (done) break; buffer += decoder.decode(value, { stream: true }); - let newlineIndex; while ((newlineIndex = buffer.indexOf('\n')) !== -1) { const line = buffer.slice(0, newlineIndex).trim(); buffer = buffer.slice(newlineIndex + 1); if (line) { - if (isFirstLine) { - cursorResponse = JSON.parse(line); - isFirstLine = false; - } else { - yield JSON.parse(line) as CursorEntry; - } + yield JSON.parse(line) as CursorEntry; } } } + + // Process any remaining data in the buffer + if (buffer.trim()) { + yield JSON.parse(buffer.trim()) as CursorEntry; + } } finally { reader!.releaseLock(); } } - const entries = parseEntries(); - - // Get the first entry to parse the cursor response - const firstEntry = await entries.next(); - if (!firstEntry.done) { - // Put the first entry back - const generator = (async function* () { - yield firstEntry.value; - yield* entries; - })(); - - return { response: cursorResponse!, entries: generator }; - } - - return { response: cursorResponse!, entries }; + return { response: cursorResponse, entries: parseEntries() }; } export async function executePipeline( diff --git a/packages/turso-serverless/src/session.ts b/packages/turso-serverless/src/session.ts index eb401fd03..6fddbfb2c 100644 --- a/packages/turso-serverless/src/session.ts +++ b/packages/turso-serverless/src/session.ts @@ -7,7 +7,10 @@ import { type CursorResponse, type CursorEntry, type PipelineRequest, - type SequenceRequest + type SequenceRequest, + type CloseRequest, + type NamedArg, + type Value } from './protocol.js'; import { DatabaseError } from './error.js'; @@ -49,10 +52,10 @@ export class Session { * Execute a SQL statement and return all results. * * @param sql - The SQL statement to execute - * @param args - Optional array of parameter values + * @param args - Optional array of parameter values or object with named parameters * @returns Promise resolving to the complete result set */ - async execute(sql: string, args: any[] = []): Promise { + async execute(sql: string, args: any[] | Record = []): Promise { const { response, entries } = await this.executeRaw(sql, args); const result = await this.processCursorEntries(entries); return result; @@ -62,17 +65,56 @@ export class Session { * Execute a SQL statement and return the raw response and entries. * * @param sql - The SQL statement to execute - * @param args - Optional array of parameter values + * @param args - Optional array of parameter values or object with named parameters * @returns Promise resolving to the raw response and cursor entries */ - async executeRaw(sql: string, args: any[] = []): Promise<{ response: CursorResponse; entries: AsyncGenerator }> { + async executeRaw(sql: string, args: any[] | Record = []): Promise<{ response: CursorResponse; entries: AsyncGenerator }> { + let positionalArgs: Value[] = []; + let namedArgs: NamedArg[] = []; + + if (Array.isArray(args)) { + positionalArgs = args.map(encodeValue); + } else { + // Check if this is an object with numeric keys (for ?1, ?2 style parameters) + const keys = Object.keys(args); + const isNumericKeys = keys.length > 0 && keys.every(key => /^\d+$/.test(key)); + + if (isNumericKeys) { + // Convert numeric-keyed object to positional args + // Sort keys numerically to ensure correct order + const sortedKeys = keys.sort((a, b) => parseInt(a) - parseInt(b)); + const maxIndex = parseInt(sortedKeys[sortedKeys.length - 1]); + + // Create array with undefined for missing indices + positionalArgs = new Array(maxIndex); + for (const key of sortedKeys) { + const index = parseInt(key) - 1; // Convert to 0-based index + positionalArgs[index] = encodeValue(args[key]); + } + + // Fill any undefined values with null + for (let i = 0; i < positionalArgs.length; i++) { + if (positionalArgs[i] === undefined) { + positionalArgs[i] = { type: 'null' }; + } + } + } else { + // Convert object with named parameters to NamedArg array + namedArgs = Object.entries(args).map(([name, value]) => ({ + name, + value: encodeValue(value) + })); + } + } + const request: CursorRequest = { baton: this.baton, batch: { steps: [{ stmt: { sql, - args: args.map(encodeValue), + args: positionalArgs, + named_args: namedArgs, want_rows: true } }] @@ -180,6 +222,7 @@ export class Session { stmt: { sql, args: [], + named_args: [], want_rows: false } })) @@ -248,4 +291,33 @@ export class Session { } } } + + /** + * Close the session. + * + * This sends a close request to the server to properly clean up the stream + * before resetting the local state. + */ + async close(): Promise { + // Only send close request if we have an active baton + if (this.baton) { + try { + const request: PipelineRequest = { + baton: this.baton, + requests: [{ + type: "close" + } as CloseRequest] + }; + + await executePipeline(this.baseUrl, this.config.authToken, request); + } catch (error) { + // Ignore errors during close, as the connection might already be closed + console.error('Error closing session:', error); + } + } + + // Reset local state + this.baton = null; + this.baseUrl = ''; + } } \ No newline at end of file diff --git a/packages/turso-serverless/src/statement.ts b/packages/turso-serverless/src/statement.ts index 72907fb2d..c5dbbca99 100644 --- a/packages/turso-serverless/src/statement.ts +++ b/packages/turso-serverless/src/statement.ts @@ -17,16 +17,36 @@ import { DatabaseError } from './error.js'; export class Statement { private session: Session; private sql: string; + private presentationMode: 'expanded' | 'raw' | 'pluck' = 'expanded'; constructor(sessionConfig: SessionConfig, sql: string) { this.session = new Session(sessionConfig); this.sql = sql; } + + /** + * Enable raw mode to return arrays instead of objects. + * + * @param raw Enable or disable raw mode. If you don't pass the parameter, raw mode is enabled. + * @returns This statement instance for chaining + * + * @example + * ```typescript + * const stmt = client.prepare("SELECT * FROM users WHERE id = ?"); + * const row = await stmt.raw().get([1]); + * console.log(row); // [1, "Alice", "alice@example.org"] + * ``` + */ + raw(raw?: boolean): Statement { + this.presentationMode = raw === false ? 'expanded' : 'raw'; + return this; + } + /** * Executes the prepared statement. * - * @param args - Optional array of parameter values for the SQL statement + * @param args - Optional array of parameter values or object with named parameters * @returns Promise resolving to the result of the statement * * @example @@ -36,16 +56,17 @@ export class Statement { * console.log(`Inserted user with ID ${result.lastInsertRowid}`); * ``` */ - async run(args: any[] = []): Promise { - const result = await this.session.execute(this.sql, args); + async run(args?: any): Promise { + const normalizedArgs = this.normalizeArgs(args); + const result = await this.session.execute(this.sql, normalizedArgs); return { changes: result.rowsAffected, lastInsertRowid: result.lastInsertRowid }; } /** * Execute the statement and return the first row. * - * @param args - Optional array of parameter values for the SQL statement - * @returns Promise resolving to the first row or null if no results + * @param args - Optional array of parameter values or object with named parameters + * @returns Promise resolving to the first row or undefined if no results * * @example * ```typescript @@ -56,15 +77,27 @@ export class Statement { * } * ``` */ - async get(args: any[] = []): Promise { - const result = await this.session.execute(this.sql, args); - return result.rows[0] || null; + async get(args?: any): Promise { + const normalizedArgs = this.normalizeArgs(args); + const result = await this.session.execute(this.sql, normalizedArgs); + const row = result.rows[0]; + if (!row) { + return undefined; + } + + if (this.presentationMode === 'raw') { + // In raw mode, return the row as a plain array (it already is one) + // The row object is already an array with column properties added + return [...row]; + } + + return row; } /** * Execute the statement and return all rows. * - * @param args - Optional array of parameter values for the SQL statement + * @param args - Optional array of parameter values or object with named parameters * @returns Promise resolving to an array of all result rows * * @example @@ -74,8 +107,16 @@ export class Statement { * console.log(`Found ${activeUsers.length} active users`); * ``` */ - async all(args: any[] = []): Promise { - const result = await this.session.execute(this.sql, args); + async all(args?: any): Promise { + const normalizedArgs = this.normalizeArgs(args); + const result = await this.session.execute(this.sql, normalizedArgs); + + if (this.presentationMode === 'raw') { + // In raw mode, return arrays of values + // Each row is already an array with column properties added + return result.rows.map((row: any) => [...row]); + } + return result.rows; } @@ -85,7 +126,7 @@ export class Statement { * This method provides memory-efficient processing of large result sets * by streaming rows one at a time instead of loading everything into memory. * - * @param args - Optional array of parameter values for the SQL statement + * @param args - Optional array of parameter values or object with named parameters * @returns AsyncGenerator that yields individual rows * * @example @@ -97,8 +138,9 @@ export class Statement { * } * ``` */ - async *iterate(args: any[] = []): AsyncGenerator { - const { response, entries } = await this.session.executeRaw(this.sql, args); + async *iterate(args?: any): AsyncGenerator { + const normalizedArgs = this.normalizeArgs(args); + const { response, entries } = await this.session.executeRaw(this.sql, normalizedArgs); let columns: string[] = []; @@ -112,8 +154,13 @@ export class Statement { case 'row': if (entry.row) { const decodedRow = entry.row.map(decodeValue); - const rowObject = this.session.createRowObject(decodedRow, columns); - yield rowObject; + if (this.presentationMode === 'raw') { + // In raw mode, yield arrays of values + yield decodedRow; + } else { + const rowObject = this.session.createRowObject(decodedRow, columns); + yield rowObject; + } } break; case 'step_error': @@ -123,4 +170,27 @@ export class Statement { } } + /** + * Normalize arguments to handle both single values and arrays. + * Matches the behavior of the native bindings. + */ + private normalizeArgs(args: any): any[] | Record { + // No arguments provided + if (args === undefined) { + return []; + } + + // If it's an array, return as-is + if (Array.isArray(args)) { + return args; + } + + // Check if it's a plain object (for named parameters) + if (args !== null && typeof args === 'object' && args.constructor === Object) { + return args; + } + + // Single value - wrap in array + return [args]; + } } diff --git a/perf/clickbench/benchmark.sh b/perf/clickbench/benchmark.sh index 32e3e29a2..189c84c57 100755 --- a/perf/clickbench/benchmark.sh +++ b/perf/clickbench/benchmark.sh @@ -23,7 +23,7 @@ rm "$CLICKBENCH_DIR/mydb"* || true # Create DB using tursodb echo "Creating DB..." -"$RELEASE_BUILD_DIR/tursodb" --quiet --experimental-indexes "$CLICKBENCH_DIR/mydb" < "$CLICKBENCH_DIR/create.sql" +"$RELEASE_BUILD_DIR/tursodb" --quiet "$CLICKBENCH_DIR/mydb" < "$CLICKBENCH_DIR/create.sql" # Download a subset of the clickbench dataset if it doesn't exist NUM_ROWS=1000000 diff --git a/perf/clickbench/run.sh b/perf/clickbench/run.sh index 09e5396a9..329a4caac 100755 --- a/perf/clickbench/run.sh +++ b/perf/clickbench/run.sh @@ -37,7 +37,7 @@ grep -v '^--' "$CLICKBENCH_DIR/queries.sql" | while read -r query; do for _ in $(seq 1 $TRIES); do clear_caches echo "----tursodb----" - ((time "$RELEASE_BUILD_DIR/tursodb" --quiet --experimental-indexes -m list "$CLICKBENCH_DIR/mydb" <<< "${query}") 2>&1) | tee -a clickbench-tursodb.txt + ((time "$RELEASE_BUILD_DIR/tursodb" --quiet -m list "$CLICKBENCH_DIR/mydb" <<< "${query}") 2>&1) | tee -a clickbench-tursodb.txt clear_caches echo echo "----sqlite----" diff --git a/perf/tpc-h/run.sh b/perf/tpc-h/run.sh index b4b40ca10..7bea14c23 100755 --- a/perf/tpc-h/run.sh +++ b/perf/tpc-h/run.sh @@ -66,7 +66,7 @@ for query_file in $(ls "$QUERIES_DIR"/*.sql | sort -V); do # Clear caches before Limbo run clear_caches # Run Limbo - limbo_output=$( { time -p "$LIMBO_BIN" "$DB_FILE" --experimental-indexes --quiet --output-mode list "$(cat $query_file)" 2>&1; } 2>&1) + limbo_output=$( { time -p "$LIMBO_BIN" "$DB_FILE" --quiet --output-mode list "$(cat $query_file)" 2>&1; } 2>&1) limbo_non_time_lines=$(echo "$limbo_output" | grep -v -e "^real" -e "^user" -e "^sys") limbo_real_time=$(echo "$limbo_output" | grep "^real" | awk '{print $2}') echo "Running $query_name with SQLite3..." >&2 diff --git a/scripts/merge-pr.py b/scripts/merge-pr.py index de3feadd2..4ff2183a7 100755 --- a/scripts/merge-pr.py +++ b/scripts/merge-pr.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 # -# Copyright 2024 the Limbo authors. All rights reserved. MIT license. +# Copyright 2024 the Turso authors. All rights reserved. MIT license. # -# A script to merge a pull requests with a nice merge commit. +# A script to merge a pull requests with a nice merge commit using GitHub CLI. # # Requirements: -# -# ``` -# pip install PyGithub -# ``` +# - GitHub CLI (`gh`) must be installed and authenticated import json import os import re @@ -17,13 +14,14 @@ import sys import tempfile import textwrap -from github import Github - -def run_command(command): - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - output, error = process.communicate() - return output.decode("utf-8").strip(), error.decode("utf-8").strip(), process.returncode +def run_command(command, capture_output=True): + if capture_output: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + output, error = process.communicate() + return output.decode("utf-8").strip(), error.decode("utf-8").strip(), process.returncode + else: + return "", "", subprocess.call(command, shell=True) def load_user_mapping(file_path=".github.json"): @@ -36,43 +34,49 @@ def load_user_mapping(file_path=".github.json"): user_mapping = load_user_mapping() -def get_user_email(g, username): +def get_user_email(username): if username in user_mapping: return f"{user_mapping[username]['name']} <{user_mapping[username]['email']}>" - try: - user = g.get_user(username) - name = user.name if user.name else username - if user.email: - return f"{name} <{user.email}>" + # Try to get user info from gh CLI + output, _, returncode = run_command(f"gh api users/{username}") + if returncode == 0: + user_data = json.loads(output) + name = user_data.get("name", username) + email = user_data.get("email") + if email: + return f"{name} <{email}>" return f"{name} (@{username})" - except Exception as e: - print(f"Error fetching email for user {username}: {str(e)}") - # If we couldn't find an email, return a noreply address + # Fallback to noreply address return f"{username} <{username}@users.noreply.github.com>" -def get_pr_info(g, repo, pr_number): - pr = repo.get_pull(int(pr_number)) - author = pr.user - author_name = author.name if author.name else author.login +def get_pr_info(pr_number): + output, error, returncode = run_command( + f"gh pr view {pr_number} --json number,title,author,headRefName,body,reviews" + ) + if returncode != 0: + print(f"Error fetching PR #{pr_number}: {error}") + sys.exit(1) + + pr_data = json.loads(output) - # Get the list of users who reviewed the PR reviewed_by = [] - reviews = pr.get_reviews() - for review in reviews: - if review.state == "APPROVED": - reviewer = review.user - reviewed_by.append(get_user_email(g, reviewer.login)) + for review in pr_data.get("reviews", []): + if review["state"] == "APPROVED": + reviewed_by.append(get_user_email(review["author"]["login"])) + + # Remove duplicates while preserving order + reviewed_by = list(dict.fromkeys(reviewed_by)) return { - "number": pr.number, - "title": pr.title, - "author": author_name, - "head": pr.head.ref, - "head_sha": pr.head.sha, - "body": pr.body.strip() if pr.body else "", + "number": pr_data["number"], + "title": pr_data["title"], + "author": pr_data["author"]["login"], + "author_name": pr_data["author"].get("name", pr_data["author"]["login"]), + "head": pr_data["headRefName"], + "body": (pr_data.get("body") or "").strip(), "reviewed_by": reviewed_by, } @@ -92,77 +96,127 @@ def wrap_text(text, width=72): return "\n".join(wrapped_lines) -def merge_pr(pr_number): - # GitHub authentication - token = os.getenv("GITHUB_TOKEN") - g = Github(token) - - # Get the repository - repo_name = os.getenv("GITHUB_REPOSITORY") - if not repo_name: - print("Error: GITHUB_REPOSITORY environment variable not set") - sys.exit(1) - repo = g.get_repo(repo_name) - - # Get PR information - pr_info = get_pr_info(g, repo, pr_number) - - # Format commit message - commit_title = f"Merge '{pr_info['title']}' from {pr_info['author']}" - commit_body = wrap_text(pr_info["body"]) - - commit_message = f"{commit_title}\n\n{commit_body}\n" - - # Add Reviewed-by lines - for approver in pr_info["reviewed_by"]: - commit_message += f"\nReviewed-by: {approver}" - - # Add Closes line - commit_message += f"\n\nCloses #{pr_info['number']}" +def merge_remote(pr_number: int, commit_message: str, commit_title: str): + output, error, returncode = run_command(f"gh pr checks {pr_number} --json state") + if returncode == 0: + checks_data = json.loads(output) + if checks_data and any(check.get("state") == "FAILURE" for check in checks_data): + print("Warning: Some checks are failing") + if input("Do you want to proceed with the merge? (y/N): ").strip().lower() != "y": + exit(0) # Create a temporary file for the commit message + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as temp_file: + temp_file.write(commit_message) + temp_file_path = temp_file.name + + try: + print(f"\nMerging PR #{pr_number} with custom commit message...") + # Use gh pr merge with the commit message file + cmd = f'gh pr merge {pr_number} --merge --subject "{commit_title}" --body-file "{temp_file_path}"' + output, error, returncode = run_command(cmd, capture_output=False) + + if returncode == 0: + print(f"\nPull request #{pr_number} merged successfully!") + print(f"\nMerge commit message:\n{commit_message}") + else: + print(f"Error merging PR: {error}") + sys.exit(1) + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + +def merge_local(pr_number: int, commit_message: str): + current_branch, _, _ = run_command("git branch --show-current") + + print(f"Fetching PR #{pr_number}...") + cmd = f"gh pr checkout {pr_number}" + _, error, returncode = run_command(cmd) + if returncode != 0: + print(f"Error checking out PR: {error}") + sys.exit(1) + + pr_branch, _, _ = run_command("git branch --show-current") + + cmd = "git checkout main" + _, error, returncode = run_command(cmd) + if returncode != 0: + print(f"Error checking out main branch: {error}") + sys.exit(1) + with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: temp_file.write(commit_message) temp_file_path = temp_file.name try: - # Instead of fetching to a branch, fetch the specific commit - cmd = f"git fetch origin pull/{pr_number}/head" - output, error, returncode = run_command(cmd) - if returncode != 0: - print(f"Error fetching PR: {error}") - sys.exit(1) - - # Checkout main branch - cmd = "git checkout main" - output, error, returncode = run_command(cmd) - if returncode != 0: - print(f"Error checking out main branch: {error}") - sys.exit(1) - - # Merge using the commit SHA instead of branch name - cmd = f"git merge --no-ff {pr_info['head_sha']} -F {temp_file_path}" - output, error, returncode = run_command(cmd) + # Merge the PR branch with the custom message + # Using -F with the full message (title + body) + cmd = f"git merge --no-ff {pr_branch} -F {temp_file_path}" + _, error, returncode = run_command(cmd) if returncode != 0: print(f"Error merging PR: {error}") + # Try to go back to original branch + run_command(f"git checkout {current_branch}") sys.exit(1) - print("Pull request merged successfully!") - print(f"Merge commit message:\n{commit_message}") + print("\nPull request merged successfully locally!") + print(f"\nMerge commit message:\n{commit_message}") finally: # Clean up the temporary file os.unlink(temp_file_path) -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python merge_pr.py ") +def merge_pr(pr_number, use_api=True): + """Merge a pull request with a formatted commit message""" + check_gh_auth() + + print(f"Fetching PR #{pr_number}...") + pr_info = get_pr_info(pr_number) + print(f"PR found: '{pr_info['title']}' by {pr_info['author']}") + + # Format commit message + commit_title = f"Merge '{pr_info['title']}' from {pr_info['author_name']}" + commit_body = wrap_text(pr_info["body"]) + + commit_message_parts = [commit_title] + if commit_body: + commit_message_parts.append("") # Empty line between title and body + commit_message_parts.append(commit_body) + if pr_info["reviewed_by"]: + commit_message_parts.append("") # Empty line before reviewed-by + for approver in pr_info["reviewed_by"]: + commit_message_parts.append(f"Reviewed-by: {approver}") + commit_message_parts.append("") # Empty line before Closes + commit_message_parts.append(f"Closes #{pr_info['number']}") + commit_message = "\n".join(commit_message_parts) + + if use_api: + # For remote merge, we need to separate title from body + commit_body_for_api = "\n".join(commit_message_parts[2:]) + merge_remote(pr_number, commit_body_for_api, commit_title) + else: + merge_local(pr_number, commit_message) + + +def check_gh_auth(): + """Check if gh CLI is authenticated""" + _, _, returncode = run_command("gh auth status") + if returncode != 0: + print("Error: GitHub CLI is not authenticated. Run 'gh auth login' first.") sys.exit(1) - pr_number = sys.argv[1] - if not re.match(r"^\d+$", pr_number): + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Merge a pull request with a nice merge commit using GitHub CLI") + parser.add_argument("pr_number", type=str, help="Pull request number to merge") + parser.add_argument("--local", action="store_true", help="Use local git commands instead of GitHub API") + args = parser.parse_args() + if not re.match(r"^\d+$", args.pr_number): print("Error: PR number must be a positive integer") sys.exit(1) - - merge_pr(pr_number) + use_api = not args.local + merge_pr(args.pr_number, use_api) diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index a9d988d2e..6fb0c7c38 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -1,9 +1,6 @@ [project] name = "scripts" version = "0.1.0" -description = "Add your description here" +description = "Assorted scripts for tursodb" readme = "README.md" requires-python = ">=3.13" -dependencies = [ - "pygithub>=2.6.1", -] diff --git a/simulator/runner/file.rs b/simulator/runner/file.rs index ba3680333..e21ab2d07 100644 --- a/simulator/runner/file.rs +++ b/simulator/runner/file.rs @@ -121,7 +121,7 @@ impl SimulatorFile { if queued_io[i].time <= now { let io = queued_io.remove(i); // your code here - let c = (io.op)(self)?; + let _c = (io.op)(self)?; } else { i += 1; } @@ -222,6 +222,34 @@ impl File for SimulatorFile { Ok(c) } + fn pwritev( + &self, + pos: usize, + buffers: Vec>>, + c: turso_core::Completion, + ) -> Result { + self.nr_pwrite_calls.set(self.nr_pwrite_calls.get() + 1); + if self.fault.get() { + tracing::debug!("pwritev fault"); + self.nr_pwrite_faults.set(self.nr_pwrite_faults.get() + 1); + return Err(turso_core::LimboError::InternalError( + FAULT_ERROR_MSG.into(), + )); + } + if let Some(latency) = self.generate_latency_duration() { + let cloned_c = c.clone(); + let op = + Box::new(move |file: &SimulatorFile| file.inner.pwritev(pos, buffers, cloned_c)); + self.queued_io + .borrow_mut() + .push(DelayedIo { time: latency, op }); + Ok(c) + } else { + let c = self.inner.pwritev(pos, buffers, c)?; + Ok(c) + } + } + fn size(&self) -> Result { self.inner.size() } diff --git a/stress/Cargo.toml b/stress/Cargo.toml index 76d81ad39..b667e773d 100644 --- a/stress/Cargo.toml +++ b/stress/Cargo.toml @@ -17,7 +17,7 @@ path = "main.rs" [features] default = ["experimental_indexes"] antithesis = ["turso/antithesis"] -experimental_indexes = ["turso/experimental_indexes"] +experimental_indexes = [] [dependencies] anarchist-readable-name-generator-lib = "0.1.0" diff --git a/testing/agg-functions.test b/testing/agg-functions.test index e83849391..9becf56a4 100755 --- a/testing/agg-functions.test +++ b/testing/agg-functions.test @@ -141,8 +141,6 @@ do_execsql_test select-agg-json-array-object { SELECT json_group_array(json_object('name', name)) FROM products; } {[{"name":"hat"},{"name":"cap"},{"name":"shirt"},{"name":"sweater"},{"name":"sweatshirt"},{"name":"shorts"},{"name":"jeans"},{"name":"sneakers"},{"name":"boots"},{"name":"coat"},{"name":"accessories"}]} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test select-distinct-agg-functions { - SELECT sum(distinct age), count(distinct age), avg(distinct age) FROM users; - } {5050|100|50.5} -} +do_execsql_test select-distinct-agg-functions { +SELECT sum(distinct age), count(distinct age), avg(distinct age) FROM users; +} {5050|100|50.5} \ No newline at end of file diff --git a/testing/alter_table.test b/testing/alter_table.test index 98676d558..24bf74fe8 100755 --- a/testing/alter_table.test +++ b/testing/alter_table.test @@ -9,16 +9,14 @@ do_execsql_test_on_specific_db {:memory:} alter-table-rename-table { SELECT name FROM sqlite_schema WHERE type = 'table'; } { "t2" } -if {[info exists ::env(SQLITE_EXEC)] && $::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental"} { - do_execsql_test_on_specific_db {:memory:} alter-table-rename-column { - CREATE TABLE t (a); - CREATE INDEX i ON t(a); - ALTER TABLE t RENAME a TO b; - SELECT sql FROM sqlite_schema; - } { - "CREATE TABLE t (b)" - "CREATE INDEX i ON t(b)" - } +do_execsql_test_on_specific_db {:memory:} alter-table-rename-column { + CREATE TABLE t (a); + CREATE INDEX i ON t (a); + ALTER TABLE t RENAME a TO b; + SELECT sql FROM sqlite_schema; +} { + "CREATE TABLE t (b)" + "CREATE INDEX i ON t (b)" } do_execsql_test_on_specific_db {:memory:} alter-table-add-column { @@ -48,34 +46,32 @@ do_execsql_test_on_specific_db {:memory:} alter-table-add-column-typed { "1|0" } -if {[info exists ::env(SQLITE_EXEC)] && $::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental"} { - do_execsql_test_on_specific_db {:memory:} alter-table-add-column-default { - CREATE TABLE test (a); - INSERT INTO test VALUES (1), (2), (3); +do_execsql_test_on_specific_db {:memory:} alter-table-add-column-default { + CREATE TABLE test (a); + INSERT INTO test VALUES (1), (2), (3); - ALTER TABLE test ADD b DEFAULT 0.1; - ALTER TABLE test ADD c DEFAULT 'hello'; - SELECT * FROM test; + ALTER TABLE test ADD b DEFAULT 0.1; + ALTER TABLE test ADD c DEFAULT 'hello'; + SELECT * FROM test; - CREATE INDEX idx ON test (b); - SELECT b, c FROM test WHERE b = 0.1; + CREATE INDEX idx ON test (b); + SELECT b, c FROM test WHERE b = 0.1; - ALTER TABLE test DROP a; - SELECT * FROM test; + ALTER TABLE test DROP a; + SELECT * FROM test; - } { - "1|0.1|hello" - "2|0.1|hello" - "3|0.1|hello" +} { +"1|0.1|hello" +"2|0.1|hello" +"3|0.1|hello" - "0.1|hello" - "0.1|hello" - "0.1|hello" +"0.1|hello" +"0.1|hello" +"0.1|hello" - "0.1|hello" - "0.1|hello" - "0.1|hello" - } +"0.1|hello" +"0.1|hello" +"0.1|hello" } do_execsql_test_on_specific_db {:memory:} alter-table-drop-column { diff --git a/testing/cli_tests/vfs_bench.py b/testing/cli_tests/vfs_bench.py index b54ababf3..dc637c37b 100644 --- a/testing/cli_tests/vfs_bench.py +++ b/testing/cli_tests/vfs_bench.py @@ -48,6 +48,9 @@ def bench_one(vfs: str, sql: str, iterations: int) -> list[float]: def setup_temp_db() -> None: + # make sure we start fresh, otherwise we could end up with + # one having to checkpoint the others from the previous run + cleanup_temp_db() cmd = ["sqlite3", "testing/testing.db", ".clone testing/temp.db"] proc = subprocess.run(cmd, check=True) proc.check_returncode() @@ -57,7 +60,9 @@ def setup_temp_db() -> None: def cleanup_temp_db() -> None: if DB_FILE.exists(): DB_FILE.unlink() - os.remove("testing/temp.db-wal") + wal_file = DB_FILE.with_suffix(".db-wal") + if wal_file.exists(): + os.remove(wal_file) def main() -> None: @@ -65,7 +70,6 @@ def main() -> None: parser.add_argument("sql", help="SQL statement to execute (quote it)") parser.add_argument("iterations", type=int, help="number of repetitions") args = parser.parse_args() - setup_temp_db() sql, iterations = args.sql, args.iterations if iterations <= 0: @@ -79,12 +83,15 @@ def main() -> None: averages: Dict[str, float] = {} for vfs in vfs_list: + setup_temp_db() test(f"\n### VFS: {vfs} ###") times = bench_one(vfs, sql, iterations) info(f"All times ({vfs}):", " ".join(f"{t:.6f}" for t in times)) avg = statistics.mean(times) averages[vfs] = avg + cleanup_temp_db() + info("\n" + "-" * 60) info("Average runtime per VFS") info("-" * 60) @@ -106,7 +113,6 @@ def main() -> None: faster_slower = "slower" if pct > 0 else "faster" info(f"{vfs:<{name_pad}} : {avg:.6f} ({abs(pct):.1f}% {faster_slower} than {baseline})") info("-" * 60) - cleanup_temp_db() if __name__ == "__main__": diff --git a/testing/create_table.test b/testing/create_table.test index 63a9dc2b2..0ffca2e7d 100755 --- a/testing/create_table.test +++ b/testing/create_table.test @@ -3,16 +3,14 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test_in_memory_any_error create_table_one_unique_set { - CREATE TABLE t4 (a, unique(b)); - } - - do_execsql_test_on_specific_db {:memory:} create_table_same_uniques_and_primary_keys { - CREATE TABLE t2 (a,b, unique(a,b), primary key(a,b)); - } {} - - do_execsql_test_on_specific_db {:memory:} create_table_unique_contained_in_primary_keys { - CREATE TABLE t4 (a,b, primary key(a,b), unique(a)); - } {} +do_execsql_test_in_memory_any_error create_table_one_unique_set { + CREATE TABLE t4 (a, unique(b)); } + +do_execsql_test_on_specific_db {:memory:} create_table_same_uniques_and_primary_keys { + CREATE TABLE t2 (a,b, unique(a,b), primary key(a,b)); +} {} + +do_execsql_test_on_specific_db {:memory:} create_table_unique_contained_in_primary_keys { + CREATE TABLE t4 (a,b, primary key(a,b), unique(a)); +} {} \ No newline at end of file diff --git a/testing/delete.test b/testing/delete.test index 8237a2533..e134270c7 100755 --- a/testing/delete.test +++ b/testing/delete.test @@ -52,16 +52,14 @@ do_execsql_test_on_specific_db {:memory:} delete-reuse-1 { } {1 2 3} # Test delete works when there are indexes -if {[info exists ::env(SQLITE_EXEC)] && $::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental"} { - do_execsql_test_on_specific_db {:memory:} delete-all-with-indexes-1 { - CREATE TABLE t (a PRIMARY KEY); - CREATE INDEX tasc ON t(a); - CREATE INDEX tdesc ON t(a DESC); - INSERT INTO t VALUES (randomblob(1000)); - DELETE FROM t; - SELECT * FROM t; - } {} -} +do_execsql_test_on_specific_db {:memory:} delete-all-with-indexes-1 { + CREATE TABLE t (a PRIMARY KEY); + CREATE INDEX tasc ON t(a); + CREATE INDEX tdesc ON t(a DESC); + INSERT INTO t VALUES (randomblob(1000)); + DELETE FROM t; + SELECT * FROM t; +} {} do_execsql_test_on_specific_db {:memory:} delete_where_falsy { CREATE TABLE resourceful_schurz (diplomatic_kaplan BLOB); diff --git a/testing/drop_index.test b/testing/drop_index.test index 5582f55f8..4edbfc2bc 100755 --- a/testing/drop_index.test +++ b/testing/drop_index.test @@ -3,46 +3,44 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - # Basic DROP INDEX functionality - do_execsql_test_on_specific_db {:memory:} drop-index-basic-1 { - CREATE TABLE t1 (x INTEGER PRIMARY KEY); - CREATE INDEX t_idx on t1 (x); - INSERT INTO t1 VALUES (1); - INSERT INTO t1 VALUES (2); - DROP INDEX t_idx; - SELECT count(*) FROM sqlite_schema WHERE type='index' AND name='t_idx'; - } {0} +# Basic DROP INDEX functionality +do_execsql_test_on_specific_db {:memory:} drop-index-basic-1 { + CREATE TABLE t1 (x INTEGER PRIMARY KEY); + CREATE INDEX t_idx on t1 (x); + INSERT INTO t1 VALUES (1); + INSERT INTO t1 VALUES (2); + DROP INDEX t_idx; + SELECT count(*) FROM sqlite_schema WHERE type='index' AND name='t_idx'; +} {0} - # Test DROP INDEX IF EXISTS on existing index - do_execsql_test_on_specific_db {:memory:} drop-index-if-exists-1 { - CREATE TABLE t2 (x INTEGER PRIMARY KEY); - CREATE INDEX t_idx2 on t2 (x); - DROP INDEX IF EXISTS t_idx2; - SELECT count(*) FROM sqlite_schema WHERE type='index' AND name='t_idx2'; - } {0} +# Test DROP INDEX IF EXISTS on existing index +do_execsql_test_on_specific_db {:memory:} drop-index-if-exists-1 { + CREATE TABLE t2 (x INTEGER PRIMARY KEY); + CREATE INDEX t_idx2 on t2 (x); + DROP INDEX IF EXISTS t_idx2; + SELECT count(*) FROM sqlite_schema WHERE type='index' AND name='t_idx2'; +} {0} - # Test DROP INDEX IF EXISTS on non-existent index - do_execsql_test_on_specific_db {:memory:} drop-index-if-exists-2 { - DROP TABLE IF EXISTS nonexistent_index; - SELECT 'success'; - } {success} +# Test DROP INDEX IF EXISTS on non-existent index +do_execsql_test_on_specific_db {:memory:} drop-index-if-exists-2 { + DROP TABLE IF EXISTS nonexistent_index; + SELECT 'success'; +} {success} - # Test dropping non-existant index produces an error - do_execsql_test_error_content drop-index-no-index { - DROP INDEX t_idx; - } {"No such index: t_idx"} +# Test dropping non-existant index produces an error +do_execsql_test_error_content drop-index-no-index { + DROP INDEX t_idx; +} {"No such index: t_idx"} - # Test dropping index after multiple inserts and deletes - do_execsql_test_on_specific_db {:memory:} drop-index-after-ops-1 { - CREATE TABLE t6 (x INTEGER PRIMARY KEY); - CREATE INDEX t_idx6 on t6 (x); - INSERT INTO t6 VALUES (1); - INSERT INTO t6 VALUES (2); - DELETE FROM t6 WHERE x = 1; - INSERT INTO t6 VALUES (3); - DROP INDEX t_idx6; - SELECT count(*) FROM sqlite_schema WHERE type='index' AND name='t_idx6'; - } {0} -} +# Test dropping index after multiple inserts and deletes +do_execsql_test_on_specific_db {:memory:} drop-index-after-ops-1 { + CREATE TABLE t6 (x INTEGER PRIMARY KEY); + CREATE INDEX t_idx6 on t6 (x); + INSERT INTO t6 VALUES (1); + INSERT INTO t6 VALUES (2); + DELETE FROM t6 WHERE x = 1; + INSERT INTO t6 VALUES (3); + DROP INDEX t_idx6; + SELECT count(*) FROM sqlite_schema WHERE type='index' AND name='t_idx6'; +} {0} diff --git a/testing/drop_table.test b/testing/drop_table.test index 3e2b56f85..9365b70d4 100755 --- a/testing/drop_table.test +++ b/testing/drop_table.test @@ -25,26 +25,23 @@ do_execsql_test_on_specific_db {:memory:} drop-table-if-exists-2 { SELECT 'success'; } {success} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - # Test dropping table with index - do_execsql_test_on_specific_db {:memory:} drop-table-with-index-1 { - CREATE TABLE t3 (x INTEGER PRIMARY KEY, y TEXT); - CREATE INDEX idx_t3_y ON t3(y); - INSERT INTO t3 VALUES(1, 'one'); - DROP TABLE t3; - SELECT count(*) FROM sqlite_schema WHERE tbl_name='t3'; - } {0} - # Test dropping table cleans up related schema entries - do_execsql_test_on_specific_db {:memory:} drop-table-schema-cleanup-1 { - CREATE TABLE t4 (x INTEGER PRIMARY KEY, y TEXT); - CREATE INDEX idx1_t4 ON t4(x); - CREATE INDEX idx2_t4 ON t4(y); - INSERT INTO t4 VALUES(1, 'one'); - DROP TABLE t4; - SELECT count(*) FROM sqlite_schema WHERE tbl_name='t4'; - } {0} -} - +# Test dropping table with index +do_execsql_test_on_specific_db {:memory:} drop-table-with-index-1 { + CREATE TABLE t3 (x INTEGER PRIMARY KEY, y TEXT); + CREATE INDEX idx_t3_y ON t3(y); + INSERT INTO t3 VALUES(1, 'one'); + DROP TABLE t3; + SELECT count(*) FROM sqlite_schema WHERE tbl_name='t3'; +} {0} +# Test dropping table cleans up related schema entries +do_execsql_test_on_specific_db {:memory:} drop-table-schema-cleanup-1 { + CREATE TABLE t4 (x INTEGER PRIMARY KEY, y TEXT); + CREATE INDEX idx1_t4 ON t4(x); + CREATE INDEX idx2_t4 ON t4(y); + INSERT INTO t4 VALUES(1, 'one'); + DROP TABLE t4; + SELECT count(*) FROM sqlite_schema WHERE tbl_name='t4'; +} {0} # Test dropping table after multiple inserts and deletes do_execsql_test_on_specific_db {:memory:} drop-table-after-ops-1 { diff --git a/testing/groupby.test b/testing/groupby.test index 3a2a05086..c159b9892 100755 --- a/testing/groupby.test +++ b/testing/groupby.test @@ -206,33 +206,29 @@ do_execsql_test group_by_no_sorting_required_and_const_agg_arg { } {CA,PW,ME,AS,LA,OH,AL,UT,WA,MO,WA,SC,AR,CO,OK,ME,FM,AR,CT,MT,TN,FL,MA,ND,LA,NE,KS,IN,RI,NH,IL,FM,WA,MH,RI,SC,AS,IL,VA,MI,ID,ME,WY,TN,IN,IN,UT,WA,AZ,VA,NM,IA,MP,WY,RI,OR,OR,FM,WA,DC,RI,GU,TX,HI,IL,TX,WY,OH,TX,CT,KY,NE,MH,AR,MN,IL,NH,HI,NV,UT,FL,MS,NM,NJ,CA,MS,GA,MT,GA,AL,IN,SC,PA,FL,CT,PA,GA,RI,HI,WV,VT,IA,PR,FM,MA,TX,MS,LA,MD,PA,TX,WY OR,SD,KS,MP,WA,VI,SC,SD,SD,MP,WA,MT,FM,IN,ME,OH,KY,RI,DC,MS,OK,VI,KY,MD,SC,OK,NY,WY,AK,MN,UT,NE,VA,MD,AZ,VI,SC,NV,IN,VA,HI,VI,MS,NE,WY,NY,GU,MT,AL,IA,VA,ND,MN,FM,IA,ID,IL,FL,PR,WA,AS,HI,NH,WI,FL,HI,AL,ID,DC,CT,IL,VT,AZ,VI,AK,PW,NC,SD,NV,WA,MO,MS,WY,VA,FM,MN,NH,MN,MT,TX,MS,FM,OH,GU,IN,WA,IA,PA,ID,MI,LA,GU,ND,AR,ND,WV,DC,NY,CO,CT,FM,CT,ND} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test_on_specific_db {:memory:} group_by_no_sorting_required_reordered_columns { - create table t0 (a INT, b INT, c INT); - create index a_b_idx on t0 (a, b); - insert into t0 values - (1,1,1), - (1,1,2), - (2,1,3), - (2,2,3), - (2,2,5); +do_execsql_test_on_specific_db {:memory:} group_by_no_sorting_required_reordered_columns { + create table t0 (a INT, b INT, c INT); + create index a_b_idx on t0 (a, b); + insert into t0 values + (1,1,1), + (1,1,2), + (2,1,3), + (2,2,3), + (2,2,5); - select c, b, a from t0 group by a, b; - } {1|1|1 - 3|1|2 - 3|2|2} -} + select c, b, a from t0 group by a, b; +} {1|1|1 +3|1|2 +3|2|2} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test distinct_agg_functions { - select first_name, sum(distinct age), count(distinct age), avg(distinct age) - from users - group by 1 - limit 3; - } {Aaron|1769|33|53.6060606060606 - Abigail|833|15|55.5333333333333 - Adam|1517|30|50.5666666666667} -} +do_execsql_test distinct_agg_functions { +select first_name, sum(distinct age), count(distinct age), avg(distinct age) +from users +group by 1 +limit 3; +} {Aaron|1769|33|53.6060606060606 +Abigail|833|15|55.5333333333333 +Adam|1517|30|50.5666666666667} do_execsql_test_on_specific_db {:memory:} having_or { CREATE TABLE users (first_name TEXT, age INTEGER); diff --git a/testing/insert.test b/testing/insert.test index 8944d1b77..78f881094 100755 --- a/testing/insert.test +++ b/testing/insert.test @@ -188,23 +188,21 @@ do_execsql_test_on_specific_db {:memory:} multi-rows { } {1|1 2|1} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test_on_specific_db {:memory:} unique_insert_no_pkey { - CREATE TABLE t2 (x INTEGER, y INTEGER UNIQUE); - INSERT INTO t2 (y) VALUES (1); - INSERT INTO t2 (y) VALUES (6); - SELECT * FROM t2; - } {|1 - |6} +do_execsql_test_on_specific_db {:memory:} unique_insert_no_pkey { + CREATE TABLE t2 (x INTEGER, y INTEGER UNIQUE); + INSERT INTO t2 (y) VALUES (1); + INSERT INTO t2 (y) VALUES (6); + SELECT * FROM t2; +} {|1 +|6} - do_execsql_test_on_specific_db {:memory:} unique_insert_with_pkey { - CREATE TABLE t2 (x INTEGER PRIMARY KEY, y INTEGER UNIQUE); - INSERT INTO t2 (y) VALUES (1); - INSERT INTO t2 (y) VALUES (6); - SELECT * FROM t2; - } {1|1 - 2|6} -} +do_execsql_test_on_specific_db {:memory:} unique_insert_with_pkey { + CREATE TABLE t2 (x INTEGER PRIMARY KEY, y INTEGER UNIQUE); + INSERT INTO t2 (y) VALUES (1); + INSERT INTO t2 (y) VALUES (6); + SELECT * FROM t2; +} {1|1 +2|6} do_execsql_test_on_specific_db {:memory:} not_null_insert { CREATE TABLE t2 (y INTEGER NOT NULL); @@ -333,61 +331,59 @@ do_execsql_test_on_specific_db {:memory:} insert_from_select_same_table_2 { 5|2|200 6|3|300} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test_on_specific_db {:memory:} insert_from_select_union { - CREATE TABLE t (a, b); - CREATE TABLE t2 (b, c); +do_execsql_test_on_specific_db {:memory:} insert_from_select_union { + CREATE TABLE t (a, b); + CREATE TABLE t2 (b, c); - INSERT INTO t2 VALUES (1, 100), (2, 200); - INSERT INTO t SELECT * FROM t UNION SELECT * FROM t2; - SELECT * FROM t; - } {1|100 - 2|200} + INSERT INTO t2 VALUES (1, 100), (2, 200); + INSERT INTO t SELECT * FROM t UNION SELECT * FROM t2; + SELECT * FROM t; +} {1|100 +2|200} - do_execsql_test_on_specific_db {:memory:} insert_from_select_union-2 { - CREATE TABLE t (a, b); - CREATE TABLE t2 (b, c); +do_execsql_test_on_specific_db {:memory:} insert_from_select_union-2 { + CREATE TABLE t (a, b); + CREATE TABLE t2 (b, c); - INSERT INTO t SELECT * FROM t UNION values(1, 100), (2, 200); - SELECT * FROM t; - } {1|100 - 2|200} + INSERT INTO t SELECT * FROM t UNION values(1, 100), (2, 200); + SELECT * FROM t; +} {1|100 +2|200} - do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect { - CREATE TABLE t (a, b); - CREATE TABLE t1 (a, b); - CREATE TABLE t2 (a, b); +do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect { + CREATE TABLE t (a, b); + CREATE TABLE t1 (a, b); + CREATE TABLE t2 (a, b); - INSERT INTO t1 VALUES (1, 100), (2, 200); - INSERT INTO t2 VALUES (2, 200), (3, 300); - INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2; - SELECT * FROM t; - } {2|200} + INSERT INTO t1 VALUES (1, 100), (2, 200); + INSERT INTO t2 VALUES (2, 200), (3, 300); + INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2; + SELECT * FROM t; +} {2|200} - do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect-2 { - CREATE TABLE t (a, b); - CREATE TABLE t1 (a, b); - CREATE TABLE t2 (a, b); - CREATE TABLE t3 (a, b); +do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect-2 { + CREATE TABLE t (a, b); + CREATE TABLE t1 (a, b); + CREATE TABLE t2 (a, b); + CREATE TABLE t3 (a, b); - INSERT INTO t1 VALUES (1, 100), (2, 200); - INSERT INTO t2 VALUES (2, 200), (3, 300); - INSERT INTO t3 VALUES (2, 200), (4, 400); - INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2 INTERSECT SELECT * FROM t3; - SELECT * FROM t; - } {2|200} + INSERT INTO t1 VALUES (1, 100), (2, 200); + INSERT INTO t2 VALUES (2, 200), (3, 300); + INSERT INTO t3 VALUES (2, 200), (4, 400); + INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2 INTERSECT SELECT * FROM t3; + SELECT * FROM t; +} {2|200} - do_execsql_test_on_specific_db {:memory:} insert_from_select_except { - CREATE TABLE t (a, b); - CREATE TABLE t1 (a, b); - CREATE TABLE t2 (a, b); +do_execsql_test_on_specific_db {:memory:} insert_from_select_except { + CREATE TABLE t (a, b); + CREATE TABLE t1 (a, b); + CREATE TABLE t2 (a, b); - INSERT INTO t1 VALUES (1, 100), (2, 200); - INSERT INTO t2 VALUES (2, 200), (3, 300); - INSERT INTO t SELECT * FROM t1 EXCEPT SELECT * FROM t2; - SELECT * FROM t; - } {1|100} -} + INSERT INTO t1 VALUES (1, 100), (2, 200); + INSERT INTO t2 VALUES (2, 200), (3, 300); + INSERT INTO t SELECT * FROM t1 EXCEPT SELECT * FROM t2; + SELECT * FROM t; +} {1|100} do_execsql_test_on_specific_db {:memory:} negative-primary-integer-key { CREATE TABLE t (a INTEGER PRIMARY KEY); @@ -411,21 +407,19 @@ do_execsql_test_on_specific_db {:memory:} rowid-overflow-random-generation { } {3} # regression test for incorrect processing of record header in the case of large text columns -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test_on_specific_db {:memory:} large-text-index-seek { - CREATE TABLE t (x TEXT, y); - CREATE INDEX t_idx ON t(x); - INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'a', 1); - INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'b', 2); - INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'c', 3); - INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'd', 4); - INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'e', 5); - INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'f', 6); - INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'g', 7); - INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'h', 8); - SELECT COUNT(*) FROM t WHERE x >= replace(hex(zeroblob(100)), '00', 'a'); - } {8} -} +do_execsql_test_on_specific_db {:memory:} large-text-index-seek { + CREATE TABLE t (x TEXT, y); + CREATE INDEX t_idx ON t(x); + INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'a', 1); + INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'b', 2); + INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'c', 3); + INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'd', 4); + INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'e', 5); + INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'f', 6); + INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'g', 7); + INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'h', 8); + SELECT COUNT(*) FROM t WHERE x >= replace(hex(zeroblob(100)), '00', 'a'); +} {8} do_execsql_test_skip_lines_on_specific_db 1 {:memory:} double-quote-string-literals { .dbconfig dqs_dml on diff --git a/testing/javascript/__test__/async.test.js b/testing/javascript/__test__/async.test.js index 2b6264d84..a27797fff 100644 --- a/testing/javascript/__test__/async.test.js +++ b/testing/javascript/__test__/async.test.js @@ -44,6 +44,10 @@ test.afterEach.always(async (t) => { }); test.serial("Open in-memory database", async (t) => { + if (process.env.PROVIDER === "serverless") { + t.pass("Skipping in-memory database test for serverless"); + return; + } const [db] = await connect(":memory:"); t.is(db.memory, true); }); @@ -63,7 +67,7 @@ test.serial("Statement.run() [positional]", async (t) => { const db = t.context.db; const stmt = await db.prepare("INSERT INTO users(name, email) VALUES (?, ?)"); - const info = stmt.run(["Carol", "carol@example.net"]); + const info = await stmt.run(["Carol", "carol@example.net"]); t.is(info.changes, 1); t.is(info.lastInsertRowid, 3); }); @@ -74,7 +78,7 @@ test.serial("Statement.get() [no parameters]", async (t) => { var stmt = 0; stmt = await db.prepare("SELECT * FROM users"); - t.is(stmt.get().name, "Alice"); + t.is((await stmt.get()).name, "Alice"); t.deepEqual(await stmt.raw().get(), [1, 'Alice', 'alice@example.org']); }); @@ -84,15 +88,15 @@ test.serial("Statement.get() [positional]", async (t) => { var stmt = 0; stmt = await db.prepare("SELECT * FROM users WHERE id = ?"); - t.is(stmt.get(0), undefined); - t.is(stmt.get([0]), undefined); - t.is(stmt.get(1).name, "Alice"); - t.is(stmt.get(2).name, "Bob"); + t.is(await stmt.get(0), undefined); + t.is(await stmt.get([0]), undefined); + t.is((await stmt.get(1)).name, "Alice"); + t.is((await stmt.get(2)).name, "Bob"); stmt = await db.prepare("SELECT * FROM users WHERE id = ?1"); - t.is(stmt.get({1: 0}), undefined); - t.is(stmt.get({1: 1}).name, "Alice"); - t.is(stmt.get({1: 2}).name, "Bob"); + t.is(await stmt.get({1: 0}), undefined); + t.is((await stmt.get({1: 1})).name, "Alice"); + t.is((await stmt.get({1: 2})).name, "Bob"); }); test.serial("Statement.get() [named]", async (t) => { @@ -101,19 +105,19 @@ test.serial("Statement.get() [named]", async (t) => { var stmt = undefined; stmt = await db.prepare("SELECT * FROM users WHERE id = :id"); - t.is(stmt.get({ id: 0 }), undefined); - t.is(stmt.get({ id: 1 }).name, "Alice"); - t.is(stmt.get({ id: 2 }).name, "Bob"); + t.is(await stmt.get({ id: 0 }), undefined); + t.is((await stmt.get({ id: 1 })).name, "Alice"); + t.is((await stmt.get({ id: 2 })).name, "Bob"); stmt = await db.prepare("SELECT * FROM users WHERE id = @id"); - t.is(stmt.get({ id: 0 }), undefined); - t.is(stmt.get({ id: 1 }).name, "Alice"); - t.is(stmt.get({ id: 2 }).name, "Bob"); + t.is(await stmt.get({ id: 0 }), undefined); + t.is((await stmt.get({ id: 1 })).name, "Alice"); + t.is((await stmt.get({ id: 2 })).name, "Bob"); stmt = await db.prepare("SELECT * FROM users WHERE id = $id"); - t.is(stmt.get({ id: 0 }), undefined); - t.is(stmt.get({ id: 1 }).name, "Alice"); - t.is(stmt.get({ id: 2 }).name, "Bob"); + t.is(await stmt.get({ id: 0 }), undefined); + t.is((await stmt.get({ id: 1 })).name, "Alice"); + t.is((await stmt.get({ id: 2 })).name, "Bob"); }); @@ -121,7 +125,7 @@ test.serial("Statement.get() [raw]", async (t) => { const db = t.context.db; const stmt = await db.prepare("SELECT * FROM users WHERE id = ?"); - t.deepEqual(stmt.raw().get(1), [1, "Alice", "alice@example.org"]); + t.deepEqual(await stmt.raw().get(1), [1, "Alice", "alice@example.org"]); }); test.skip("Statement.iterate() [empty]", async (t) => { @@ -399,7 +403,7 @@ test.skip("Timeout option", async (t) => { fs.unlinkSync(path); }); -test.serial("Concurrent writes over same connection", async (t) => { +test.skip("Concurrent writes over same connection", async (t) => { const db = t.context.db; await db.exec(` DROP TABLE IF EXISTS users; diff --git a/testing/join.test b/testing/join.test index 8a82a4f5c..664be10e6 100755 --- a/testing/join.test +++ b/testing/join.test @@ -228,21 +228,11 @@ do_execsql_test left-join-constant-condition-true-inner-join-constant-condition- select u.first_name, p.name, u2.first_name from users u left join products as p on 1 join users u2 on 0 limit 5; } {} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test join-utilizing-both-seekrowid-and-secondary-index { +do_execsql_test join-utilizing-both-seekrowid-and-secondary-index { select u.first_name, p.name from users u join products p on u.id = p.id and u.age > 70; - } {Matthew|boots - Nicholas|shorts - Jamie|hat} -} else { - # without index experimental the order is different since we don't use indexes - do_execsql_test join-utilizing-both-seekrowid-and-secondary-index { - select u.first_name, p.name from users u join products p on u.id = p.id and u.age > 70; - } {Jamie|hat - Nicholas|shorts - Matthew|boots} - -} +} {Matthew|boots +Nicholas|shorts +Jamie|hat} # important difference between regular SELECT * join and a SELECT * USING join is that the join keys are deduplicated # from the result in the USING case. diff --git a/testing/orderby.test b/testing/orderby.test index 555fccf6d..277ac9652 100755 --- a/testing/orderby.test +++ b/testing/orderby.test @@ -142,13 +142,11 @@ do_execsql_test case-insensitive-alias { select u.first_name as fF, count(1) > 0 as cC from users u where fF = 'Jamie' group by fF order by cC; } {Jamie|1} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test age_idx_order_desc { - select first_name from users order by age desc limit 3; - } {Robert - Sydney - Matthew} -} +do_execsql_test age_idx_order_desc { + select first_name from users order by age desc limit 3; +} {Robert +Sydney +Matthew} do_execsql_test rowid_or_integer_pk_desc { select first_name from users order by id desc limit 3; @@ -165,21 +163,19 @@ do_execsql_test orderby_desc_verify_rows { select count(1) from (select * from users order by age desc) } {10000} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test orderby_desc_with_offset { - select first_name, age from users order by age desc limit 3 offset 666; - } {Francis|94 - Matthew|94 - Theresa|94} +do_execsql_test orderby_desc_with_offset { + select first_name, age from users order by age desc limit 3 offset 666; +} {Francis|94 +Matthew|94 +Theresa|94} - do_execsql_test orderby_desc_with_filter { - select first_name, age from users where age <= 50 order by age desc limit 5; - } {Gerald|50 - Nicole|50 - Tammy|50 - Marissa|50 - Daniel|50} -} +do_execsql_test orderby_desc_with_filter { + select first_name, age from users where age <= 50 order by age desc limit 5; +} {Gerald|50 +Nicole|50 +Tammy|50 +Marissa|50 +Daniel|50} do_execsql_test orderby_asc_with_filter_range { select first_name, age from users where age <= 50 and age >= 49 order by age asc limit 5; diff --git a/testing/rollback.test b/testing/rollback.test index a22f024c7..1132b6dc9 100755 --- a/testing/rollback.test +++ b/testing/rollback.test @@ -127,15 +127,13 @@ do_execsql_test_on_specific_db {:memory:} schema-alter-rollback-and-repeat { select sql from sqlite_schema; } {"CREATE TABLE t (x, y)"} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test_on_specific_db {:memory:} schema-create-index-rollback { - create table t (x); - begin; - create index i on t(x); - rollback; - select sql from sqlite_schema; - } {"CREATE TABLE t (x)"} -} +do_execsql_test_on_specific_db {:memory:} schema-create-index-rollback { + create table t (x); + begin; + create index i on t(x); + rollback; + select sql from sqlite_schema; +} {"CREATE TABLE t (x)"} do_execsql_test_on_specific_db {:memory:} schema-drop-table-rollback { create table t (x); diff --git a/testing/select.test b/testing/select.test index ec434b538..9ed482e4e 100755 --- a/testing/select.test +++ b/testing/select.test @@ -309,322 +309,366 @@ do_execsql_test_error select-star-subquery { SELECT 1 FROM (SELECT *); } {no tables specified} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test_on_specific_db {:memory:} select-union-1 { + do_execsql_test_on_specific_db {:memory:} select-union-1 { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'); + + select * from t UNION select * from u; + } {x|x + y|y} + + do_execsql_test_on_specific_db {:memory:} select-union-all-union { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'); + INSERT INTO v VALUES('x','x'),('y','y'); + + select * from t UNION select * from u UNION ALL select * from v; + } {x|x + y|y + x|x + y|y} + + do_execsql_test_on_specific_db {:memory:} select-union-all-union-2 { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'); + INSERT INTO v VALUES('x','x'),('y','y'); + + select * from t UNION ALL select * from u UNION select * from v; + } {x|x + y|y} + + do_execsql_test_on_specific_db {:memory:} select-union-3 { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'); + INSERT INTO v VALUES('x','x'),('y','y'); + + select * from t UNION select * from u UNION select * from v; + } {x|x + y|y} + + do_execsql_test_on_specific_db {:memory:} select-union-4 { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'); + INSERT INTO v VALUES('x','x'),('y','y'); + + select * from t UNION select * from u UNION select * from v UNION select * from t; + } {x|x + y|y} + + do_execsql_test_on_specific_db {:memory:} select-union-all-union-3 { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'); + INSERT INTO v VALUES('x','x'),('y','y'); + + select * from t UNION select * from u UNION select * from v UNION ALL select * from t; + } {x|x + y|y + x|x + y|y} + + do_execsql_test_on_specific_db {:memory:} select-union-all-with-offset { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'),('z', 'z'); + + select * from t UNION ALL select * from u limit 1 offset 1; + } {y|y} + + do_execsql_test_on_specific_db {:memory:} select-union-with-offset { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'),('z', 'z'); + + select * from t UNION select * from u limit 1 offset 1; + } {y|y} + + do_execsql_test_on_specific_db {:memory:} select-intersect-1 { CREATE TABLE t (x TEXT, y TEXT); CREATE TABLE u (x TEXT, y TEXT); INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); - select * from t UNION select * from u; - } {x|x - y|y} + select * from t INTERSECT select * from u; + } {x|x} - do_execsql_test_on_specific_db {:memory:} select-union-all-union { + do_execsql_test_on_specific_db {:memory:} select-intersect-2 { CREATE TABLE t (x TEXT, y TEXT); CREATE TABLE u (x TEXT, y TEXT); CREATE TABLE v (x TEXT, y TEXT); INSERT INTO t VALUES('x','x'),('y','y'); INSERT INTO u VALUES('x','x'),('y','y'); - INSERT INTO v VALUES('x','x'),('y','y'); + INSERT INTO v VALUES('a','x'),('y','y'); - select * from t UNION select * from u UNION ALL select * from v; - } {x|x - y|y - x|x - y|y} + select * from t INTERSECT select * from u INTERSECT select * from v INTERSECT select * from t; + } {y|y} - do_execsql_test_on_specific_db {:memory:} select-union-all-union-2 { + do_execsql_test_on_specific_db {:memory:} select-intersect-union { CREATE TABLE t (x TEXT, y TEXT); CREATE TABLE u (x TEXT, y TEXT); CREATE TABLE v (x TEXT, y TEXT); INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('y','y'); - INSERT INTO v VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t UNION ALL select * from u UNION select * from v; - } {x|x - y|y} + select * from t INTERSECT select * from u UNION select * from v; + } {x|x + z|z} - do_execsql_test_on_specific_db {:memory:} select-union-3 { + do_execsql_test_on_specific_db {:memory:} select-union-intersect { CREATE TABLE t (x TEXT, y TEXT); CREATE TABLE u (x TEXT, y TEXT); CREATE TABLE v (x TEXT, y TEXT); INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('y','y'); - INSERT INTO v VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t UNION select * from u UNION select * from v; - } {x|x - y|y} + select * from t UNION select * from u INTERSECT select * from v; + } {x|x} - do_execsql_test_on_specific_db {:memory:} select-union-4 { + do_execsql_test_on_specific_db {:memory:} select-union-all-intersect { CREATE TABLE t (x TEXT, y TEXT); CREATE TABLE u (x TEXT, y TEXT); CREATE TABLE v (x TEXT, y TEXT); INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('y','y'); - INSERT INTO v VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t UNION select * from u UNION select * from v UNION select * from t; - } {x|x - y|y} + select * from t UNION ALL select * from u INTERSECT select * from v; + } {x|x} - do_execsql_test_on_specific_db {:memory:} select-union-all-union-3 { + do_execsql_test_on_specific_db {:memory:} select-intersect-union-all { CREATE TABLE t (x TEXT, y TEXT); CREATE TABLE u (x TEXT, y TEXT); CREATE TABLE v (x TEXT, y TEXT); INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); + + select * from t INTERSECT select * from u UNION ALL select * from v; + } {x|x + x|x + z|z} + + do_execsql_test_on_specific_db {:memory:} select-intersect-with-limit { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); + INSERT INTO u VALUES('x','x'),('y','y'), ('z','z'); + + select * from t INTERSECT select * from u limit 2; + } {x|x + y|y} + + do_execsql_test_on_specific_db {:memory:} select-intersect-with-offset { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); + INSERT INTO u VALUES('x','x'),('y','y'), ('z','z'); + + select * from t INTERSECT select * from u limit 2 offset 1; + } {y|y + z|z} + + do_execsql_test_on_specific_db {:memory:} select-intersect-union-with-limit { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); + INSERT INTO u VALUES('d','d'),('e','e'), ('z','z'); + INSERT INTO v VALUES('a','a'),('b','b'); + + select * from t INTERSECT select * from u UNION select * from v limit 3; + } {a|a + b|b + z|z} + + do_execsql_test_on_specific_db {:memory:} select-except-1 { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + + select * from t EXCEPT select * from u; + } {y|y} + + do_execsql_test_on_specific_db {:memory:} select-except-2 { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); INSERT INTO u VALUES('x','x'),('y','y'); + + select * from t EXCEPT select * from u; + } {} + + do_execsql_test_on_specific_db {:memory:} select-except-3 { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('a','y'); + INSERT INTO v VALUES('a','x'),('b','y'); + + select * from t EXCEPT select * from u EXCEPT select * from v; + } {y|y} + + do_execsql_test_on_specific_db {:memory:} select-except-limit { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + INSERT INTO t VALUES('a', 'a'),('x','x'),('y','y'),('z','z'); + INSERT INTO u VALUES('x','x'),('z','y'); + + select * from t EXCEPT select * from u limit 2; + } {a|a + y|y} + + do_execsql_test_on_specific_db {:memory:} select-except-union-all { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); INSERT INTO v VALUES('x','x'),('y','y'); - select * from t UNION select * from u UNION select * from v UNION ALL select * from t; - } {x|x - y|y - x|x - y|y} + select * from t EXCEPT select * from u UNION ALL select * from v; + } {y|y + x|x + y|y} - do_execsql_test_on_specific_db {:memory:} select-intersect-1 { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); + do_execsql_test_on_specific_db {:memory:} select-union-all-except { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('y','y'); - select * from t INTERSECT select * from u; - } {x|x} + select * from t UNION ALL select * from u EXCEPT select * from v; + } {z|y} - do_execsql_test_on_specific_db {:memory:} select-intersect-2 { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('y','y'); - INSERT INTO v VALUES('a','x'),('y','y'); + do_execsql_test_on_specific_db {:memory:} select-except-union { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t INTERSECT select * from u INTERSECT select * from v INTERSECT select * from t; - } {y|y} + select * from t EXCEPT select * from u UNION select * from v; + } {x|x + y|y + z|z} - do_execsql_test_on_specific_db {:memory:} select-intersect-union { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); + do_execsql_test_on_specific_db {:memory:} select-union-except { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t INTERSECT select * from u UNION select * from v; - } {x|x - z|z} + select * from t UNION select * from u EXCEPT select * from v; + } {y|y + z|y} - do_execsql_test_on_specific_db {:memory:} select-union-intersect { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); + do_execsql_test_on_specific_db {:memory:} select-except-intersect { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('y','y'),('z','z'); - select * from t UNION select * from u INTERSECT select * from v; - } {x|x} + select * from t EXCEPT select * from u INTERSECT select * from v; + } {y|y} - do_execsql_test_on_specific_db {:memory:} select-union-all-intersect { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); + do_execsql_test_on_specific_db {:memory:} select-intersect-except { + CREATE TABLE t (x TEXT, y TEXT); + CREATE TABLE u (x TEXT, y TEXT); + CREATE TABLE v (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t UNION ALL select * from u INTERSECT select * from v; - } {x|x} + select * from t INTERSECT select * from u EXCEPT select * from v; + } {} - do_execsql_test_on_specific_db {:memory:} select-intersect-union-all { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); + do_execsql_test_on_specific_db {:memory:} select-values-union { + CREATE TABLE t (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); - select * from t INTERSECT select * from u UNION ALL select * from v; - } {x|x - x|x - z|z} + values('x', 'x') UNION select * from t; + } {x|x + y|y} - do_execsql_test_on_specific_db {:memory:} select-intersect-with-limit { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); - INSERT INTO u VALUES('x','x'),('y','y'), ('z','z'); + do_execsql_test_on_specific_db {:memory:} select-values-union-2 { + CREATE TABLE t (x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); - select * from t INTERSECT select * from u limit 2; - } {x|x - y|y} + values('x', 'x'), ('y', 'y') UNION select * from t; + } {x|x + y|y} - do_execsql_test_on_specific_db {:memory:} select-intersect-union-with-limit { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); - INSERT INTO u VALUES('d','d'),('e','e'), ('z','z'); - INSERT INTO v VALUES('a','a'),('b','b'); - - select * from t INTERSECT select * from u UNION select * from v limit 3; - } {a|a - b|b - z|z} - - do_execsql_test_on_specific_db {:memory:} select-except-1 { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - - select * from t EXCEPT select * from u; - } {y|y} - - do_execsql_test_on_specific_db {:memory:} select-except-2 { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('y','y'); - - select * from t EXCEPT select * from u; - } {} - - do_execsql_test_on_specific_db {:memory:} select-except-3 { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('a','y'); - INSERT INTO v VALUES('a','x'),('b','y'); - - select * from t EXCEPT select * from u EXCEPT select * from v; - } {y|y} - - do_execsql_test_on_specific_db {:memory:} select-except-limit { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - INSERT INTO t VALUES('a', 'a'),('x','x'),('y','y'),('z','z'); - INSERT INTO u VALUES('x','x'),('z','y'); - - select * from t EXCEPT select * from u limit 2; - } {a|a - y|y} - - do_execsql_test_on_specific_db {:memory:} select-except-union-all { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('y','y'); - - select * from t EXCEPT select * from u UNION ALL select * from v; - } {y|y - x|x - y|y} - - do_execsql_test_on_specific_db {:memory:} select-union-all-except { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('y','y'); - - select * from t UNION ALL select * from u EXCEPT select * from v; - } {z|y} - - do_execsql_test_on_specific_db {:memory:} select-except-union { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); - - select * from t EXCEPT select * from u UNION select * from v; - } {x|x - y|y - z|z} - - do_execsql_test_on_specific_db {:memory:} select-union-except { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); - - select * from t UNION select * from u EXCEPT select * from v; - } {y|y - z|y} - - do_execsql_test_on_specific_db {:memory:} select-except-intersect { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('y','y'),('z','z'); - - select * from t EXCEPT select * from u INTERSECT select * from v; - } {y|y} - - do_execsql_test_on_specific_db {:memory:} select-intersect-except { - CREATE TABLE t (x TEXT, y TEXT); - CREATE TABLE u (x TEXT, y TEXT); - CREATE TABLE v (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); - - select * from t INTERSECT select * from u EXCEPT select * from v; - } {} - - do_execsql_test_on_specific_db {:memory:} select-values-union { + do_execsql_test_on_specific_db {:memory:} select-values-except { CREATE TABLE t (x TEXT, y TEXT); INSERT INTO t VALUES('x','x'),('y','y'); - values('x', 'x') UNION select * from t; - } {x|x - y|y} + select * from t EXCEPT values('x','x'),('z','y'); + } {y|y} - do_execsql_test_on_specific_db {:memory:} select-values-union-2 { - CREATE TABLE t (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); + do_execsql_test_on_specific_db {:memory:} select-values-union-all-limit { + CREATE TABLE t (x TEXT); + INSERT INTO t VALUES('x'), ('y'), ('z'); - values('x', 'x'), ('y', 'y') UNION select * from t; - } {x|x - y|y} + values('x') UNION ALL select * from t limit 3; + } {x + x + y} - do_execsql_test_on_specific_db {:memory:} select-values-except { - CREATE TABLE t (x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); + do_execsql_test_on_specific_db {:memory:} select-values-union-all-limit-1 { + CREATE TABLE t (x TEXT); + INSERT INTO t VALUES('x'), ('y'), ('z'); - select * from t EXCEPT values('x','x'),('z','y'); - } {y|y} + values('a'), ('b') UNION ALL select * from t limit 3; + } {a + b + x} - do_execsql_test_on_specific_db {:memory:} select-values-union-all-limit { - CREATE TABLE t (x TEXT); - INSERT INTO t VALUES('x'), ('y'), ('z'); + do_execsql_test_on_specific_db {:memory:} select-values-union-all-offset { + CREATE TABLE t (x TEXT); + INSERT INTO t VALUES('x'), ('y'), ('z'); - values('x') UNION ALL select * from t limit 3; - } {x - x - y} + values('a'), ('b') UNION ALL select * from t limit 3 offset 1; + } {b + x + y} - do_execsql_test_on_specific_db {:memory:} select-values-union-all-limit-2 { - CREATE TABLE t (x TEXT); - INSERT INTO t VALUES('x'), ('y'), ('z'); + do_execsql_test_on_specific_db {:memory:} select-values-union-all-offset-1 { + CREATE TABLE t (x TEXT); + INSERT INTO t VALUES('i'), ('j'), ('x'), ('y'), ('z'); - values('a'), ('b') UNION ALL select * from t limit 3; - } {a - b - x} -} + values('a') UNION ALL select * from t limit 3 offset 1; + } {i + j + x} do_execsql_test_on_specific_db {:memory:} select-no-match-in-leaf-page { CREATE TABLE t (a INTEGER PRIMARY KEY, b); @@ -687,3 +731,15 @@ do_execsql_test_skip_lines_on_specific_db 1 {:memory:} select-double-quotes-lite SELECT "literal_string" AS col; } {literal_string} +do_execsql_test_on_specific_db {:memory:} select-in-simple { + SELECT 1 IN (1, 2, 3); + SELECT 4 IN (1, 2, 3); +} {1 +0} + +do_execsql_test_on_specific_db {:memory:} select-in-complex { + CREATE TABLE test_table (id INTEGER, category TEXT, value INTEGER); + INSERT INTO test_table VALUES (1, 'A', 10), (2, 'B', 20), (3, 'A', 30), (4, 'C', 40); + SELECT * FROM test_table WHERE category IN ('A', 'B') AND value IN (10, 30, 40); +} {1|A|10 +3|A|30} diff --git a/testing/subquery.test b/testing/subquery.test index 1cec4c8ec..98ecec001 100644 --- a/testing/subquery.test +++ b/testing/subquery.test @@ -412,21 +412,19 @@ do_execsql_test subquery-ignore-unused-cte { select * from sub; } {Jamie} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - # Test verifying that select distinct works (distinct ages are 1-100) - do_execsql_test subquery-count-distinct-age { - select count(1) from (select distinct age from users); - } {100} +# Test verifying that select distinct works (distinct ages are 1-100) +do_execsql_test subquery-count-distinct-age { + select count(1) from (select distinct age from users); +} {100} - # Test verifying that select distinct works for multiple columns, and across joins - do_execsql_test subquery-count-distinct { - select count(1) from ( - select distinct first_name, name - from users u join products p - where u.id < 100 - ); - } {902} -} +# Test verifying that select distinct works for multiple columns, and across joins +do_execsql_test subquery-count-distinct { + select count(1) from ( + select distinct first_name, name + from users u join products p + where u.id < 100 + ); +} {902} do_execsql_test subquery-count-all { select count(1) from ( diff --git a/testing/update.test b/testing/update.test index 6f49f2354..091b5c1f0 100755 --- a/testing/update.test +++ b/testing/update.test @@ -190,64 +190,62 @@ do_execsql_test_on_specific_db {:memory:} update_cache_full_regression_test_#162 SELECT count(*) FROM t; } {1} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test_on_specific_db {:memory:} update_index_regression_test { - CREATE TABLE t (x, y); - CREATE INDEX tx ON t (x); - CREATE UNIQUE INDEX tyu ON t (y); - INSERT INTO t VALUES (1, 1); - SELECT x FROM t; -- uses tx index - SELECT y FROM t; -- uses ty index - UPDATE t SET x=2, y=2; - SELECT x FROM t; -- uses tx index - SELECT y FROM t; -- uses ty index - } {1 - 1 - 2 - 2} +do_execsql_test_on_specific_db {:memory:} update_index_regression_test { + CREATE TABLE t (x, y); + CREATE INDEX tx ON t (x); + CREATE UNIQUE INDEX tyu ON t (y); + INSERT INTO t VALUES (1, 1); + SELECT x FROM t; -- uses tx index + SELECT y FROM t; -- uses ty index + UPDATE t SET x=2, y=2; + SELECT x FROM t; -- uses tx index + SELECT y FROM t; -- uses ty index +} {1 +1 +2 +2} - do_execsql_test_on_specific_db {:memory:} update_rowid_alias_index_regression_test { - CREATE TABLE t (a INTEGER PRIMARY KEY, b); - CREATE INDEX idx_b ON t (b); - INSERT INTO t VALUES (1, 'foo'); - SELECT a FROM t WHERE b = 'foo'; - UPDATE t SET a = 2, b = 'bar'; - SELECT a FROM t WHERE b = 'bar'; - } {1 - 2} -} +do_execsql_test_on_specific_db {:memory:} update_rowid_alias_index_regression_test { + CREATE TABLE t (a INTEGER PRIMARY KEY, b); + CREATE INDEX idx_b ON t (b); + INSERT INTO t VALUES (1, 'foo'); + SELECT a FROM t WHERE b = 'foo'; + UPDATE t SET a = 2, b = 'bar'; + SELECT a FROM t WHERE b = 'bar'; +} {1 +2} do_execsql_test_on_specific_db {:memory:} update_where_or_regression_test { CREATE TABLE t (a INTEGER); INSERT INTO t VALUES (1), ('hi'); - UPDATE t SET a = X'6C6F76656C795F7265766F6C74' WHERE ~ 'gorgeous_thropy' OR NOT -3830873834.233324; + UPDATE t SET a = X'6C6F76656C795F7265766F6C74' WHERE ~ 'gorgeous_thropy' OR NOT -3830873834.233324; SELECT * from t; } {lovely_revolt lovely_revolt} do_execsql_test_in_memory_any_error update_primary_key_constraint_error { - CREATE TABLE eye (study REAL, spring BLOB, save TEXT, thank REAL, election INTEGER, PRIMARY KEY (election)); + CREATE TABLE eye (study REAL, spring BLOB, save TEXT, thank REAL, election INTEGER, PRIMARY KEY (election)); INSERT INTO eye VALUES (183559032.521585, x'6625d092', 'Trial six should.', 2606132742.43174, 2817); INSERT INTO eye VALUES (78255586.9204539, x'651061e8', 'World perhaps.', -5815764.49018679, 1917); UPDATE eye SET election = 6150; } do_execsql_test_in_memory_any_error update_primary_key_constraint_error_2 { - CREATE TABLE eye (study REAL, spring BLOB, save TEXT, thank REAL, election INTEGER, PRIMARY KEY (election)); - INSERT INTO eye VALUES (183559032.521585, x'6625d092', 'Trial six should.', 2606132742.43174, 2817); - INSERT INTO eye VALUES (78255586.9204539, x'651061e8', 'World perhaps.', -5815764.49018679, 1917); - INSERT INTO eye VALUES (53.3274327094467, x'f574c507', 'Senior wish degree.', -423.432750526747, 2650); - INSERT INTO eye VALUES (-908148213048.983, x'6d812051', 'Possible able.', 101.171781837336, 4100); + CREATE TABLE eye (study REAL, spring BLOB, save TEXT, thank REAL, election INTEGER, PRIMARY KEY (election)); + INSERT INTO eye VALUES (183559032.521585, x'6625d092', 'Trial six should.', 2606132742.43174, 2817); + INSERT INTO eye VALUES (78255586.9204539, x'651061e8', 'World perhaps.', -5815764.49018679, 1917); + INSERT INTO eye VALUES (53.3274327094467, x'f574c507', 'Senior wish degree.', -423.432750526747, 2650); + INSERT INTO eye VALUES (-908148213048.983, x'6d812051', 'Possible able.', 101.171781837336, 4100); INSERT INTO eye VALUES (-572332773760.924, x'd7a4d9fb', 'Money catch expect.', -271065488.756746, 4667); UPDATE eye SET election = 6150 WHERE election != 1917; } do_execsql_test_in_memory_any_error update_primary_key_constraint_error_3 { - CREATE TABLE eye (study REAL, spring BLOB, save TEXT, thank REAL, election INTEGER, PRIMARY KEY (election)); - INSERT INTO eye VALUES (183559032.521585, x'6625d092', 'Trial six should.', 2606132742.43174, 2817); - INSERT INTO eye VALUES (78255586.9204539, x'651061e8', 'World perhaps.', -5815764.49018679, 1917); - INSERT INTO eye VALUES (53.3274327094467, x'f574c507', 'Senior wish degree.', -423.432750526747, 2650); - INSERT INTO eye VALUES (-908148213048.983, x'6d812051', 'Possible able.', 101.171781837336, 4100); + CREATE TABLE eye (study REAL, spring BLOB, save TEXT, thank REAL, election INTEGER, PRIMARY KEY (election)); + INSERT INTO eye VALUES (183559032.521585, x'6625d092', 'Trial six should.', 2606132742.43174, 2817); + INSERT INTO eye VALUES (78255586.9204539, x'651061e8', 'World perhaps.', -5815764.49018679, 1917); + INSERT INTO eye VALUES (53.3274327094467, x'f574c507', 'Senior wish degree.', -423.432750526747, 2650); + INSERT INTO eye VALUES (-908148213048.983, x'6d812051', 'Possible able.', 101.171781837336, 4100); INSERT INTO eye VALUES (-572332773760.924, x'd7a4d9fb', 'Money catch expect.', -271065488.756746, 4667); UPDATE eye SET election = 6150 WHERE election > 1000 AND study > 1; } @@ -350,3 +348,21 @@ do_execsql_test_on_specific_db {:memory:} update-returning-null-values { INSERT INTO test (id, name, value) VALUES (1, 'test', 10); UPDATE test SET name = NULL, value = NULL WHERE id = 1 RETURNING id, name, value; } {1||} + +do_execsql_test_on_specific_db {:memory:} basic-row-values { + CREATE TABLE test (id INTEGER, name TEXT); + INSERT INTO test (id, name) VALUES (1, 'test'); + UPDATE test SET (id, name) = (2, 'mordor') RETURNING id, name; +} {2|mordor} + +do_execsql_test_in_memory_any_error parse-error-row-values { + CREATE TABLE test (id INTEGER, name TEXT); + INSERT INTO test (id, name) VALUES (1, 'test'); + UPDATE test SET (id, name) = (2); +} + +do_execsql_test_on_specific_db {:memory:} row-values-repeated-values-should-take-latter { + CREATE TABLE test (id INTEGER, name TEXT); + INSERT INTO test (id, name) VALUES (1, 'test'); + UPDATE test SET (name, name) = ('mordor', 'shire') RETURNING id, name; +} {1|shire} diff --git a/testing/where.test b/testing/where.test index 42a51e9d8..4763f142f 100755 --- a/testing/where.test +++ b/testing/where.test @@ -163,47 +163,24 @@ do_execsql_test where-clause-no-table-constant-condition-false-7 { select 1 where 'hamburger'; } {} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - # this test functions as an assertion that the index on users.age is being used, since the results are ordered by age without an order by. - do_execsql_test select-where-and { - select first_name, age from users where first_name = 'Jamie' and age > 80 - } {Jamie|87 - Jamie|88 - Jamie|88 - Jamie|92 - Jamie|94 - Jamie|99 - } - do_execsql_test select-where-or { - select first_name, age from users where first_name = 'Jamie' and age > 80 - } {Jamie|87 - Jamie|88 - Jamie|88 - Jamie|92 - Jamie|94 - Jamie|99 - } -} else { - # this test functions as an assertion that the index on users.age is being used, since the results are ordered by age without an order by. - do_execsql_test select-where-and { - select first_name, age from users where first_name = 'Jamie' and age > 80 - } {Jamie|94 - Jamie|88 - Jamie|99 - Jamie|92 - Jamie|87 - Jamie|88 - } - do_execsql_test select-where-or { - select first_name, age from users where first_name = 'Jamie' and age > 80 - } {Jamie|94 - Jamie|88 - Jamie|99 - Jamie|92 - Jamie|87 - Jamie|88 - } - +# this test functions as an assertion that the index on users.age is being used, since the results are ordered by age without an order by. +do_execsql_test select-where-and { + select first_name, age from users where first_name = 'Jamie' and age > 80 +} {Jamie|87 +Jamie|88 +Jamie|88 +Jamie|92 +Jamie|94 +Jamie|99 +} +do_execsql_test select-where-or { + select first_name, age from users where first_name = 'Jamie' and age > 80 +} {Jamie|87 +Jamie|88 +Jamie|88 +Jamie|92 +Jamie|94 +Jamie|99 } @@ -414,16 +391,9 @@ do_execsql_test where-age-index-seek-regression-test-2 { select count(1) from users where age > 0; } {10000} -if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-sqlite3-index-experimental" || $::env(SQLITE_EXEC) eq "sqlite3")} { - do_execsql_test where-age-index-seek-regression-test-3 { - select age from users where age > 90 limit 1; - } {91} -} else { - do_execsql_test where-age-index-seek-regression-test-3 { - select age from users where age > 90 limit 1; - } {94} - -} +do_execsql_test where-age-index-seek-regression-test-3 { + select age from users where age > 90 limit 1; +} {91} do_execsql_test where-simple-between { SELECT * FROM products WHERE price BETWEEN 70 AND 100; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index c50c69e40..445e1e335 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -18,6 +18,8 @@ path = "integration/mod.rs" anyhow.workspace = true env_logger = "0.10.1" turso_core = { path = "../core" } +turso = { path = "../bindings/rust" } +tokio = { version = "1.47", features = ["full"] } rusqlite = { version = "0.34", features = ["bundled"] } tempfile = "3.0.7" log = "0.4.22" diff --git a/tests/integration/fuzz/mod.rs b/tests/integration/fuzz/mod.rs index 5b669f455..5dd504aff 100644 --- a/tests/integration/fuzz/mod.rs +++ b/tests/integration/fuzz/mod.rs @@ -607,7 +607,13 @@ mod tests { // if the right most SELECT is a VALUES clause, no limit is not allowed if rng.random_bool(0.8) && !has_right_most_values { let limit_val = rng.random_range(0..=MAX_LIMIT_VALUE); // LIMIT 0 is valid - query = format!("{query} LIMIT {limit_val}"); + + if rng.random_bool(0.8) { + query = format!("{query} LIMIT {limit_val}"); + } else { + let offset_val = rng.random_range(0..=MAX_LIMIT_VALUE); + query = format!("{query} LIMIT {limit_val} OFFSET {offset_val}"); + } } log::debug!( diff --git a/bindings/rust/tests/transaction_isolation_fuzz.rs b/tests/integration/fuzz_transaction/mod.rs similarity index 80% rename from bindings/rust/tests/transaction_isolation_fuzz.rs rename to tests/integration/fuzz_transaction/mod.rs index 84190d7d4..c8f507c0d 100644 --- a/bindings/rust/tests/transaction_isolation_fuzz.rs +++ b/tests/integration/fuzz_transaction/mod.rs @@ -1,4 +1,4 @@ -use rand::seq::SliceRandom; +use rand::seq::IndexedRandom; use rand::Rng; use rand_chacha::{rand_core::SeedableRng, ChaCha8Rng}; use std::collections::HashMap; @@ -67,7 +67,10 @@ impl ShadowDb { fn commit_transaction(&mut self, tx_id: usize) { if let Some(tx_state) = self.transactions.remove(&tx_id) { - let tx_state = tx_state.unwrap(); + let Some(tx_state) = tx_state else { + // Transaction hasn't accessed the DB yet -> do nothing + return; + }; // Apply pending changes to committed state for op in tx_state.pending_changes { match op { @@ -158,7 +161,10 @@ impl ShadowDb { return self.committed_rows.values().cloned().collect(); }; if let Some(tx_state) = self.transactions.get(&tx_id) { - let tx_state = tx_state.as_ref().unwrap(); + let Some(tx_state) = tx_state.as_ref() else { + // Transaction hasn't accessed the DB yet -> see committed state + return self.committed_rows.values().cloned().collect(); + }; tx_state.visible_rows.values().cloned().collect() } else { // No transaction - see committed state @@ -167,6 +173,23 @@ impl ShadowDb { } } +#[derive(Debug, Clone)] +enum CheckpointMode { + Passive, + Restart, + Truncate, +} + +impl std::fmt::Display for CheckpointMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CheckpointMode::Passive => write!(f, "PASSIVE"), + CheckpointMode::Restart => write!(f, "RESTART"), + CheckpointMode::Truncate => write!(f, "TRUNCATE"), + } + } +} + #[derive(Debug, Clone)] enum Operation { Begin, @@ -175,6 +198,7 @@ enum Operation { Insert { id: i64, text: String }, Update { id: i64, text: String }, Delete { id: i64 }, + Checkpoint { mode: CheckpointMode }, Select, } @@ -192,6 +216,7 @@ impl std::fmt::Display for Operation { } Operation::Delete { id } => write!(f, "DELETE FROM test_table WHERE id = {id}"), Operation::Select => write!(f, "SELECT * FROM test_table"), + Operation::Checkpoint { mode } => write!(f, "PRAGMA wal_checkpoint({mode})"), } } } @@ -254,19 +279,8 @@ async fn test_multiple_connections_fuzz() { for op_num in 0..OPERATIONS_PER_CONNECTION { for (conn, conn_id, current_tx_id) in &mut connections { // Generate operation based on current transaction state - let visible_rows = if let Some(tx_id) = *current_tx_id { - // Take snapshot during first operation after a BEGIN, not immediately at BEGIN (the semantics is BEGIN DEFERRED) - let tx_state = shared_shadow_db.transactions.get(&tx_id).unwrap(); - if tx_state.is_none() { - shared_shadow_db.take_snapshot(tx_id); - } - shared_shadow_db.get_visible_rows(Some(tx_id)) - } else { - shared_shadow_db.get_visible_rows(None) // No transaction - }; - - let operation = - generate_operation(&mut rng, current_tx_id.is_some(), &visible_rows); + let (operation, visible_rows) = + generate_operation(&mut rng, *current_tx_id, &mut shared_shadow_db); println!("Connection {conn_id}(op={op_num}): {operation}"); @@ -484,6 +498,34 @@ async fn test_multiple_connections_fuzz() { ); } } + Operation::Checkpoint { mode } => { + let query = format!("PRAGMA wal_checkpoint({mode})"); + let mut rows = conn.query(&query, ()).await.unwrap(); + + match rows.next().await { + Ok(Some(row)) => { + let checkpoint_ok = matches!(row.get_value(0).unwrap(), Value::Integer(0)); + let wal_page_count = match row.get_value(1).unwrap() { + Value::Integer(count) => count.to_string(), + Value::Null => "NULL".to_string(), + _ => panic!("Unexpected value for wal_page_count: {:?}", row.get_value(1)), + }; + let checkpoint_count = match row.get_value(2).unwrap() { + Value::Integer(count) => count.to_string(), + Value::Null => "NULL".to_string(), + _ => panic!("Unexpected value for checkpoint_count: {:?}", row.get_value(2)), + }; + println!("Connection {conn_id}(op={op_num}) Checkpoint {mode}: OK: {checkpoint_ok}, wal_page_count: {wal_page_count}, checkpointed_count: {checkpoint_count}"); + } + Ok(None) => panic!("Connection {conn_id}(op={op_num}) Checkpoint {mode}: No rows returned"), + Err(e) => { + println!("Connection {conn_id}(op={op_num}) FAILED: {e}"); + if !e.to_string().contains("database is locked") && !e.to_string().contains("database table is locked") { + panic!("Unexpected error during checkpoint: {e}"); + } + } + } + } } } } @@ -492,62 +534,87 @@ async fn test_multiple_connections_fuzz() { fn generate_operation( rng: &mut ChaCha8Rng, - in_transaction: bool, - visible_rows: &[DbRow], -) -> Operation { - match rng.gen_range(0..100) { - // 10% chance to begin transaction + current_tx_id: Option, + shadow_db: &mut ShadowDb, +) -> (Operation, Vec) { + let in_transaction = current_tx_id.is_some(); + let mut get_visible_rows = |accesses_db: bool| { + if let Some(tx_id) = current_tx_id { + let tx_state = shadow_db.transactions.get(&tx_id).unwrap(); + // Take snapshot during first operation that accesses the DB after a BEGIN, not immediately at BEGIN (the semantics is BEGIN DEFERRED) + if accesses_db && tx_state.is_none() { + shadow_db.take_snapshot(tx_id); + } + shadow_db.get_visible_rows(Some(tx_id)) + } else { + shadow_db.get_visible_rows(None) // No transaction + } + }; + match rng.random_range(0..100) { 0..=9 => { if !in_transaction { - Operation::Begin + (Operation::Begin, get_visible_rows(false)) } else { - generate_data_operation(rng, visible_rows) + let visible_rows = get_visible_rows(true); + (generate_data_operation(rng, &visible_rows), visible_rows) } } - // 5% chance to commit 10..=14 => { if in_transaction { - Operation::Commit + (Operation::Commit, get_visible_rows(false)) } else { - generate_data_operation(rng, visible_rows) + let visible_rows = get_visible_rows(true); + (generate_data_operation(rng, &visible_rows), visible_rows) } } - // 5% chance to rollback 15..=19 => { if in_transaction { - Operation::Rollback + (Operation::Rollback, get_visible_rows(false)) } else { - generate_data_operation(rng, visible_rows) + let visible_rows = get_visible_rows(true); + (generate_data_operation(rng, &visible_rows), visible_rows) } } + 20..=22 => { + let mode = match rng.random_range(0..3) { + 0 => CheckpointMode::Passive, + 1 => CheckpointMode::Restart, + 2 => CheckpointMode::Truncate, + _ => unreachable!(), + }; + (Operation::Checkpoint { mode }, get_visible_rows(false)) + } // 80% chance for data operations - _ => generate_data_operation(rng, visible_rows), + _ => { + let visible_rows = get_visible_rows(true); + (generate_data_operation(rng, &visible_rows), visible_rows) + } } } fn generate_data_operation(rng: &mut ChaCha8Rng, visible_rows: &[DbRow]) -> Operation { - match rng.gen_range(0..4) { + match rng.random_range(0..4) { 0 => { // Insert - generate a new ID that doesn't exist let id = if visible_rows.is_empty() { - rng.gen_range(1..1000) + rng.random_range(1..1000) } else { let max_id = visible_rows.iter().map(|r| r.id).max().unwrap(); - rng.gen_range(max_id + 1..max_id + 100) + rng.random_range(max_id + 1..max_id + 100) }; - let text = format!("text_{}", rng.gen_range(1..1000)); + let text = format!("text_{}", rng.random_range(1..1000)); Operation::Insert { id, text } } 1 => { // Update - only if there are visible rows if visible_rows.is_empty() { // No rows to update, try insert instead - let id = rng.gen_range(1..1000); - let text = format!("text_{}", rng.gen_range(1..1000)); + let id = rng.random_range(1..1000); + let text = format!("text_{}", rng.random_range(1..1000)); Operation::Insert { id, text } } else { let id = visible_rows.choose(rng).unwrap().id; - let text = format!("updated_{}", rng.gen_range(1..1000)); + let text = format!("updated_{}", rng.random_range(1..1000)); Operation::Update { id, text } } } @@ -555,8 +622,8 @@ fn generate_data_operation(rng: &mut ChaCha8Rng, visible_rows: &[DbRow]) -> Oper // Delete - only if there are visible rows if visible_rows.is_empty() { // No rows to delete, try insert instead - let id = rng.gen_range(1..1000); - let text = format!("text_{}", rng.gen_range(1..1000)); + let id = rng.random_range(1..1000); + let text = format!("text_{}", rng.random_range(1..1000)); Operation::Insert { id, text } } else { let id = visible_rows.choose(rng).unwrap().id; diff --git a/tests/integration/mod.rs b/tests/integration/mod.rs index 5e99524ab..9d68aff5d 100644 --- a/tests/integration/mod.rs +++ b/tests/integration/mod.rs @@ -1,5 +1,6 @@ mod common; mod functions; mod fuzz; +mod fuzz_transaction; mod query_processing; mod wal; diff --git a/uv.lock b/uv.lock index 0d4aaeb13..7608d00bb 100644 --- a/uv.lock +++ b/uv.lock @@ -47,15 +47,6 @@ requires-dist = [ { name = "pyturso", editable = "bindings/python" }, ] -[[package]] -name = "certifi" -version = "2025.1.31" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/c9f1e32b7b1bf505bf26f0ef697775960db7932abeb7b516de930ba2705f/certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", size = 167577, upload-time = "2025-01-31T02:16:47.166Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393, upload-time = "2025-01-31T02:16:45.015Z" }, -] - [[package]] name = "cffi" version = "1.17.1" @@ -78,28 +69,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, ] -[[package]] -name = "charset-normalizer" -version = "3.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", size = 123188, upload-time = "2024-12-24T18:12:35.43Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/38/94/ce8e6f63d18049672c76d07d119304e1e2d7c6098f0841b51c666e9f44a0/charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda", size = 195698, upload-time = "2024-12-24T18:11:05.834Z" }, - { url = "https://files.pythonhosted.org/packages/24/2e/dfdd9770664aae179a96561cc6952ff08f9a8cd09a908f259a9dfa063568/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313", size = 140162, upload-time = "2024-12-24T18:11:07.064Z" }, - { url = "https://files.pythonhosted.org/packages/24/4e/f646b9093cff8fc86f2d60af2de4dc17c759de9d554f130b140ea4738ca6/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9", size = 150263, upload-time = "2024-12-24T18:11:08.374Z" }, - { url = "https://files.pythonhosted.org/packages/5e/67/2937f8d548c3ef6e2f9aab0f6e21001056f692d43282b165e7c56023e6dd/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b", size = 142966, upload-time = "2024-12-24T18:11:09.831Z" }, - { url = "https://files.pythonhosted.org/packages/52/ed/b7f4f07de100bdb95c1756d3a4d17b90c1a3c53715c1a476f8738058e0fa/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11", size = 144992, upload-time = "2024-12-24T18:11:12.03Z" }, - { url = "https://files.pythonhosted.org/packages/96/2c/d49710a6dbcd3776265f4c923bb73ebe83933dfbaa841c5da850fe0fd20b/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f", size = 147162, upload-time = "2024-12-24T18:11:13.372Z" }, - { url = "https://files.pythonhosted.org/packages/b4/41/35ff1f9a6bd380303dea55e44c4933b4cc3c4850988927d4082ada230273/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd", size = 140972, upload-time = "2024-12-24T18:11:14.628Z" }, - { url = "https://files.pythonhosted.org/packages/fb/43/c6a0b685fe6910d08ba971f62cd9c3e862a85770395ba5d9cad4fede33ab/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2", size = 149095, upload-time = "2024-12-24T18:11:17.672Z" }, - { url = "https://files.pythonhosted.org/packages/4c/ff/a9a504662452e2d2878512115638966e75633519ec11f25fca3d2049a94a/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886", size = 152668, upload-time = "2024-12-24T18:11:18.989Z" }, - { url = "https://files.pythonhosted.org/packages/6c/71/189996b6d9a4b932564701628af5cee6716733e9165af1d5e1b285c530ed/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601", size = 150073, upload-time = "2024-12-24T18:11:21.507Z" }, - { url = "https://files.pythonhosted.org/packages/e4/93/946a86ce20790e11312c87c75ba68d5f6ad2208cfb52b2d6a2c32840d922/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd", size = 145732, upload-time = "2024-12-24T18:11:22.774Z" }, - { url = "https://files.pythonhosted.org/packages/cd/e5/131d2fb1b0dddafc37be4f3a2fa79aa4c037368be9423061dccadfd90091/charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407", size = 95391, upload-time = "2024-12-24T18:11:24.139Z" }, - { url = "https://files.pythonhosted.org/packages/27/f2/4f9a69cc7712b9b5ad8fdb87039fd89abba997ad5cbe690d1835d40405b0/charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971", size = 102702, upload-time = "2024-12-24T18:11:26.535Z" }, - { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767, upload-time = "2024-12-24T18:12:32.852Z" }, -] - [[package]] name = "colorama" version = "0.4.6" @@ -137,53 +106,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9f/b0/e0dca6da9170aefc07515cce067b97178cefafb512d00a87a1c717d2efd5/coverage-7.6.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b9f222de8cded79c49bf184bdbc06630d4c58eec9459b939b4a690c82ed05657", size = 211453, upload-time = "2024-08-04T19:44:45.677Z" }, ] -[[package]] -name = "cryptography" -version = "44.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cd/25/4ce80c78963834b8a9fd1cc1266be5ed8d1840785c0f2e1b73b8d128d505/cryptography-44.0.2.tar.gz", hash = "sha256:c63454aa261a0cf0c5b4718349629793e9e634993538db841165b3df74f37ec0", size = 710807, upload-time = "2025-03-02T00:01:37.692Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/92/ef/83e632cfa801b221570c5f58c0369db6fa6cef7d9ff859feab1aae1a8a0f/cryptography-44.0.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:efcfe97d1b3c79e486554efddeb8f6f53a4cdd4cf6086642784fa31fc384e1d7", size = 6676361, upload-time = "2025-03-02T00:00:06.528Z" }, - { url = "https://files.pythonhosted.org/packages/30/ec/7ea7c1e4c8fc8329506b46c6c4a52e2f20318425d48e0fe597977c71dbce/cryptography-44.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29ecec49f3ba3f3849362854b7253a9f59799e3763b0c9d0826259a88efa02f1", size = 3952350, upload-time = "2025-03-02T00:00:09.537Z" }, - { url = "https://files.pythonhosted.org/packages/27/61/72e3afdb3c5ac510330feba4fc1faa0fe62e070592d6ad00c40bb69165e5/cryptography-44.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc821e161ae88bfe8088d11bb39caf2916562e0a2dc7b6d56714a48b784ef0bb", size = 4166572, upload-time = "2025-03-02T00:00:12.03Z" }, - { url = "https://files.pythonhosted.org/packages/26/e4/ba680f0b35ed4a07d87f9e98f3ebccb05091f3bf6b5a478b943253b3bbd5/cryptography-44.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c00b6b757b32ce0f62c574b78b939afab9eecaf597c4d624caca4f9e71e7843", size = 3958124, upload-time = "2025-03-02T00:00:14.518Z" }, - { url = "https://files.pythonhosted.org/packages/9c/e8/44ae3e68c8b6d1cbc59040288056df2ad7f7f03bbcaca6b503c737ab8e73/cryptography-44.0.2-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7bdcd82189759aba3816d1f729ce42ffded1ac304c151d0a8e89b9996ab863d5", size = 3678122, upload-time = "2025-03-02T00:00:17.212Z" }, - { url = "https://files.pythonhosted.org/packages/27/7b/664ea5e0d1eab511a10e480baf1c5d3e681c7d91718f60e149cec09edf01/cryptography-44.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4973da6ca3db4405c54cd0b26d328be54c7747e89e284fcff166132eb7bccc9c", size = 4191831, upload-time = "2025-03-02T00:00:19.696Z" }, - { url = "https://files.pythonhosted.org/packages/2a/07/79554a9c40eb11345e1861f46f845fa71c9e25bf66d132e123d9feb8e7f9/cryptography-44.0.2-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4e389622b6927d8133f314949a9812972711a111d577a5d1f4bee5e58736b80a", size = 3960583, upload-time = "2025-03-02T00:00:22.488Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6d/858e356a49a4f0b591bd6789d821427de18432212e137290b6d8a817e9bf/cryptography-44.0.2-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f514ef4cd14bb6fb484b4a60203e912cfcb64f2ab139e88c2274511514bf7308", size = 4191753, upload-time = "2025-03-02T00:00:25.038Z" }, - { url = "https://files.pythonhosted.org/packages/b2/80/62df41ba4916067fa6b125aa8c14d7e9181773f0d5d0bd4dcef580d8b7c6/cryptography-44.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1bc312dfb7a6e5d66082c87c34c8a62176e684b6fe3d90fcfe1568de675e6688", size = 4079550, upload-time = "2025-03-02T00:00:26.929Z" }, - { url = "https://files.pythonhosted.org/packages/f3/cd/2558cc08f7b1bb40683f99ff4327f8dcfc7de3affc669e9065e14824511b/cryptography-44.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b721b8b4d948b218c88cb8c45a01793483821e709afe5f622861fc6182b20a7", size = 4298367, upload-time = "2025-03-02T00:00:28.735Z" }, - { url = "https://files.pythonhosted.org/packages/71/59/94ccc74788945bc3bd4cf355d19867e8057ff5fdbcac781b1ff95b700fb1/cryptography-44.0.2-cp37-abi3-win32.whl", hash = "sha256:51e4de3af4ec3899d6d178a8c005226491c27c4ba84101bfb59c901e10ca9f79", size = 2772843, upload-time = "2025-03-02T00:00:30.592Z" }, - { url = "https://files.pythonhosted.org/packages/ca/2c/0d0bbaf61ba05acb32f0841853cfa33ebb7a9ab3d9ed8bb004bd39f2da6a/cryptography-44.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:c505d61b6176aaf982c5717ce04e87da5abc9a36a5b39ac03905c4aafe8de7aa", size = 3209057, upload-time = "2025-03-02T00:00:33.393Z" }, - { url = "https://files.pythonhosted.org/packages/9e/be/7a26142e6d0f7683d8a382dd963745e65db895a79a280a30525ec92be890/cryptography-44.0.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e0ddd63e6bf1161800592c71ac794d3fb8001f2caebe0966e77c5234fa9efc3", size = 6677789, upload-time = "2025-03-02T00:00:36.009Z" }, - { url = "https://files.pythonhosted.org/packages/06/88/638865be7198a84a7713950b1db7343391c6066a20e614f8fa286eb178ed/cryptography-44.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81276f0ea79a208d961c433a947029e1a15948966658cf6710bbabb60fcc2639", size = 3951919, upload-time = "2025-03-02T00:00:38.581Z" }, - { url = "https://files.pythonhosted.org/packages/d7/fc/99fe639bcdf58561dfad1faa8a7369d1dc13f20acd78371bb97a01613585/cryptography-44.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1e657c0f4ea2a23304ee3f964db058c9e9e635cc7019c4aa21c330755ef6fd", size = 4167812, upload-time = "2025-03-02T00:00:42.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/7b/aafe60210ec93d5d7f552592a28192e51d3c6b6be449e7fd0a91399b5d07/cryptography-44.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6210c05941994290f3f7f175a4a57dbbb2afd9273657614c506d5976db061181", size = 3958571, upload-time = "2025-03-02T00:00:46.026Z" }, - { url = "https://files.pythonhosted.org/packages/16/32/051f7ce79ad5a6ef5e26a92b37f172ee2d6e1cce09931646eef8de1e9827/cryptography-44.0.2-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1c3572526997b36f245a96a2b1713bf79ce99b271bbcf084beb6b9b075f29ea", size = 3679832, upload-time = "2025-03-02T00:00:48.647Z" }, - { url = "https://files.pythonhosted.org/packages/78/2b/999b2a1e1ba2206f2d3bca267d68f350beb2b048a41ea827e08ce7260098/cryptography-44.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b042d2a275c8cee83a4b7ae30c45a15e6a4baa65a179a0ec2d78ebb90e4f6699", size = 4193719, upload-time = "2025-03-02T00:00:51.397Z" }, - { url = "https://files.pythonhosted.org/packages/72/97/430e56e39a1356e8e8f10f723211a0e256e11895ef1a135f30d7d40f2540/cryptography-44.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d03806036b4f89e3b13b6218fefea8d5312e450935b1a2d55f0524e2ed7c59d9", size = 3960852, upload-time = "2025-03-02T00:00:53.317Z" }, - { url = "https://files.pythonhosted.org/packages/89/33/c1cf182c152e1d262cac56850939530c05ca6c8d149aa0dcee490b417e99/cryptography-44.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c7362add18b416b69d58c910caa217f980c5ef39b23a38a0880dfd87bdf8cd23", size = 4193906, upload-time = "2025-03-02T00:00:56.49Z" }, - { url = "https://files.pythonhosted.org/packages/e1/99/87cf26d4f125380dc674233971069bc28d19b07f7755b29861570e513650/cryptography-44.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8cadc6e3b5a1f144a039ea08a0bdb03a2a92e19c46be3285123d32029f40a922", size = 4081572, upload-time = "2025-03-02T00:00:59.995Z" }, - { url = "https://files.pythonhosted.org/packages/b3/9f/6a3e0391957cc0c5f84aef9fbdd763035f2b52e998a53f99345e3ac69312/cryptography-44.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6f101b1f780f7fc613d040ca4bdf835c6ef3b00e9bd7125a4255ec574c7916e4", size = 4298631, upload-time = "2025-03-02T00:01:01.623Z" }, - { url = "https://files.pythonhosted.org/packages/e2/a5/5bc097adb4b6d22a24dea53c51f37e480aaec3465285c253098642696423/cryptography-44.0.2-cp39-abi3-win32.whl", hash = "sha256:3dc62975e31617badc19a906481deacdeb80b4bb454394b4098e3f2525a488c5", size = 2773792, upload-time = "2025-03-02T00:01:04.133Z" }, - { url = "https://files.pythonhosted.org/packages/33/cf/1f7649b8b9a3543e042d3f348e398a061923ac05b507f3f4d95f11938aa9/cryptography-44.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:5f6f90b72d8ccadb9c6e311c775c8305381db88374c65fa1a68250aa8a9cb3a6", size = 3210957, upload-time = "2025-03-02T00:01:06.987Z" }, -] - -[[package]] -name = "deprecated" -version = "1.2.18" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744, upload-time = "2025-01-27T10:46:25.7Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" }, -] - [[package]] name = "faker" version = "37.1.0" @@ -196,15 +118,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/a1/8936bc8e79af80ca38288dd93ed44ed1f9d63beb25447a4c59e746e01f8d/faker-37.1.0-py3-none-any.whl", hash = "sha256:dc2f730be71cb770e9c715b13374d80dbcee879675121ab51f9683d262ae9a1c", size = 1918783, upload-time = "2025-03-24T16:14:00.051Z" }, ] -[[package]] -name = "idna" -version = "3.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, -] - [[package]] name = "iniconfig" version = "2.1.0" @@ -358,23 +271,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/4f/3fb47d6cbc08c7e00f92300e64ba655428c05c56b8ab6723bd290bae6458/pydantic_core-2.33.0-cp313-cp313t-win_amd64.whl", hash = "sha256:8a1d581e8cdbb857b0e0e81df98603376c1a5c34dc5e54039dcc00f043df81e7", size = 1931234, upload-time = "2025-03-26T20:28:29.237Z" }, ] -[[package]] -name = "pygithub" -version = "2.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecated" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "pynacl" }, - { name = "requests" }, - { name = "typing-extensions" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c0/88/e08ab18dc74b2916f48703ed1a797d57cb64eca0e23b0a9254e13cfe3911/pygithub-2.6.1.tar.gz", hash = "sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf", size = 3659473, upload-time = "2025-02-21T13:45:58.262Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/fc/a444cd19ccc8c4946a512f3827ed0b3565c88488719d800d54a75d541c0b/PyGithub-2.6.1-py3-none-any.whl", hash = "sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3", size = 410451, upload-time = "2025-02-21T13:45:55.519Z" }, -] - [[package]] name = "pygments" version = "2.19.1" @@ -384,40 +280,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload-time = "2025-01-06T17:26:25.553Z" }, ] -[[package]] -name = "pyjwt" -version = "2.10.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785, upload-time = "2024-11-28T03:43:29.933Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997, upload-time = "2024-11-28T03:43:27.893Z" }, -] - -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - -[[package]] -name = "pynacl" -version = "1.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a7/22/27582568be639dfe22ddb3902225f91f2f17ceff88ce80e4db396c8986da/PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", size = 3392854, upload-time = "2022-01-07T22:05:41.134Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/75/0b8ede18506041c0bf23ac4d8e2971b4161cd6ce630b177d0a08eb0d8857/PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", size = 349920, upload-time = "2022-01-07T22:05:49.156Z" }, - { url = "https://files.pythonhosted.org/packages/59/bb/fddf10acd09637327a97ef89d2a9d621328850a72f1fdc8c08bdf72e385f/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", size = 601722, upload-time = "2022-01-07T22:05:50.989Z" }, - { url = "https://files.pythonhosted.org/packages/5d/70/87a065c37cca41a75f2ce113a5a2c2aa7533be648b184ade58971b5f7ccc/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", size = 680087, upload-time = "2022-01-07T22:05:52.539Z" }, - { url = "https://files.pythonhosted.org/packages/ee/87/f1bb6a595f14a327e8285b9eb54d41fef76c585a0edef0a45f6fc95de125/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", size = 856678, upload-time = "2022-01-07T22:05:54.251Z" }, - { url = "https://files.pythonhosted.org/packages/66/28/ca86676b69bf9f90e710571b67450508484388bfce09acf8a46f0b8c785f/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", size = 1133660, upload-time = "2022-01-07T22:05:56.056Z" }, - { url = "https://files.pythonhosted.org/packages/3d/85/c262db650e86812585e2bc59e497a8f59948a005325a11bbbc9ecd3fe26b/PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", size = 663824, upload-time = "2022-01-07T22:05:57.434Z" }, - { url = "https://files.pythonhosted.org/packages/fd/1a/cc308a884bd299b651f1633acb978e8596c71c33ca85e9dc9fa33a5399b9/PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", size = 1117912, upload-time = "2022-01-07T22:05:58.665Z" }, - { url = "https://files.pythonhosted.org/packages/25/2d/b7df6ddb0c2a33afdb358f8af6ea3b8c4d1196ca45497dd37a56f0c122be/PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543", size = 204624, upload-time = "2022-01-07T22:06:00.085Z" }, - { url = "https://files.pythonhosted.org/packages/5e/22/d3db169895faaf3e2eda892f005f433a62db2decbcfbc2f61e6517adfa87/PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", size = 212141, upload-time = "2022-01-07T22:06:01.861Z" }, -] - [[package]] name = "pytest" version = "8.3.1" @@ -503,21 +365,6 @@ dev = [ { name = "typing-extensions", specifier = ">=4.13.0" }, ] -[[package]] -name = "requests" -version = "2.32.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218, upload-time = "2024-05-29T15:37:49.536Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928, upload-time = "2024-05-29T15:37:47.027Z" }, -] - [[package]] name = "rich" version = "14.0.0" @@ -560,12 +407,6 @@ wheels = [ name = "scripts" version = "0.1.0" source = { virtual = "scripts" } -dependencies = [ - { name = "pygithub" }, -] - -[package.metadata] -requires-dist = [{ name = "pygithub", specifier = ">=2.6.1" }] [[package]] name = "turso-test" @@ -611,43 +452,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be76 wheels = [ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, ] - -[[package]] -name = "urllib3" -version = "2.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/78/16493d9c386d8e60e442a35feac5e00f0913c0f4b7c217c11e8ec2ff53e0/urllib3-2.4.0.tar.gz", hash = "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466", size = 390672, upload-time = "2025-04-10T15:23:39.232Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/11/cc635220681e93a0183390e26485430ca2c7b5f9d33b15c74c2861cb8091/urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813", size = 128680, upload-time = "2025-04-10T15:23:37.377Z" }, -] - -[[package]] -name = "wrapt" -version = "1.17.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/fc/e91cc220803d7bc4db93fb02facd8461c37364151b8494762cc88b0fbcef/wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3", size = 55531, upload-time = "2025-01-14T10:35:45.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/b9/0ffd557a92f3b11d4c5d5e0c5e4ad057bd9eb8586615cdaf901409920b14/wrapt-1.17.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6ed6ffac43aecfe6d86ec5b74b06a5be33d5bb9243d055141e8cabb12aa08125", size = 53800, upload-time = "2025-01-14T10:34:21.571Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ef/8be90a0b7e73c32e550c73cfb2fa09db62234227ece47b0e80a05073b375/wrapt-1.17.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:35621ae4c00e056adb0009f8e86e28eb4a41a4bfa8f9bfa9fca7d343fe94f998", size = 38824, upload-time = "2025-01-14T10:34:22.999Z" }, - { url = "https://files.pythonhosted.org/packages/36/89/0aae34c10fe524cce30fe5fc433210376bce94cf74d05b0d68344c8ba46e/wrapt-1.17.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a604bf7a053f8362d27eb9fefd2097f82600b856d5abe996d623babd067b1ab5", size = 38920, upload-time = "2025-01-14T10:34:25.386Z" }, - { url = "https://files.pythonhosted.org/packages/3b/24/11c4510de906d77e0cfb5197f1b1445d4fec42c9a39ea853d482698ac681/wrapt-1.17.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cbabee4f083b6b4cd282f5b817a867cf0b1028c54d445b7ec7cfe6505057cf8", size = 88690, upload-time = "2025-01-14T10:34:28.058Z" }, - { url = "https://files.pythonhosted.org/packages/71/d7/cfcf842291267bf455b3e266c0c29dcb675b5540ee8b50ba1699abf3af45/wrapt-1.17.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49703ce2ddc220df165bd2962f8e03b84c89fee2d65e1c24a7defff6f988f4d6", size = 80861, upload-time = "2025-01-14T10:34:29.167Z" }, - { url = "https://files.pythonhosted.org/packages/d5/66/5d973e9f3e7370fd686fb47a9af3319418ed925c27d72ce16b791231576d/wrapt-1.17.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8112e52c5822fc4253f3901b676c55ddf288614dc7011634e2719718eaa187dc", size = 89174, upload-time = "2025-01-14T10:34:31.702Z" }, - { url = "https://files.pythonhosted.org/packages/a7/d3/8e17bb70f6ae25dabc1aaf990f86824e4fd98ee9cadf197054e068500d27/wrapt-1.17.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fee687dce376205d9a494e9c121e27183b2a3df18037f89d69bd7b35bcf59e2", size = 86721, upload-time = "2025-01-14T10:34:32.91Z" }, - { url = "https://files.pythonhosted.org/packages/6f/54/f170dfb278fe1c30d0ff864513cff526d624ab8de3254b20abb9cffedc24/wrapt-1.17.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:18983c537e04d11cf027fbb60a1e8dfd5190e2b60cc27bc0808e653e7b218d1b", size = 79763, upload-time = "2025-01-14T10:34:34.903Z" }, - { url = "https://files.pythonhosted.org/packages/4a/98/de07243751f1c4a9b15c76019250210dd3486ce098c3d80d5f729cba029c/wrapt-1.17.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:703919b1633412ab54bcf920ab388735832fdcb9f9a00ae49387f0fe67dad504", size = 87585, upload-time = "2025-01-14T10:34:36.13Z" }, - { url = "https://files.pythonhosted.org/packages/f9/f0/13925f4bd6548013038cdeb11ee2cbd4e37c30f8bfd5db9e5a2a370d6e20/wrapt-1.17.2-cp313-cp313-win32.whl", hash = "sha256:abbb9e76177c35d4e8568e58650aa6926040d6a9f6f03435b7a522bf1c487f9a", size = 36676, upload-time = "2025-01-14T10:34:37.962Z" }, - { url = "https://files.pythonhosted.org/packages/bf/ae/743f16ef8c2e3628df3ddfd652b7d4c555d12c84b53f3d8218498f4ade9b/wrapt-1.17.2-cp313-cp313-win_amd64.whl", hash = "sha256:69606d7bb691b50a4240ce6b22ebb319c1cfb164e5f6569835058196e0f3a845", size = 38871, upload-time = "2025-01-14T10:34:39.13Z" }, - { url = "https://files.pythonhosted.org/packages/3d/bc/30f903f891a82d402ffb5fda27ec1d621cc97cb74c16fea0b6141f1d4e87/wrapt-1.17.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:4a721d3c943dae44f8e243b380cb645a709ba5bd35d3ad27bc2ed947e9c68192", size = 56312, upload-time = "2025-01-14T10:34:40.604Z" }, - { url = "https://files.pythonhosted.org/packages/8a/04/c97273eb491b5f1c918857cd26f314b74fc9b29224521f5b83f872253725/wrapt-1.17.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:766d8bbefcb9e00c3ac3b000d9acc51f1b399513f44d77dfe0eb026ad7c9a19b", size = 40062, upload-time = "2025-01-14T10:34:45.011Z" }, - { url = "https://files.pythonhosted.org/packages/4e/ca/3b7afa1eae3a9e7fefe499db9b96813f41828b9fdb016ee836c4c379dadb/wrapt-1.17.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e496a8ce2c256da1eb98bd15803a79bee00fc351f5dfb9ea82594a3f058309e0", size = 40155, upload-time = "2025-01-14T10:34:47.25Z" }, - { url = "https://files.pythonhosted.org/packages/89/be/7c1baed43290775cb9030c774bc53c860db140397047cc49aedaf0a15477/wrapt-1.17.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d615e4fe22f4ad3528448c193b218e077656ca9ccb22ce2cb20db730f8d306", size = 113471, upload-time = "2025-01-14T10:34:50.934Z" }, - { url = "https://files.pythonhosted.org/packages/32/98/4ed894cf012b6d6aae5f5cc974006bdeb92f0241775addad3f8cd6ab71c8/wrapt-1.17.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5aaeff38654462bc4b09023918b7f21790efb807f54c000a39d41d69cf552cb", size = 101208, upload-time = "2025-01-14T10:34:52.297Z" }, - { url = "https://files.pythonhosted.org/packages/ea/fd/0c30f2301ca94e655e5e057012e83284ce8c545df7661a78d8bfca2fac7a/wrapt-1.17.2-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a7d15bbd2bc99e92e39f49a04653062ee6085c0e18b3b7512a4f2fe91f2d681", size = 109339, upload-time = "2025-01-14T10:34:53.489Z" }, - { url = "https://files.pythonhosted.org/packages/75/56/05d000de894c4cfcb84bcd6b1df6214297b8089a7bd324c21a4765e49b14/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e3890b508a23299083e065f435a492b5435eba6e304a7114d2f919d400888cc6", size = 110232, upload-time = "2025-01-14T10:34:55.327Z" }, - { url = "https://files.pythonhosted.org/packages/53/f8/c3f6b2cf9b9277fb0813418e1503e68414cd036b3b099c823379c9575e6d/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8c8b293cd65ad716d13d8dd3624e42e5a19cc2a2f1acc74b30c2c13f15cb61a6", size = 100476, upload-time = "2025-01-14T10:34:58.055Z" }, - { url = "https://files.pythonhosted.org/packages/a7/b1/0bb11e29aa5139d90b770ebbfa167267b1fc548d2302c30c8f7572851738/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c82b8785d98cdd9fed4cac84d765d234ed3251bd6afe34cb7ac523cb93e8b4f", size = 106377, upload-time = "2025-01-14T10:34:59.3Z" }, - { url = "https://files.pythonhosted.org/packages/6a/e1/0122853035b40b3f333bbb25f1939fc1045e21dd518f7f0922b60c156f7c/wrapt-1.17.2-cp313-cp313t-win32.whl", hash = "sha256:13e6afb7fe71fe7485a4550a8844cc9ffbe263c0f1a1eea569bc7091d4898555", size = 37986, upload-time = "2025-01-14T10:35:00.498Z" }, - { url = "https://files.pythonhosted.org/packages/09/5e/1655cf481e079c1f22d0cabdd4e51733679932718dc23bf2db175f329b76/wrapt-1.17.2-cp313-cp313t-win_amd64.whl", hash = "sha256:eaf675418ed6b3b31c7a989fd007fa7c3be66ce14e5c3b27336383604c9da85c", size = 40750, upload-time = "2025-01-14T10:35:03.378Z" }, - { url = "https://files.pythonhosted.org/packages/2d/82/f56956041adef78f849db6b289b282e72b55ab8045a75abad81898c28d19/wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8", size = 23594, upload-time = "2025-01-14T10:35:44.018Z" }, -] diff --git a/vendored/sqlite3-parser/src/parser/ast/mod.rs b/vendored/sqlite3-parser/src/parser/ast/mod.rs index d1e1ce501..605840f31 100644 --- a/vendored/sqlite3-parser/src/parser/ast/mod.rs +++ b/vendored/sqlite3-parser/src/parser/ast/mod.rs @@ -1292,6 +1292,39 @@ impl QualifiedName { } } +/// Ordered set of column names +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Names(Vec); + +impl Names { + /// Initialize + pub fn new(name: Name) -> Self { + let mut dn = Self(Vec::new()); + dn.0.push(name); + dn + } + /// Single column name + pub fn single(name: Name) -> Self { + let mut dn = Self(Vec::with_capacity(1)); + dn.0.push(name); + dn + } + /// Push name + pub fn insert(&mut self, name: Name) -> Result<(), ParserError> { + self.0.push(name); + Ok(()) + } +} + +impl Deref for Names { + type Target = Vec; + + fn deref(&self) -> &Vec { + &self.0 + } +} + /// Ordered set of distinct column names #[derive(Clone, Debug, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] @@ -1319,6 +1352,7 @@ impl DistinctNames { Ok(()) } } + impl Deref for DistinctNames { type Target = IndexSet; @@ -1735,7 +1769,7 @@ pub enum InsertBody { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct Set { /// column name(s) - pub col_names: DistinctNames, + pub col_names: Names, /// expression pub expr: Expr, } diff --git a/vendored/sqlite3-parser/src/parser/parse.y b/vendored/sqlite3-parser/src/parser/parse.y index dcc4c4d16..912c72bbb 100644 --- a/vendored/sqlite3-parser/src/parser/parse.y +++ b/vendored/sqlite3-parser/src/parser/parse.y @@ -802,22 +802,28 @@ cmd ::= with(C) UPDATE orconf(R) xfullname(X) indexed_opt(I) SET setlist(Y) from } %endif +%type reidlist {Names} +reidlist(A) ::= reidlist(A) COMMA nm(Y). + {let id = Y; A.insert(id)?;} +reidlist(A) ::= nm(Y). + { A = Names::new(Y); /*A-overwrites-Y*/} %type setlist {Vec} setlist(A) ::= setlist(A) COMMA nm(X) EQ expr(Y). { - let s = Set{ col_names: DistinctNames::single(X), expr: Y }; + let s = Set{ col_names: Names::single(X), expr: Y }; A.push(s); } -setlist(A) ::= setlist(A) COMMA LP idlist(X) RP EQ expr(Y). { +setlist(A) ::= setlist(A) COMMA LP reidlist(X) RP EQ expr(Y). { let s = Set{ col_names: X, expr: Y }; A.push(s); } setlist(A) ::= nm(X) EQ expr(Y). { - A = vec![Set{ col_names: DistinctNames::single(X), expr: Y }]; + A = vec![Set{ col_names: Names::single(X), expr: Y }]; + } -setlist(A) ::= LP idlist(X) RP EQ expr(Y). { +setlist(A) ::= LP reidlist(X) RP EQ expr(Y). { A = vec![Set{ col_names: X, expr: Y }]; }