diff --git a/.github/workflows/rust_perf.yml b/.github/workflows/rust_perf.yml index cd8deee60..eecd59c20 100644 --- a/.github/workflows/rust_perf.yml +++ b/.github/workflows/rust_perf.yml @@ -11,7 +11,7 @@ env: jobs: bench: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: useblacksmith/setup-node@v5 @@ -37,7 +37,7 @@ jobs: # on pull events isn't compatible with this workflow being required to pass branch protection. fail-on-alert: false comment-on-alert: true - comment-always: false + comment-always: true # Nyrkiö configuration # Get yours from https://nyrkio.com/docs/getting-started nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} @@ -54,7 +54,7 @@ jobs: nyrkio-settings-threshold: 0% clickbench: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: useblacksmith/setup-node@v5 @@ -77,7 +77,7 @@ jobs: # on pull events isn't compatible with this workflow being required to pass branch protection. fail-on-alert: false comment-on-alert: true - comment-always: false + comment-always: true # Nyrkiö configuration # Get yours from https://nyrkio.com/docs/getting-started nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} @@ -101,7 +101,7 @@ jobs: nyrkio-public: true tpc-h-criterion: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-latest env: DB_FILE: "perf/tpc-h/TPC-H.db" steps: @@ -138,7 +138,7 @@ jobs: # on pull events isn't compatible with this workflow being required to pass branch protection. fail-on-alert: false comment-on-alert: true - comment-always: false + comment-always: true # Nyrkiö configuration # Get yours from https://nyrkio.com/docs/getting-started nyrkio-token: ${{ secrets.NYRKIO_JWT_TOKEN }} @@ -155,14 +155,14 @@ jobs: nyrkio-settings-threshold: 0% tpc-h: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: TPC-H run: ./perf/tpc-h/benchmark.sh vfs-bench-compile: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: useblacksmith/rust-cache@v3 diff --git a/Cargo.lock b/Cargo.lock index 8de5085c3..d2e512cf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,12 +122,6 @@ dependencies = [ "rand 0.9.2", ] -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_log-sys" version = "0.3.2" @@ -320,9 +314,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" dependencies = [ "serde", ] @@ -356,7 +350,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", - "regex-automata 0.4.9", + "regex-automata", "serde", ] @@ -466,11 +460,10 @@ checksum = "18758054972164c3264f7c8386f5fc6da6114cb46b619fd365d4e3b2dc3ae487" [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", @@ -518,9 +511,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.32" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" +checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" dependencies = [ "clap_builder", "clap_derive", @@ -528,9 +521,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.32" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8" +checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" dependencies = [ "anstream", "anstyle", @@ -552,9 +545,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" dependencies = [ "heck", "proc-macro2", @@ -666,7 +659,7 @@ dependencies = [ "anyhow", "assert_cmd", "ctor 0.5.0", - "env_logger 0.10.2", + "env_logger 0.11.7", "log", "rand 0.9.2", "rand_chacha 0.9.0", @@ -796,7 +789,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "crossterm_winapi", "parking_lot", "rustix 0.38.44", @@ -1120,7 +1113,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" dependencies = [ "log", - "regex", ] [[package]] @@ -1129,11 +1121,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ - "humantime", - "is-terminal", "log", "regex", - "termcolor", ] [[package]] @@ -1145,7 +1134,6 @@ dependencies = [ "anstream", "anstyle", "env_filter", - "jiff", "log", ] @@ -1514,7 +1502,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5220b8ba44c68a9a7f7a7659e864dd73692e417ef0211bea133c7b74e031eeb9" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "libc", "libgit2-sys", "log", @@ -1605,12 +1593,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "humantime" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" - [[package]] name = "iana-time-zone" version = "0.1.62" @@ -1793,9 +1775,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1815,7 +1797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.11.0", + "indexmap 2.11.1", "is-terminal", "itoa", "log", @@ -1832,7 +1814,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "inotify-sys", "libc", ] @@ -1861,7 +1843,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c2f96dfbc20c12b9b4f12eef60472d8c29b9c3f29463570dcb47e4a48551168" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "cfg-if", "libc", ] @@ -1928,30 +1910,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" -[[package]] -name = "jiff" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c102670231191d07d37a35af3eb77f1f0dbf7a71be51a962dcd57ea607be7260" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde", -] - -[[package]] -name = "jiff-static" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdde31a9d349f1b1f51a0b3714a5940ac022976f4b49485fc04be052b183b4c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - [[package]] name = "jni" version = "0.21.1" @@ -2089,7 +2047,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "libc", "redox_syscall", ] @@ -2182,10 +2140,10 @@ dependencies = [ "chrono", "clap", "dirs 6.0.0", - "env_logger 0.10.2", + "env_logger 0.11.7", "garde", "hex", - "indexmap 2.11.0", + "indexmap 2.11.1", "itertools 0.14.0", "json5", "log", @@ -2194,7 +2152,7 @@ dependencies = [ "rand 0.9.2", "rand_chacha 0.9.0", "regex", - "regex-syntax 0.8.5", + "regex-syntax", "rusqlite", "schemars 1.0.4", "serde", @@ -2276,11 +2234,11 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -2399,7 +2357,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96671d5c84cee3ae4cab96386b9f953b22569ece9677b9fdd1492550a165eca5" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "ctor 0.4.2", "napi-build", "napi-sys", @@ -2475,7 +2433,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "cfg-if", "cfg_aliases", "libc", @@ -2493,7 +2451,7 @@ version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "filetime", "fsevent-sys", "inotify", @@ -2512,16 +2470,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "nu-ansi-term" version = "0.50.1" @@ -2588,7 +2536,7 @@ version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "libc", "once_cell", "onig_sys", @@ -2633,12 +2581,6 @@ dependencies = [ "log", ] -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "owo-colors" version = "4.2.0" @@ -2696,7 +2638,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.12", + "thiserror 2.0.16", "ucd-trie", ] @@ -2758,7 +2700,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eac26e981c03a6e53e0aee43c113e3202f5581d5360dae7bd2c70e800dd0451d" dependencies = [ "base64", - "indexmap 2.11.0", + "indexmap 2.11.1", "quick-xml 0.32.0", "serde", "time", @@ -2825,15 +2767,6 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -3176,7 +3109,7 @@ version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", ] [[package]] @@ -3198,7 +3131,7 @@ checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ "getrandom 0.2.15", "libredox", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -3229,17 +3162,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -3250,15 +3174,9 @@ checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.5" @@ -3339,7 +3257,7 @@ version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -3384,7 +3302,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "errno", "libc", "linux-raw-sys 0.4.15", @@ -3397,7 +3315,7 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "errno", "libc", "linux-raw-sys 0.9.3", @@ -3416,7 +3334,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "cfg-if", "clipboard-win", "fd-lock", @@ -3656,7 +3574,7 @@ dependencies = [ "anyhow", "garde", "hex", - "indexmap 2.11.0", + "indexmap 2.11.1", "itertools 0.14.0", "rand 0.9.2", "rand_chacha 0.9.0", @@ -3817,7 +3735,7 @@ dependencies = [ "once_cell", "onig", "plist", - "regex-syntax 0.8.5", + "regex-syntax", "serde", "serde_derive", "serde_json", @@ -3845,15 +3763,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "terminal_size" version = "0.4.2" @@ -3913,11 +3822,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.16", ] [[package]] @@ -3933,9 +3842,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -4049,7 +3958,7 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.11.1", "serde", "serde_spanned", "toml_datetime", @@ -4071,7 +3980,7 @@ version = "0.22.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.11.1", "serde", "serde_spanned", "toml_datetime", @@ -4142,14 +4051,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", - "nu-ansi-term 0.46.0", + "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", @@ -4162,10 +4071,10 @@ dependencies = [ name = "turso" version = "0.2.0-pre.3" dependencies = [ - "rand 0.8.5", - "rand_chacha 0.3.1", + "rand 0.9.2", + "rand_chacha 0.9.0", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "turso_core", ] @@ -4175,7 +4084,7 @@ name = "turso-java" version = "0.2.0-pre.3" dependencies = [ "jni", - "thiserror 2.0.12", + "thiserror 2.0.16", "turso_core", ] @@ -4191,12 +4100,12 @@ dependencies = [ "csv", "ctrlc", "dirs 5.0.1", - "env_logger 0.10.2", + "env_logger 0.11.7", "libc", "limbo_completion", "miette", "mimalloc", - "nu-ansi-term 0.50.1", + "nu-ansi-term", "rustyline", "schemars 0.8.22", "serde", @@ -4220,7 +4129,7 @@ dependencies = [ "aes", "aes-gcm", "antithesis_sdk", - "bitflags 2.9.0", + "bitflags 2.9.4", "built", "bytemuck", "cfg_block", @@ -4249,7 +4158,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.9.0", "regex", - "regex-syntax 0.8.5", + "regex-syntax", "rstest", "rusqlite", "rustix 1.0.7", @@ -4260,12 +4169,11 @@ dependencies = [ "strum_macros", "tempfile", "test-log", - "thiserror 1.0.69", + "thiserror 2.0.16", "tracing", "turso_ext", "turso_macros", "turso_parser", - "turso_sqlite3_parser", "twox-hash", "uncased", "uuid", @@ -4324,7 +4232,7 @@ dependencies = [ name = "turso_parser" version = "0.2.0-pre.3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "criterion", "fallible-iterator", "miette", @@ -4332,7 +4240,7 @@ dependencies = [ "serde", "strum", "strum_macros", - "thiserror 1.0.69", + "thiserror 2.0.16", "turso_macros", ] @@ -4353,11 +4261,11 @@ dependencies = [ name = "turso_sqlite3_parser" version = "0.2.0-pre.3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", "cc", "env_logger 0.11.7", "fallible-iterator", - "indexmap 2.11.0", + "indexmap 2.11.1", "log", "memchr", "miette", @@ -4400,7 +4308,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", "tracing-subscriber", @@ -4773,9 +4681,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" [[package]] name = "windows-sys" @@ -5006,7 +4914,26 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.4", +] + +[[package]] +name = "write-throughput" +version = "0.1.0" +dependencies = [ + "clap", + "futures", + "tokio", + "tracing-subscriber", + "turso", +] + +[[package]] +name = "write-throughput-sqlite" +version = "0.1.0" +dependencies = [ + "clap", + "rusqlite", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ab0a875fc..2771c2a31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,11 +30,11 @@ members = [ "sync/engine", "sql_generation", "whopper", + "perf/throughput/turso", + "perf/throughput/rusqlite", ] exclude = [ "perf/latency/limbo", - "perf/throughput/rusqlite", - "perf/throughput/turso" ] [workspace.package] @@ -71,10 +71,30 @@ mimalloc = { version = "0.1.47", default-features = false } rusqlite = { version = "0.37.0", features = ["bundled"] } itertools = "0.14.0" rand = "0.9.2" +rand_chacha = "0.9.0" tracing = "0.1.41" schemars = "1.0.4" garde = "0.22" parking_lot = "0.12.4" +tokio = { version = "1.0", default-features = false } +tracing-subscriber = "0.3.20" +futures = "0.3" +clap = "4.5.47" +thiserror = "2.0.16" +tempfile = "3.20.0" +indexmap = "2.11.1" +miette = "7.6.0" +bitflags = "2.9.4" +fallible-iterator = "0.3.0" +criterion = "0.5" +chrono = { version = "0.4.42", default-features = false } +hex = "0.4" +antithesis_sdk = "0.2" +cfg-if = "1.0.0" +tracing-appender = "0.2.3" +env_logger = { version = "0.11.6", default-features = false } +regex = "1.11.1" +regex-syntax = { version = "0.8.5", default-features = false } [profile.release] debug = "line-tables-only" diff --git a/Dockerfile.antithesis b/Dockerfile.antithesis index cd97169ae..825262673 100644 --- a/Dockerfile.antithesis +++ b/Dockerfile.antithesis @@ -23,6 +23,7 @@ COPY ./extensions ./extensions/ COPY ./macros ./macros/ COPY ./packages ./packages/ COPY ./parser ./parser/ +COPY ./perf/throughput/turso ./perf/throughput/turso/ COPY ./simulator ./simulator/ COPY ./sql_generation ./sql_generation COPY ./sqlite3 ./sqlite3/ @@ -63,6 +64,7 @@ COPY --from=planner /app/extensions ./extensions/ COPY --from=planner /app/macros ./macros/ COPY --from=planner /app/packages ./packages/ COPY --from=planner /app/parser ./parser/ +COPY --from=planner /perf/throughput/turso ./perf/throughput/turso/ COPY --from=planner /app/simulator ./simulator/ COPY --from=planner /app/sql_generation ./sql_generation COPY --from=planner /app/sqlite3 ./sqlite3/ diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 93858e0a1..e978fc3a7 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -14,6 +14,6 @@ crate-type = ["cdylib"] path = "rs_src/lib.rs" [dependencies] -turso_core = { path = "../../core", features = ["io_uring"] } +turso_core = { workspace = true, features = ["io_uring"] } jni = "0.21.1" -thiserror = "2.0.9" +thiserror = { workspace = true } diff --git a/bindings/javascript/Cargo.toml b/bindings/javascript/Cargo.toml index 836780122..1b5001839 100644 --- a/bindings/javascript/Cargo.toml +++ b/bindings/javascript/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["cdylib", "lib"] turso_core = { workspace = true } napi = { version = "3.1.3", default-features = false, features = ["napi6"] } napi-derive = { version = "3.1.1", default-features = true } -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +tracing-subscriber = { workspace = true, features = ["env-filter"] } tracing.workspace = true [features] diff --git a/bindings/javascript/packages/native/index.js b/bindings/javascript/packages/native/index.js index 49c26ac10..503940658 100644 --- a/bindings/javascript/packages/native/index.js +++ b/bindings/javascript/packages/native/index.js @@ -81,8 +81,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-android-arm64') const bindingPackageVersion = require('@tursodatabase/database-android-arm64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -97,8 +97,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-android-arm-eabi') const bindingPackageVersion = require('@tursodatabase/database-android-arm-eabi/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -117,8 +117,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-win32-x64-msvc') const bindingPackageVersion = require('@tursodatabase/database-win32-x64-msvc/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -133,8 +133,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-win32-ia32-msvc') const bindingPackageVersion = require('@tursodatabase/database-win32-ia32-msvc/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -149,8 +149,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-win32-arm64-msvc') const bindingPackageVersion = require('@tursodatabase/database-win32-arm64-msvc/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -168,8 +168,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-darwin-universal') const bindingPackageVersion = require('@tursodatabase/database-darwin-universal/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -184,8 +184,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-darwin-x64') const bindingPackageVersion = require('@tursodatabase/database-darwin-x64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -200,8 +200,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-darwin-arm64') const bindingPackageVersion = require('@tursodatabase/database-darwin-arm64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -220,8 +220,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-freebsd-x64') const bindingPackageVersion = require('@tursodatabase/database-freebsd-x64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -236,8 +236,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-freebsd-arm64') const bindingPackageVersion = require('@tursodatabase/database-freebsd-arm64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -257,8 +257,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-x64-musl') const bindingPackageVersion = require('@tursodatabase/database-linux-x64-musl/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -273,8 +273,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-x64-gnu') const bindingPackageVersion = require('@tursodatabase/database-linux-x64-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -291,8 +291,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-arm64-musl') const bindingPackageVersion = require('@tursodatabase/database-linux-arm64-musl/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -307,8 +307,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-arm64-gnu') const bindingPackageVersion = require('@tursodatabase/database-linux-arm64-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -325,8 +325,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-arm-musleabihf') const bindingPackageVersion = require('@tursodatabase/database-linux-arm-musleabihf/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -341,8 +341,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-arm-gnueabihf') const bindingPackageVersion = require('@tursodatabase/database-linux-arm-gnueabihf/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -359,8 +359,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-riscv64-musl') const bindingPackageVersion = require('@tursodatabase/database-linux-riscv64-musl/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -375,8 +375,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-riscv64-gnu') const bindingPackageVersion = require('@tursodatabase/database-linux-riscv64-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -392,8 +392,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-ppc64-gnu') const bindingPackageVersion = require('@tursodatabase/database-linux-ppc64-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -408,8 +408,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-linux-s390x-gnu') const bindingPackageVersion = require('@tursodatabase/database-linux-s390x-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -428,8 +428,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-openharmony-arm64') const bindingPackageVersion = require('@tursodatabase/database-openharmony-arm64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -444,8 +444,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-openharmony-x64') const bindingPackageVersion = require('@tursodatabase/database-openharmony-x64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -460,8 +460,8 @@ function requireNative() { try { const binding = require('@tursodatabase/database-openharmony-arm') const bindingPackageVersion = require('@tursodatabase/database-openharmony-arm/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { diff --git a/bindings/javascript/src/browser.rs b/bindings/javascript/src/browser.rs index b2c2047d2..92c818b4c 100644 --- a/bindings/javascript/src/browser.rs +++ b/bindings/javascript/src/browser.rs @@ -146,9 +146,9 @@ impl IO for Opfs { if result >= 0 { Ok(Arc::new(OpfsFile { handle: result })) } else if result == -404 { - Err(turso_core::LimboError::InternalError( - "files must be created in advance for OPFS IO".to_string(), - )) + Err(turso_core::LimboError::InternalError(format!( + "unexpected path {path}: files must be created in advance for OPFS IO" + ))) } else { Err(turso_core::LimboError::InternalError(format!( "unexpected file lookup error: {result}" diff --git a/bindings/javascript/sync/Cargo.toml b/bindings/javascript/sync/Cargo.toml index 029a04fb1..2f3f3d177 100644 --- a/bindings/javascript/sync/Cargo.toml +++ b/bindings/javascript/sync/Cargo.toml @@ -17,7 +17,7 @@ turso_sync_engine = { workspace = true } turso_core = { workspace = true } turso_node = { workspace = true } genawaiter = { version = "0.99.1", default-features = false } -tracing-subscriber = "0.3.19" +tracing-subscriber = { workspace = true } [build-dependencies] napi-build = "2.2.3" diff --git a/bindings/javascript/sync/packages/browser/promise.test.ts b/bindings/javascript/sync/packages/browser/promise.test.ts index 152a7841a..e30163af0 100644 --- a/bindings/javascript/sync/packages/browser/promise.test.ts +++ b/bindings/javascript/sync/packages/browser/promise.test.ts @@ -160,7 +160,7 @@ test('checkpoint', async () => { expect((await db1.stats()).revertWal).toBe(revertWal); }) -test('persistence', async () => { +test('persistence-push', async () => { { const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL }); await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)"); @@ -203,6 +203,63 @@ test('persistence', async () => { } }) +test('persistence-offline', async () => { + { + const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL }); + await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)"); + await db.exec("DELETE FROM q"); + await db.push(); + await db.close(); + } + const path = `test-${(Math.random() * 10000) | 0}.db`; + { + const db = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL }); + await db.exec(`INSERT INTO q VALUES ('k1', 'v1')`); + await db.exec(`INSERT INTO q VALUES ('k2', 'v2')`); + await db.push(); + await db.close(); + } + { + const db = await connect({ path: path, url: "https://not-valid-url.localhost" }); + const rows = await db.prepare("SELECT * FROM q").all(); + const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }]; + expect(rows.sort(localeCompare)).toEqual(expected.sort(localeCompare)) + await db.close(); + } +}) + +test('persistence-pull-push', async () => { + { + const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL }); + await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)"); + await db.exec("DELETE FROM q"); + await db.push(); + await db.close(); + } + const path1 = `test-${(Math.random() * 10000) | 0}.db`; + const path2 = `test-${(Math.random() * 10000) | 0}.db`; + const db1 = await connect({ path: path1, url: process.env.VITE_TURSO_DB_URL }); + await db1.exec(`INSERT INTO q VALUES ('k1', 'v1')`); + await db1.exec(`INSERT INTO q VALUES ('k2', 'v2')`); + const stats1 = await db1.stats(); + + const db2 = await connect({ path: path2, url: process.env.VITE_TURSO_DB_URL }); + await db2.exec(`INSERT INTO q VALUES ('k3', 'v3')`); + await db2.exec(`INSERT INTO q VALUES ('k4', 'v4')`); + + await Promise.all([db1.push(), db2.push()]); + await Promise.all([db1.pull(), db2.pull()]); + const stats2 = await db1.stats(); + console.info(stats1, stats2); + expect(stats1.revision).not.toBe(stats2.revision); + + const rows1 = await db1.prepare('SELECT * FROM q').all(); + const rows2 = await db2.prepare('SELECT * FROM q').all(); + const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }, { x: 'k3', y: 'v3' }, { x: 'k4', y: 'v4' }]; + expect(rows1.sort(localeCompare)).toEqual(expected.sort(localeCompare)) + expect(rows2.sort(localeCompare)).toEqual(expected.sort(localeCompare)) +}) + test('transform', async () => { { const db = await connect({ diff --git a/bindings/javascript/sync/packages/browser/promise.ts b/bindings/javascript/sync/packages/browser/promise.ts index f6198a3fd..3f43b81b6 100644 --- a/bindings/javascript/sync/packages/browser/promise.ts +++ b/bindings/javascript/sync/packages/browser/promise.ts @@ -1,6 +1,6 @@ import { registerFileAtWorker, unregisterFileAtWorker } from "@tursodatabase/database-browser-common" import { DatabasePromise, DatabaseOpts, NativeDatabase } from "@tursodatabase/database-common" -import { ProtocolIo, run, SyncOpts, RunOpts, memoryIO } from "@tursodatabase/sync-common"; +import { ProtocolIo, run, SyncOpts, RunOpts, memoryIO, SyncEngineStats } from "@tursodatabase/sync-common"; let BrowserIo: ProtocolIo = { async read(path: string): Promise { @@ -44,7 +44,7 @@ class Database extends DatabasePromise { async checkpoint() { await run(this.runOpts, this.io, this.engine, this.engine.checkpoint()); } - async stats(): Promise<{ operations: number, mainWal: number, revertWal: number, lastPullUnixTime: number, lastPushUnixTime: number | null }> { + async stats(): Promise { return (await run(this.runOpts, this.io, this.engine, this.engine.stats())); } override async close(): Promise { @@ -54,7 +54,7 @@ class Database extends DatabasePromise { await Promise.all([ unregisterFileAtWorker(this.worker, this.fsPath), unregisterFileAtWorker(this.worker, `${this.fsPath}-wal`), - unregisterFileAtWorker(this.worker, `${this.fsPath}-revert`), + unregisterFileAtWorker(this.worker, `${this.fsPath}-wal-revert`), unregisterFileAtWorker(this.worker, `${this.fsPath}-info`), unregisterFileAtWorker(this.worker, `${this.fsPath}-changes`), ]); @@ -95,7 +95,7 @@ async function connect(opts: SyncOpts, connect: (any) => any, init: () => Promis await Promise.all([ registerFileAtWorker(worker, opts.path), registerFileAtWorker(worker, `${opts.path}-wal`), - registerFileAtWorker(worker, `${opts.path}-revert`), + registerFileAtWorker(worker, `${opts.path}-wal-revert`), registerFileAtWorker(worker, `${opts.path}-info`), registerFileAtWorker(worker, `${opts.path}-changes`), ]); diff --git a/bindings/javascript/sync/packages/common/index.ts b/bindings/javascript/sync/packages/common/index.ts index 1b264c80b..822a8c24f 100644 --- a/bindings/javascript/sync/packages/common/index.ts +++ b/bindings/javascript/sync/packages/common/index.ts @@ -1,5 +1,5 @@ import { run, memoryIO } from "./run.js" -import { SyncOpts, ProtocolIo, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult } from "./types.js" +import { SyncOpts, ProtocolIo, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult, SyncEngineStats } from "./types.js" export { run, memoryIO, } -export type { SyncOpts, ProtocolIo, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult } \ No newline at end of file +export type { SyncOpts, ProtocolIo, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult, SyncEngineStats } \ No newline at end of file diff --git a/bindings/javascript/sync/packages/common/types.ts b/bindings/javascript/sync/packages/common/types.ts index 25fa1e47e..49b140103 100644 --- a/bindings/javascript/sync/packages/common/types.ts +++ b/bindings/javascript/sync/packages/common/types.ts @@ -44,7 +44,13 @@ export interface DatabaseRowStatement { values: Array } -export type GeneratorResponse = - | { type: 'IO' } - | { type: 'Done' } - | { type: 'SyncEngineStats', operations: number, mainWal: number, revertWal: number, lastPullUnixTime: number, lastPushUnixTime: number | null } \ No newline at end of file +export interface SyncEngineStats { + operations: number; + mainWal: number; + revertWal: number; + lastPullUnixTime: number; + lastPushUnixTime: number | null; + revision: string | null; +} + +export type GeneratorResponse = { type: 'IO' } | { type: 'Done' } | ({ type: 'SyncEngineStats' } & SyncEngineStats) \ No newline at end of file diff --git a/bindings/javascript/sync/packages/native/index.js b/bindings/javascript/sync/packages/native/index.js index 1576bb640..709ca74e4 100644 --- a/bindings/javascript/sync/packages/native/index.js +++ b/bindings/javascript/sync/packages/native/index.js @@ -81,8 +81,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-android-arm64') const bindingPackageVersion = require('@tursodatabase/sync-android-arm64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -97,8 +97,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-android-arm-eabi') const bindingPackageVersion = require('@tursodatabase/sync-android-arm-eabi/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -117,8 +117,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-win32-x64-msvc') const bindingPackageVersion = require('@tursodatabase/sync-win32-x64-msvc/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -133,8 +133,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-win32-ia32-msvc') const bindingPackageVersion = require('@tursodatabase/sync-win32-ia32-msvc/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -149,8 +149,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-win32-arm64-msvc') const bindingPackageVersion = require('@tursodatabase/sync-win32-arm64-msvc/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -168,8 +168,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-darwin-universal') const bindingPackageVersion = require('@tursodatabase/sync-darwin-universal/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -184,8 +184,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-darwin-x64') const bindingPackageVersion = require('@tursodatabase/sync-darwin-x64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -200,8 +200,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-darwin-arm64') const bindingPackageVersion = require('@tursodatabase/sync-darwin-arm64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -220,8 +220,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-freebsd-x64') const bindingPackageVersion = require('@tursodatabase/sync-freebsd-x64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -236,8 +236,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-freebsd-arm64') const bindingPackageVersion = require('@tursodatabase/sync-freebsd-arm64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -257,8 +257,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-x64-musl') const bindingPackageVersion = require('@tursodatabase/sync-linux-x64-musl/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -273,8 +273,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-x64-gnu') const bindingPackageVersion = require('@tursodatabase/sync-linux-x64-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -291,8 +291,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-arm64-musl') const bindingPackageVersion = require('@tursodatabase/sync-linux-arm64-musl/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -307,8 +307,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-arm64-gnu') const bindingPackageVersion = require('@tursodatabase/sync-linux-arm64-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -325,8 +325,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-arm-musleabihf') const bindingPackageVersion = require('@tursodatabase/sync-linux-arm-musleabihf/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -341,8 +341,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-arm-gnueabihf') const bindingPackageVersion = require('@tursodatabase/sync-linux-arm-gnueabihf/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -359,8 +359,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-riscv64-musl') const bindingPackageVersion = require('@tursodatabase/sync-linux-riscv64-musl/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -375,8 +375,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-riscv64-gnu') const bindingPackageVersion = require('@tursodatabase/sync-linux-riscv64-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -392,8 +392,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-ppc64-gnu') const bindingPackageVersion = require('@tursodatabase/sync-linux-ppc64-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -408,8 +408,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-linux-s390x-gnu') const bindingPackageVersion = require('@tursodatabase/sync-linux-s390x-gnu/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -428,8 +428,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-openharmony-arm64') const bindingPackageVersion = require('@tursodatabase/sync-openharmony-arm64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -444,8 +444,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-openharmony-x64') const bindingPackageVersion = require('@tursodatabase/sync-openharmony-x64/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { @@ -460,8 +460,8 @@ function requireNative() { try { const binding = require('@tursodatabase/sync-openharmony-arm') const bindingPackageVersion = require('@tursodatabase/sync-openharmony-arm/package.json').version - if (bindingPackageVersion !== '0.2.0-pre.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { - throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) + if (bindingPackageVersion !== '0.2.0-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + throw new Error(`Native binding package version mismatch, expected 0.2.0-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding } catch (e) { diff --git a/bindings/javascript/sync/packages/native/promise.test.ts b/bindings/javascript/sync/packages/native/promise.test.ts index ec8381190..cae58db11 100644 --- a/bindings/javascript/sync/packages/native/promise.test.ts +++ b/bindings/javascript/sync/packages/native/promise.test.ts @@ -4,6 +4,14 @@ import { connect, DatabaseRowMutation, DatabaseRowTransformResult } from './prom const localeCompare = (a, b) => a.x.localeCompare(b.x); +function cleanup(path) { + unlinkSync(path); + unlinkSync(`${path}-wal`); + unlinkSync(`${path}-info`); + unlinkSync(`${path}-changes`); + try { unlinkSync(`${path}-wal-revert`) } catch (e) { } +} + test('select-after-push', async () => { { const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL }); @@ -161,7 +169,8 @@ test('checkpoint', async () => { expect((await db1.stats()).revertWal).toBe(revertWal); }) -test('persistence', async () => { + +test('persistence-push', async () => { { const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL }); await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)"); @@ -182,9 +191,11 @@ test('persistence', async () => { const db2 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL }); await db2.exec(`INSERT INTO q VALUES ('k3', 'v3')`); await db2.exec(`INSERT INTO q VALUES ('k4', 'v4')`); - const rows = await db2.prepare('SELECT * FROM q').all(); + const stmt = db2.prepare('SELECT * FROM q'); + const rows = await stmt.all(); const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }, { x: 'k3', y: 'v3' }, { x: 'k4', y: 'v4' }]; expect(rows).toEqual(expected) + stmt.close(); await db2.close(); } @@ -201,12 +212,75 @@ test('persistence', async () => { expect(rows).toEqual(expected) await db4.close(); } + } + finally { + cleanup(path); + } +}) + +test('persistence-offline', async () => { + { + const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL }); + await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)"); + await db.exec("DELETE FROM q"); + await db.push(); + await db.close(); + } + const path = `test-${(Math.random() * 10000) | 0}.db`; + try { + { + const db = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL }); + await db.exec(`INSERT INTO q VALUES ('k1', 'v1')`); + await db.exec(`INSERT INTO q VALUES ('k2', 'v2')`); + await db.push(); + await db.close(); + } + { + const db = await connect({ path: path, url: "https://not-valid-url.localhost" }); + const rows = await db.prepare("SELECT * FROM q").all(); + const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }]; + expect(rows.sort(localeCompare)).toEqual(expected.sort(localeCompare)) + await db.close(); + } } finally { - unlinkSync(path); - unlinkSync(`${path}-wal`); - unlinkSync(`${path}-info`); - unlinkSync(`${path}-changes`); - try { unlinkSync(`${path}-revert`) } catch (e) { } + cleanup(path); + } +}) + +test('persistence-pull-push', async () => { + { + const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL }); + await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)"); + await db.exec("DELETE FROM q"); + await db.push(); + await db.close(); + } + const path1 = `test-${(Math.random() * 10000) | 0}.db`; + const path2 = `test-${(Math.random() * 10000) | 0}.db`; + try { + const db1 = await connect({ path: path1, url: process.env.VITE_TURSO_DB_URL }); + await db1.exec(`INSERT INTO q VALUES ('k1', 'v1')`); + await db1.exec(`INSERT INTO q VALUES ('k2', 'v2')`); + const stats1 = await db1.stats(); + + const db2 = await connect({ path: path2, url: process.env.VITE_TURSO_DB_URL }); + await db2.exec(`INSERT INTO q VALUES ('k3', 'v3')`); + await db2.exec(`INSERT INTO q VALUES ('k4', 'v4')`); + + await Promise.all([db1.push(), db2.push()]); + await Promise.all([db1.pull(), db2.pull()]); + const stats2 = await db1.stats(); + console.info(stats1, stats2); + expect(stats1.revision).not.toBe(stats2.revision); + + const rows1 = await db1.prepare('SELECT * FROM q').all(); + const rows2 = await db2.prepare('SELECT * FROM q').all(); + const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }, { x: 'k3', y: 'v3' }, { x: 'k4', y: 'v4' }]; + expect(rows1.sort(localeCompare)).toEqual(expected.sort(localeCompare)) + expect(rows2.sort(localeCompare)).toEqual(expected.sort(localeCompare)) + } finally { + cleanup(path1); + cleanup(path2); } }) diff --git a/bindings/javascript/sync/packages/native/promise.ts b/bindings/javascript/sync/packages/native/promise.ts index 86f020109..3d473c8a9 100644 --- a/bindings/javascript/sync/packages/native/promise.ts +++ b/bindings/javascript/sync/packages/native/promise.ts @@ -1,5 +1,5 @@ import { DatabasePromise, DatabaseOpts, NativeDatabase } from "@tursodatabase/database-common" -import { ProtocolIo, run, SyncOpts, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult } from "@tursodatabase/sync-common"; +import { ProtocolIo, run, SyncOpts, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult, SyncEngineStats } from "@tursodatabase/sync-common"; import { Database as NativeDB, SyncEngine } from "#index"; import { promises } from "node:fs"; @@ -61,7 +61,7 @@ class Database extends DatabasePromise { async checkpoint() { await run(this.runOpts, this.io, this.engine, this.engine.checkpoint()); } - async stats(): Promise<{ operations: number, mainWal: number, revertWal: number, lastPullUnixTime: number, lastPushUnixTime: number | null }> { + async stats(): Promise { return (await run(this.runOpts, this.io, this.engine, this.engine.stats())); } override async close(): Promise { diff --git a/bindings/javascript/sync/src/generator.rs b/bindings/javascript/sync/src/generator.rs index 141dec016..2aae4f373 100644 --- a/bindings/javascript/sync/src/generator.rs +++ b/bindings/javascript/sync/src/generator.rs @@ -45,6 +45,7 @@ pub enum GeneratorResponse { revert_wal: i64, last_pull_unix_time: i64, last_push_unix_time: Option, + revision: Option, }, } diff --git a/bindings/javascript/sync/src/lib.rs b/bindings/javascript/sync/src/lib.rs index fd4d88e78..e92603508 100644 --- a/bindings/javascript/sync/src/lib.rs +++ b/bindings/javascript/sync/src/lib.rs @@ -269,13 +269,14 @@ impl SyncEngine { self.run(async move |coro, sync_engine| { let sync_engine = try_read(sync_engine)?; let sync_engine = try_unwrap(&sync_engine)?; - let changes = sync_engine.stats(coro).await?; + let stats = sync_engine.stats(coro).await?; Ok(Some(GeneratorResponse::SyncEngineStats { - operations: changes.cdc_operations, - main_wal: changes.main_wal_size as i64, - revert_wal: changes.revert_wal_size as i64, - last_pull_unix_time: changes.last_pull_unix_time, - last_push_unix_time: changes.last_push_unix_time, + operations: stats.cdc_operations, + main_wal: stats.main_wal_size as i64, + revert_wal: stats.revert_wal_size as i64, + last_pull_unix_time: stats.last_pull_unix_time, + last_push_unix_time: stats.last_push_unix_time, + revision: stats.revision, })) }) } diff --git a/bindings/rust/Cargo.toml b/bindings/rust/Cargo.toml index 63be50f42..d799b5320 100644 --- a/bindings/rust/Cargo.toml +++ b/bindings/rust/Cargo.toml @@ -18,10 +18,10 @@ tracing_release = ["turso_core/tracing_release"] [dependencies] turso_core = { workspace = true, features = ["io_uring"] } -thiserror = "2.0.9" +thiserror = { workspace = true } [dev-dependencies] -tempfile = "3.20.0" -tokio = { version = "1.29.1", features = ["full"] } -rand = "0.8.5" -rand_chacha = "0.3.1" +tempfile = { workspace = true } +tokio = { workspace = true, features = ["full"] } +rand = { workspace = true } +rand_chacha = { workspace = true } diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index 923542cdb..39706fdd5 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -393,6 +393,29 @@ impl Connection { Ok(conn.get_auto_commit()) } + + /// Sets maximum total accumuated timeout. If the duration is None or Zero, we unset the busy handler for this Connection + /// + /// This api defers slighty from: https://www.sqlite.org/c3ref/busy_timeout.html + /// + /// Instead of sleeping for linear amount of time specified by the user, + /// we will sleep in phases, until the the total amount of time is reached. + /// This means we first sleep of 1ms, then if we still return busy, we sleep for 2 ms, and repeat until a maximum of 100 ms per phase. + /// + /// Example: + /// 1. Set duration to 5ms + /// 2. Step through query -> returns Busy -> sleep/yield for 1 ms + /// 3. Step through query -> returns Busy -> sleep/yield for 2 ms + /// 4. Step through query -> returns Busy -> sleep/yield for 2 ms (totaling 5 ms of sleep) + /// 5. Step through query -> returns Busy -> return Busy to user + pub fn busy_timeout(&self, duration: Option) -> Result<()> { + let conn = self + .inner + .lock() + .map_err(|e| Error::MutexError(e.to_string()))?; + conn.busy_timeout(duration); + Ok(()) + } } impl Debug for Connection { diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 92f384c6f..d7e9c4c92 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -19,27 +19,27 @@ path = "main.rs" [dependencies] anyhow.workspace = true -cfg-if = "1.0.0" -clap = { version = "4.5.31", features = ["derive"] } +cfg-if = { workspace = true } +clap = { workspace = true, features = ["derive"] } clap_complete = { version = "=4.5.47", features = ["unstable-dynamic"] } comfy-table = "7.1.4" csv = "1.3.1" ctrlc = "3.4.4" dirs = "5.0.1" -env_logger = "0.10.1" +env_logger = { workspace = true } libc = "0.2.172" turso_core = { path = "../core", default-features = true, features = [] } limbo_completion = { path = "../extensions/completion", features = ["static"] } -miette = { version = "7.4.0", features = ["fancy"] } +miette = { workspace = true, features = ["fancy"] } nu-ansi-term = {version = "0.50.1", features = ["serde", "derive_serde_style"]} rustyline = { version = "15.0.0", default-features = true, features = [ "derive", ] } shlex = "1.3.0" syntect = { git = "https://github.com/trishume/syntect.git", rev = "64644ffe064457265cbcee12a0c1baf9485ba6ee" } -tracing = "0.1.41" -tracing-appender = "0.2.3" -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } toml = {version = "0.8.20", features = ["preserve_order"]} schemars = {version = "0.8.22", features = ["preserve_order"]} serde = { workspace = true, features = ["derive"]} diff --git a/cli/app.rs b/cli/app.rs index fdc7ae913..30b5d6ec6 100644 --- a/cli/app.rs +++ b/cli/app.rs @@ -1102,7 +1102,7 @@ impl Limbo { table_name: &str, ) -> anyhow::Result { let sql = format!( - "SELECT sql, type, name FROM {db_prefix}.sqlite_schema WHERE type IN ('table', 'index', 'view') AND (tbl_name = '{table_name}' OR name = '{table_name}') AND name NOT LIKE 'sqlite_%' ORDER BY CASE type WHEN 'table' THEN 1 WHEN 'view' THEN 2 WHEN 'index' THEN 3 END, rowid" + "SELECT sql, type, name FROM {db_prefix}.sqlite_schema WHERE type IN ('table', 'index', 'view') AND (tbl_name = '{table_name}' OR name = '{table_name}') AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__turso_internal_%' ORDER BY CASE type WHEN 'table' THEN 1 WHEN 'view' THEN 2 WHEN 'index' THEN 3 END, rowid" ); let mut found = false; @@ -1135,7 +1135,7 @@ impl Limbo { db_prefix: &str, db_display_name: &str, ) -> anyhow::Result<()> { - let sql = format!("SELECT sql, type, name FROM {db_prefix}.sqlite_schema WHERE type IN ('table', 'index', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY CASE type WHEN 'table' THEN 1 WHEN 'view' THEN 2 WHEN 'index' THEN 3 END, rowid"); + let sql = format!("SELECT sql, type, name FROM {db_prefix}.sqlite_schema WHERE type IN ('table', 'index', 'view') AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__turso_internal_%' ORDER BY CASE type WHEN 'table' THEN 1 WHEN 'view' THEN 2 WHEN 'index' THEN 3 END, rowid"); match self.conn.query(&sql) { Ok(Some(ref mut rows)) => loop { diff --git a/core/Cargo.toml b/core/Cargo.toml index e28c64280..b3898dc1b 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -44,36 +44,35 @@ libc = { version = "0.2.172" } libloading = "0.8.6" [dependencies] -antithesis_sdk = { version = "0.2.5", optional = true } +antithesis_sdk = { workspace = true, optional = true } turso_ext = { workspace = true, features = ["core_only"] } cfg_block = "0.1.1" -fallible-iterator = "0.3.0" -hex = "0.4.3" -turso_sqlite3_parser = { workspace = true } -thiserror = "1.0.61" +fallible-iterator = { workspace = true } +hex = { workspace = true } +thiserror = { workspace = true } getrandom = { version = "0.2.15" } -regex = "1.11.1" -regex-syntax = { version = "0.8.5", default-features = false, features = [ +regex = { workspace = true } +regex-syntax = { workspace = true, default-features = false, features = [ "unicode", ] } -chrono = { version = "0.4.38", default-features = false, features = ["clock"] } +chrono = { workspace = true, default-features = false, features = ["clock"] } julian_day_converter = "0.4.5" rand = "0.8.5" libm = "0.2" turso_macros = { workspace = true } -miette = "7.6.0" +miette = { workspace = true } strum = { workspace = true } parking_lot = { workspace = true } crossbeam-skiplist = "0.1.3" -tracing = "0.1.41" +tracing = { workspace = true } ryu = "1.0.19" uncased = "0.9.10" strum_macros = { workspace = true } -bitflags = "2.9.0" +bitflags = { workspace = true } serde = { workspace = true, optional = true, features = ["derive"] } paste = "1.0.15" uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true } -tempfile = "3.8.0" +tempfile = { workspace = true } pack1 = { version = "1.0.0", features = ["bytemuck"] } bytemuck = "1.23.1" aes-gcm = { version = "0.10.3"} @@ -83,7 +82,7 @@ aegis = "0.9.0" twox-hash = "2.1.1" [build-dependencies] -chrono = { version = "0.4.38", default-features = false } +chrono = { workspace = true, default-features = false } built = { version = "0.7.5", features = ["git2", "chrono"] } [target.'cfg(not(target_family = "windows"))'.dev-dependencies] @@ -91,7 +90,7 @@ pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] } [dev-dependencies] memory-stats = "1.2.0" -criterion = { version = "0.5", features = [ +criterion = { workspace = true, features = [ "html_reports", "async", "async_futures", @@ -101,11 +100,11 @@ rusqlite.workspace = true quickcheck = { version = "1.0", default-features = false } quickcheck_macros = { version = "1.0", default-features = false } rand = "0.8.5" # Required for quickcheck -rand_chacha = "0.9.0" -env_logger = "0.11.6" +rand_chacha = { workspace = true } +env_logger = { workspace = true } test-log = { version = "0.2.17", features = ["trace"] } sorted-vec = "0.8.6" -mimalloc = { version = "0.1.46", default-features = false } +mimalloc = { workspace = true, default-features = false } [[bench]] name = "benchmark" diff --git a/core/benches/mvcc_benchmark.rs b/core/benches/mvcc_benchmark.rs index 69754f416..de8d4bdff 100644 --- a/core/benches/mvcc_benchmark.rs +++ b/core/benches/mvcc_benchmark.rs @@ -35,8 +35,10 @@ fn bench(c: &mut Criterion) { let db = bench_db(); b.to_async(FuturesExecutor).iter(|| async { let conn = db.conn.clone(); - let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()); - db.mvcc_store.rollback_tx(tx_id, conn.get_pager().clone()) + let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()).unwrap(); + db.mvcc_store + .rollback_tx(tx_id, conn.get_pager().clone(), &conn) + .unwrap(); }) }); @@ -44,7 +46,7 @@ fn bench(c: &mut Criterion) { group.bench_function("begin_tx + commit_tx", |b| { b.to_async(FuturesExecutor).iter(|| async { let conn = &db.conn; - let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()); + let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()).unwrap(); let mv_store = &db.mvcc_store; let mut sm = mv_store .commit_tx(tx_id, conn.get_pager().clone(), conn) @@ -65,7 +67,7 @@ fn bench(c: &mut Criterion) { group.bench_function("begin_tx-read-commit_tx", |b| { b.to_async(FuturesExecutor).iter(|| async { let conn = &db.conn; - let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()); + let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()).unwrap(); db.mvcc_store .read( tx_id, @@ -97,7 +99,7 @@ fn bench(c: &mut Criterion) { group.bench_function("begin_tx-update-commit_tx", |b| { b.to_async(FuturesExecutor).iter(|| async { let conn = &db.conn; - let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()); + let tx_id = db.mvcc_store.begin_tx(conn.get_pager().clone()).unwrap(); db.mvcc_store .update( tx_id, @@ -109,7 +111,6 @@ fn bench(c: &mut Criterion) { data: record_data.clone(), column_count: 1, }, - conn.get_pager().clone(), ) .unwrap(); let mv_store = &db.mvcc_store; @@ -129,7 +130,7 @@ fn bench(c: &mut Criterion) { }); let db = bench_db(); - let tx_id = db.mvcc_store.begin_tx(db.conn.get_pager().clone()); + let tx_id = db.mvcc_store.begin_tx(db.conn.get_pager().clone()).unwrap(); db.mvcc_store .insert( tx_id, @@ -158,8 +159,7 @@ fn bench(c: &mut Criterion) { }); let db = bench_db(); - let tx_id = db.mvcc_store.begin_tx(db.conn.get_pager().clone()); - let conn = &db.conn; + let tx_id = db.mvcc_store.begin_tx(db.conn.get_pager().clone()).unwrap(); db.mvcc_store .insert( tx_id, @@ -186,7 +186,6 @@ fn bench(c: &mut Criterion) { data: record_data.clone(), column_count: 1, }, - conn.get_pager().clone(), ) .unwrap(); }) diff --git a/core/incremental/compiler.rs b/core/incremental/compiler.rs index 9dc85ad79..80163f5e9 100644 --- a/core/incremental/compiler.rs +++ b/core/incremental/compiler.rs @@ -8,15 +8,15 @@ use crate::incremental::dbsp::{Delta, DeltaPair}; use crate::incremental::expr_compiler::CompiledExpression; use crate::incremental::operator::{ - EvalState, FilterOperator, FilterPredicate, IncrementalOperator, InputOperator, ProjectOperator, + create_dbsp_state_index, DbspStateCursors, EvalState, FilterOperator, FilterPredicate, + IncrementalOperator, InputOperator, ProjectOperator, }; -use crate::incremental::persistence::WriteRow; -use crate::storage::btree::BTreeCursor; +use crate::storage::btree::{BTreeCursor, BTreeKey}; // Note: logical module must be made pub(crate) in translate/mod.rs use crate::translate::logical::{ BinaryOperator, LogicalExpr, LogicalPlan, LogicalSchema, SchemaRef, }; -use crate::types::{IOResult, SeekKey, Value}; +use crate::types::{IOResult, ImmutableRecord, SeekKey, SeekOp, SeekResult, Value}; use crate::Pager; use crate::{return_and_restore_if_io, return_if_io, LimboError, Result}; use std::collections::HashMap; @@ -24,8 +24,120 @@ use std::fmt::{self, Display, Formatter}; use std::rc::Rc; use std::sync::Arc; -// The state table is always a key-value store with 3 columns: key, state, and weight. -const OPERATOR_COLUMNS: usize = 3; +// The state table has 5 columns: operator_id, zset_id, element_id, value, weight +const OPERATOR_COLUMNS: usize = 5; + +/// State machine for writing rows to simple materialized views (table-only, no index) +#[derive(Debug, Default)] +pub enum WriteRowView { + #[default] + GetRecord, + Delete, + Insert { + final_weight: isize, + }, + Done, +} + +impl WriteRowView { + pub fn new() -> Self { + Self::default() + } + + /// Write a row with weight management for table-only storage. + /// + /// # Arguments + /// * `cursor` - BTree cursor for the storage + /// * `key` - The key to seek (TableRowId) + /// * `build_record` - Function that builds the record values to insert. + /// Takes the final_weight and returns the complete record values. + /// * `weight` - The weight delta to apply + pub fn write_row( + &mut self, + cursor: &mut BTreeCursor, + key: SeekKey, + build_record: impl Fn(isize) -> Vec, + weight: isize, + ) -> Result> { + loop { + match self { + WriteRowView::GetRecord => { + let res = return_if_io!(cursor.seek(key.clone(), SeekOp::GE { eq_only: true })); + if !matches!(res, SeekResult::Found) { + *self = WriteRowView::Insert { + final_weight: weight, + }; + } else { + let existing_record = return_if_io!(cursor.record()); + let r = existing_record.ok_or_else(|| { + LimboError::InternalError(format!( + "Found key {key:?} in storage but could not read record" + )) + })?; + let values = r.get_values(); + + // Weight is always the last value + let existing_weight = match values.last() { + Some(val) => match val.to_owned() { + Value::Integer(w) => w as isize, + _ => { + return Err(LimboError::InternalError(format!( + "Invalid weight value in storage for key {key:?}" + ))) + } + }, + None => { + return Err(LimboError::InternalError(format!( + "No weight value found in storage for key {key:?}" + ))) + } + }; + + let final_weight = existing_weight + weight; + if final_weight <= 0 { + *self = WriteRowView::Delete + } else { + *self = WriteRowView::Insert { final_weight } + } + } + } + WriteRowView::Delete => { + // Mark as Done before delete to avoid retry on I/O + *self = WriteRowView::Done; + return_if_io!(cursor.delete()); + } + WriteRowView::Insert { final_weight } => { + return_if_io!(cursor.seek(key.clone(), SeekOp::GE { eq_only: true })); + + // Extract the row ID from the key + let key_i64 = match key { + SeekKey::TableRowId(id) => id, + _ => { + return Err(LimboError::InternalError( + "Expected TableRowId for storage".to_string(), + )) + } + }; + + // Build the record values using the provided function + let record_values = build_record(*final_weight); + + // Create an ImmutableRecord from the values + let immutable_record = + ImmutableRecord::from_values(&record_values, record_values.len()); + let btree_key = BTreeKey::new_table_rowid(key_i64, Some(&immutable_record)); + + // Mark as Done before insert to avoid retry on I/O + *self = WriteRowView::Done; + return_if_io!(cursor.insert(&btree_key)); + } + WriteRowView::Done => { + return Ok(IOResult::Done(())); + } + } + } + } +} /// State machine for commit operations pub enum CommitState { @@ -36,8 +148,8 @@ pub enum CommitState { CommitOperators { /// Execute state for running the circuit execute_state: Box, - /// Persistent cursor for operator state btree (internal_state_root) - state_cursor: Box, + /// Persistent cursors for operator state (table and index) + state_cursors: Box, }, /// Updating the materialized view with the delta @@ -47,7 +159,7 @@ pub enum CommitState { /// Current index in delta.changes being processed current_index: usize, /// State for writing individual rows - write_row_state: WriteRow, + write_row_state: WriteRowView, /// Cursor for view data btree - created fresh for each row view_cursor: Box, }, @@ -60,7 +172,8 @@ impl std::fmt::Debug for CommitState { Self::CommitOperators { execute_state, .. } => f .debug_struct("CommitOperators") .field("execute_state", execute_state) - .field("has_state_cursor", &true) + .field("has_state_table_cursor", &true) + .field("has_state_index_cursor", &true) .finish(), Self::UpdateView { delta, @@ -221,25 +334,13 @@ impl std::fmt::Debug for DbspNode { impl DbspNode { fn process_node( &mut self, - pager: Rc, eval_state: &mut EvalState, - root_page: usize, commit_operators: bool, - state_cursor: Option<&mut Box>, + cursors: &mut DbspStateCursors, ) -> Result> { // Process delta using the executable operator let op = &mut self.executable; - // Use provided cursor or create a local one - let mut local_cursor; - let cursor = if let Some(cursor) = state_cursor { - cursor.as_mut() - } else { - // Create a local cursor if none was provided - local_cursor = BTreeCursor::new_table(None, pager.clone(), root_page, OPERATOR_COLUMNS); - &mut local_cursor - }; - let state = if commit_operators { // Clone the deltas from eval_state - don't extract them // in case we need to re-execute due to I/O @@ -247,12 +348,12 @@ impl DbspNode { EvalState::Init { deltas } => deltas.clone(), _ => panic!("commit can only be called when eval_state is in Init state"), }; - let result = return_if_io!(op.commit(deltas, cursor)); + let result = return_if_io!(op.commit(deltas, cursors)); // After successful commit, move state to Done *eval_state = EvalState::Done; result } else { - return_if_io!(op.eval(eval_state, cursor)) + return_if_io!(op.eval(eval_state, cursors)) }; Ok(IOResult::Done(state)) } @@ -275,14 +376,20 @@ pub struct DbspCircuit { /// Root page for the main materialized view data pub(super) main_data_root: usize, - /// Root page for internal DBSP state + /// Root page for internal DBSP state table pub(super) internal_state_root: usize, + /// Root page for the DBSP state table's primary key index + pub(super) internal_state_index_root: usize, } impl DbspCircuit { /// Create a new empty circuit with initial empty schema /// The actual output schema will be set when the root node is established - pub fn new(main_data_root: usize, internal_state_root: usize) -> Self { + pub fn new( + main_data_root: usize, + internal_state_root: usize, + internal_state_index_root: usize, + ) -> Self { // Start with an empty schema - will be updated when root is set let empty_schema = Arc::new(LogicalSchema::new(vec![])); Self { @@ -293,6 +400,7 @@ impl DbspCircuit { commit_state: CommitState::Init, main_data_root, internal_state_root, + internal_state_index_root, } } @@ -326,18 +434,18 @@ impl DbspCircuit { pub fn run_circuit( &mut self, - pager: Rc, execute_state: &mut ExecuteState, + pager: &Rc, + state_cursors: &mut DbspStateCursors, commit_operators: bool, - state_cursor: &mut Box, ) -> Result> { if let Some(root_id) = self.root { self.execute_node( root_id, - pager, + pager.clone(), execute_state, commit_operators, - Some(state_cursor), + state_cursors, ) } else { Err(LimboError::ParseError( @@ -358,7 +466,23 @@ impl DbspCircuit { execute_state: &mut ExecuteState, ) -> Result> { if let Some(root_id) = self.root { - self.execute_node(root_id, pager, execute_state, false, None) + // Create temporary cursors for execute (non-commit) operations + let table_cursor = BTreeCursor::new_table( + None, + pager.clone(), + self.internal_state_root, + OPERATOR_COLUMNS, + ); + let index_def = create_dbsp_state_index(self.internal_state_index_root); + let index_cursor = BTreeCursor::new_index( + None, + pager.clone(), + self.internal_state_index_root, + &index_def, + 3, + ); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + self.execute_node(root_id, pager, execute_state, false, &mut cursors) } else { Err(LimboError::ParseError( "Circuit has no root node".to_string(), @@ -398,29 +522,42 @@ impl DbspCircuit { let mut state = std::mem::replace(&mut self.commit_state, CommitState::Init); match &mut state { CommitState::Init => { - // Create state cursor when entering CommitOperators state - let state_cursor = Box::new(BTreeCursor::new_table( + // Create state cursors when entering CommitOperators state + let state_table_cursor = BTreeCursor::new_table( None, pager.clone(), self.internal_state_root, OPERATOR_COLUMNS, + ); + let index_def = create_dbsp_state_index(self.internal_state_index_root); + let state_index_cursor = BTreeCursor::new_index( + None, + pager.clone(), + self.internal_state_index_root, + &index_def, + 3, // Index on first 3 columns + ); + + let state_cursors = Box::new(DbspStateCursors::new( + state_table_cursor, + state_index_cursor, )); self.commit_state = CommitState::CommitOperators { execute_state: Box::new(ExecuteState::Init { input_data: input_delta_set.clone(), }), - state_cursor, + state_cursors, }; } CommitState::CommitOperators { ref mut execute_state, - ref mut state_cursor, + ref mut state_cursors, } => { let delta = return_and_restore_if_io!( &mut self.commit_state, state, - self.run_circuit(pager.clone(), execute_state, true, state_cursor) + self.run_circuit(execute_state, &pager, state_cursors, true,) ); // Create view cursor when entering UpdateView state @@ -434,7 +571,7 @@ impl DbspCircuit { self.commit_state = CommitState::UpdateView { delta, current_index: 0, - write_row_state: WriteRow::new(), + write_row_state: WriteRowView::new(), view_cursor, }; } @@ -453,7 +590,7 @@ impl DbspCircuit { // If we're starting a new row (GetRecord state), we need a fresh cursor // due to btree cursor state machine limitations - if matches!(write_row_state, WriteRow::GetRecord) { + if matches!(write_row_state, WriteRowView::GetRecord) { *view_cursor = Box::new(BTreeCursor::new_table( None, pager.clone(), @@ -493,7 +630,7 @@ impl DbspCircuit { self.commit_state = CommitState::UpdateView { delta, current_index: *current_index + 1, - write_row_state: WriteRow::new(), + write_row_state: WriteRowView::new(), view_cursor, }; } @@ -509,7 +646,7 @@ impl DbspCircuit { pager: Rc, execute_state: &mut ExecuteState, commit_operators: bool, - state_cursor: Option<&mut Box>, + cursors: &mut DbspStateCursors, ) -> Result> { loop { match execute_state { @@ -577,12 +714,30 @@ impl DbspCircuit { // Get the (node_id, state) pair for the current index let (input_node_id, input_state) = &mut input_states[*current_index]; + // Create temporary cursors for the recursive call + let temp_table_cursor = BTreeCursor::new_table( + None, + pager.clone(), + self.internal_state_root, + OPERATOR_COLUMNS, + ); + let index_def = create_dbsp_state_index(self.internal_state_index_root); + let temp_index_cursor = BTreeCursor::new_index( + None, + pager.clone(), + self.internal_state_index_root, + &index_def, + 3, + ); + let mut temp_cursors = + DbspStateCursors::new(temp_table_cursor, temp_index_cursor); + let delta = return_if_io!(self.execute_node( *input_node_id, pager.clone(), input_state, commit_operators, - None // Input nodes don't need state cursor + &mut temp_cursors )); input_deltas.push(delta); *current_index += 1; @@ -595,13 +750,8 @@ impl DbspCircuit { .get_mut(&node_id) .ok_or_else(|| LimboError::ParseError("Node not found".to_string()))?; - let output_delta = return_if_io!(node.process_node( - pager.clone(), - eval_state, - self.internal_state_root, - commit_operators, - state_cursor, - )); + let output_delta = + return_if_io!(node.process_node(eval_state, commit_operators, cursors)); return Ok(IOResult::Done(output_delta)); } } @@ -660,9 +810,17 @@ pub struct DbspCompiler { impl DbspCompiler { /// Create a new DBSP compiler - pub fn new(main_data_root: usize, internal_state_root: usize) -> Self { + pub fn new( + main_data_root: usize, + internal_state_root: usize, + internal_state_index_root: usize, + ) -> Self { Self { - circuit: DbspCircuit::new(main_data_root, internal_state_root), + circuit: DbspCircuit::new( + main_data_root, + internal_state_root, + internal_state_index_root, + ), } } @@ -781,9 +939,9 @@ impl DbspCompiler { use crate::function::AggFunc; use crate::incremental::operator::AggregateFunction; - let agg_fn = match fun { + match fun { AggFunc::Count | AggFunc::Count0 => { - AggregateFunction::Count + aggregate_functions.push(AggregateFunction::Count); } AggFunc::Sum => { if args.is_empty() { @@ -791,7 +949,7 @@ impl DbspCompiler { } // Extract column name from the argument if let LogicalExpr::Column(col) = &args[0] { - AggregateFunction::Sum(col.name.clone()) + aggregate_functions.push(AggregateFunction::Sum(col.name.clone())); } else { return Err(LimboError::ParseError( "Only column references are supported in aggregate functions for incremental views".to_string() @@ -803,36 +961,43 @@ impl DbspCompiler { return Err(LimboError::ParseError("AVG requires an argument".to_string())); } if let LogicalExpr::Column(col) = &args[0] { - AggregateFunction::Avg(col.name.clone()) + aggregate_functions.push(AggregateFunction::Avg(col.name.clone())); } else { return Err(LimboError::ParseError( "Only column references are supported in aggregate functions for incremental views".to_string() )); } } - // MIN and MAX are not supported in incremental views due to storage overhead. - // To correctly handle deletions, these operators would need to track all values - // in each group, resulting in O(n) storage overhead. This is prohibitive for - // large datasets. Alternative approaches like maintaining sorted indexes still - // require O(n) storage. Until a more efficient solution is found, MIN/MAX - // aggregations are not supported in materialized views. AggFunc::Min => { - return Err(LimboError::ParseError( - "MIN aggregation is not supported in incremental materialized views due to O(n) storage overhead required for handling deletions".to_string() - )); + if args.is_empty() { + return Err(LimboError::ParseError("MIN requires an argument".to_string())); + } + if let LogicalExpr::Column(col) = &args[0] { + aggregate_functions.push(AggregateFunction::Min(col.name.clone())); + } else { + return Err(LimboError::ParseError( + "Only column references are supported in MIN for incremental views".to_string() + )); + } } AggFunc::Max => { - return Err(LimboError::ParseError( - "MAX aggregation is not supported in incremental materialized views due to O(n) storage overhead required for handling deletions".to_string() - )); + if args.is_empty() { + return Err(LimboError::ParseError("MAX requires an argument".to_string())); + } + if let LogicalExpr::Column(col) = &args[0] { + aggregate_functions.push(AggregateFunction::Max(col.name.clone())); + } else { + return Err(LimboError::ParseError( + "Only column references are supported in MAX for incremental views".to_string() + )); + } } _ => { return Err(LimboError::ParseError( format!("Unsupported aggregate function in DBSP compiler: {fun:?}") )); } - }; - aggregate_functions.push(agg_fn); + } } else { return Err(LimboError::ParseError( "Expected aggregate function in aggregate expressions".to_string() @@ -840,19 +1005,17 @@ impl DbspCompiler { } } - // Create the AggregateOperator with a unique operator_id - // Use the next_node_id as the operator_id to ensure uniqueness let operator_id = self.circuit.next_id; + use crate::incremental::operator::AggregateOperator; let executable: Box = Box::new(AggregateOperator::new( - operator_id, // Use next_node_id as operator_id - group_by_columns, + operator_id, + group_by_columns.clone(), aggregate_functions.clone(), - input_column_names, + input_column_names.clone(), )); - // Create aggregate node - let node_id = self.circuit.add_node( + let result_node_id = self.circuit.add_node( DbspOperator::Aggregate { group_exprs: dbsp_group_exprs, aggr_exprs: aggregate_functions, @@ -861,7 +1024,8 @@ impl DbspCompiler { vec![input_id], executable, ); - Ok(node_id) + + Ok(result_node_id) } LogicalPlan::TableScan(scan) => { // Create input node with InputOperator for uniform handling @@ -1252,7 +1416,7 @@ mod tests { }}; } - fn setup_btree_for_circuit() -> (Rc, usize, usize) { + fn setup_btree_for_circuit() -> (Rc, usize, usize, usize) { let io: Arc = Arc::new(MemoryIO::new()); let db = Database::open_file(io.clone(), ":memory:", false, false).unwrap(); let conn = db.connect().unwrap(); @@ -1270,13 +1434,24 @@ mod tests { .block(|| pager.btree_create(&CreateBTreeFlags::new_table())) .unwrap() as usize; - (pager, main_root_page, dbsp_state_page) + let dbsp_state_index_page = pager + .io + .block(|| pager.btree_create(&CreateBTreeFlags::new_index())) + .unwrap() as usize; + + ( + pager, + main_root_page, + dbsp_state_page, + dbsp_state_index_page, + ) } // Macro to compile SQL to DBSP circuit macro_rules! compile_sql { ($sql:expr) => {{ - let (pager, main_root_page, dbsp_state_page) = setup_btree_for_circuit(); + let (pager, main_root_page, dbsp_state_page, dbsp_state_index_page) = + setup_btree_for_circuit(); let schema = test_schema!(); let mut parser = Parser::new($sql.as_bytes()); let cmd = parser @@ -1289,7 +1464,7 @@ mod tests { let mut builder = LogicalPlanBuilder::new(&schema); let logical_plan = builder.build_statement(&stmt).unwrap(); ( - DbspCompiler::new(main_root_page, dbsp_state_page) + DbspCompiler::new(main_root_page, dbsp_state_page, dbsp_state_index_page) .compile(&logical_plan) .unwrap(), pager, @@ -3162,10 +3337,10 @@ mod tests { #[test] fn test_circuit_rowid_update_consolidation() { - let (pager, p1, p2) = setup_btree_for_circuit(); + let (pager, p1, p2, p3) = setup_btree_for_circuit(); // Test that circuit properly consolidates state when rowid changes - let mut circuit = DbspCircuit::new(p1, p2); + let mut circuit = DbspCircuit::new(p1, p2, p3); // Create a simple filter node let schema = Arc::new(LogicalSchema::new(vec![ diff --git a/core/incremental/operator.rs b/core/incremental/operator.rs index 825290bef..8d0abe284 100644 --- a/core/incremental/operator.rs +++ b/core/incremental/operator.rs @@ -5,9 +5,10 @@ use crate::function::{AggFunc, Func}; use crate::incremental::dbsp::{Delta, DeltaPair, HashableRow}; use crate::incremental::expr_compiler::CompiledExpression; -use crate::incremental::persistence::{ReadRecord, WriteRow}; +use crate::incremental::persistence::{MinMaxPersistState, ReadRecord, RecomputeMinMax, WriteRow}; +use crate::schema::{Index, IndexColumn}; use crate::storage::btree::BTreeCursor; -use crate::types::{IOResult, SeekKey, Text}; +use crate::types::{IOResult, ImmutableRecord, SeekKey, SeekOp, SeekResult, Text}; use crate::{ return_and_restore_if_io, return_if_io, Connection, Database, Result, SymbolTable, Value, }; @@ -17,7 +18,84 @@ use std::sync::{Arc, Mutex}; use turso_macros::match_ignore_ascii_case; use turso_parser::ast::{As, Expr, Literal, Name, OneSelect, Operator, ResultColumn}; -type ComputedStates = HashMap, AggregateState)>; // group_key_str -> (group_key, state) +/// Struct to hold both table and index cursors for DBSP state operations +pub struct DbspStateCursors { + /// Cursor for the DBSP state table + pub table_cursor: BTreeCursor, + /// Cursor for the DBSP state table's primary key index + pub index_cursor: BTreeCursor, +} + +impl DbspStateCursors { + /// Create a new DbspStateCursors with both table and index cursors + pub fn new(table_cursor: BTreeCursor, index_cursor: BTreeCursor) -> Self { + Self { + table_cursor, + index_cursor, + } + } +} + +/// Create an index definition for the DBSP state table +/// This defines the primary key index on (operator_id, zset_id, element_id) +pub fn create_dbsp_state_index(root_page: usize) -> Index { + Index { + name: "dbsp_state_pk".to_string(), + table_name: "dbsp_state".to_string(), + root_page, + columns: vec![ + IndexColumn { + name: "operator_id".to_string(), + order: turso_parser::ast::SortOrder::Asc, + collation: None, + pos_in_table: 0, + default: None, + }, + IndexColumn { + name: "zset_id".to_string(), + order: turso_parser::ast::SortOrder::Asc, + collation: None, + pos_in_table: 1, + default: None, + }, + IndexColumn { + name: "element_id".to_string(), + order: turso_parser::ast::SortOrder::Asc, + collation: None, + pos_in_table: 2, + default: None, + }, + ], + unique: true, + ephemeral: false, + has_rowid: true, + } +} + +/// Constants for aggregate type encoding in storage IDs (2 bits) +pub const AGG_TYPE_REGULAR: u8 = 0b00; // COUNT/SUM/AVG +pub const AGG_TYPE_MINMAX: u8 = 0b01; // MIN/MAX (BTree ordering gives both) +pub const AGG_TYPE_RESERVED1: u8 = 0b10; // Reserved for future use +pub const AGG_TYPE_RESERVED2: u8 = 0b11; // Reserved for future use + +/// Generate a storage ID with column index and operation type encoding +/// Storage ID = (operator_id << 16) | (column_index << 2) | operation_type +/// Bit layout (64-bit integer): +/// - Bits 16-63 (48 bits): operator_id +/// - Bits 2-15 (14 bits): column_index (supports up to 16,384 columns) +/// - Bits 0-1 (2 bits): operation type (AGG_TYPE_REGULAR, AGG_TYPE_MINMAX, etc.) +pub fn generate_storage_id(operator_id: usize, column_index: usize, op_type: u8) -> i64 { + assert!(op_type <= 3, "Invalid operation type"); + assert!(column_index < 16384, "Column index too large"); + + ((operator_id as i64) << 16) | ((column_index as i64) << 2) | (op_type as i64) +} + +// group_key_str -> (group_key, state) +type ComputedStates = HashMap, AggregateState)>; +// group_key_str -> (column_name, value_as_hashable_row) -> accumulated_weight +pub type MinMaxDeltas = HashMap>; + #[derive(Debug)] enum AggregateCommitState { Idle, @@ -29,6 +107,11 @@ enum AggregateCommitState { computed_states: ComputedStates, current_idx: usize, write_row: WriteRow, + min_max_deltas: MinMaxDeltas, + }, + PersistMinMax { + delta: Delta, + min_max_persist_state: MinMaxPersistState, }, Done { delta: Delta, @@ -44,14 +127,28 @@ pub enum EvalState { Init { deltas: DeltaPair, }, + FetchKey { + delta: Delta, // Keep original delta for merge operation + current_idx: usize, + groups_to_read: Vec<(String, Vec)>, // Changed to Vec for index-based access + existing_groups: HashMap, + old_values: HashMap>, + }, FetchData { delta: Delta, // Keep original delta for merge operation current_idx: usize, groups_to_read: Vec<(String, Vec)>, // Changed to Vec for index-based access existing_groups: HashMap, old_values: HashMap>, + rowid: Option, // Rowid found by FetchKey (None if not found) read_record_state: Box, }, + RecomputeMinMax { + delta: Delta, + existing_groups: HashMap, + old_values: HashMap>, + recompute_state: Box, + }, Done, } @@ -70,7 +167,7 @@ impl From for EvalState { } impl EvalState { - fn from_delta(delta: Delta) -> Self { + pub fn from_delta(delta: Delta) -> Self { Self::Init { deltas: delta.into(), } @@ -101,20 +198,19 @@ impl EvalState { let _ = std::mem::replace( self, - EvalState::FetchData { + EvalState::FetchKey { delta, current_idx: 0, groups_to_read: groups_to_read.into_iter().collect(), // Convert BTreeMap to Vec existing_groups: HashMap::new(), old_values: HashMap::new(), - read_record_state: Box::new(ReadRecord::new()), }, ); } fn process_delta( &mut self, operator: &mut AggregateOperator, - cursor: &mut BTreeCursor, + cursors: &mut DbspStateCursors, ) -> Result> { loop { match self { @@ -124,47 +220,144 @@ impl EvalState { EvalState::Init { .. } => { panic!("State machine not supposed to reach the init state! advance() should have been called"); } + EvalState::FetchKey { + delta, + current_idx, + groups_to_read, + existing_groups, + old_values, + } => { + if *current_idx >= groups_to_read.len() { + // All groups have been fetched, move to RecomputeMinMax + // Extract MIN/MAX deltas from the input delta + let min_max_deltas = operator.extract_min_max_deltas(delta); + + let recompute_state = Box::new(RecomputeMinMax::new( + min_max_deltas, + existing_groups, + operator, + )); + + *self = EvalState::RecomputeMinMax { + delta: std::mem::take(delta), + existing_groups: std::mem::take(existing_groups), + old_values: std::mem::take(old_values), + recompute_state, + }; + } else { + // Get the current group to read + let (group_key_str, _group_key) = &groups_to_read[*current_idx]; + + // Build the key for the index: (operator_id, zset_id, element_id) + // For regular aggregates, use column_index=0 and AGG_TYPE_REGULAR + let operator_storage_id = + generate_storage_id(operator.operator_id, 0, AGG_TYPE_REGULAR); + let zset_id = operator.generate_group_rowid(group_key_str); + let element_id = 0i64; // Always 0 for aggregators + + // Create index key values + let index_key_values = vec![ + Value::Integer(operator_storage_id), + Value::Integer(zset_id), + Value::Integer(element_id), + ]; + + // Create an immutable record for the index key + let index_record = + ImmutableRecord::from_values(&index_key_values, index_key_values.len()); + + // Seek in the index to find if this row exists + let seek_result = return_if_io!(cursors.index_cursor.seek( + SeekKey::IndexKey(&index_record), + SeekOp::GE { eq_only: true } + )); + + let rowid = if matches!(seek_result, SeekResult::Found) { + // Found in index, get the table rowid + // The btree code handles extracting the rowid from the index record for has_rowid indexes + return_if_io!(cursors.index_cursor.rowid()) + } else { + // Not found in index, no existing state + None + }; + + // Always transition to FetchData + let taken_existing = std::mem::take(existing_groups); + let taken_old_values = std::mem::take(old_values); + let next_state = EvalState::FetchData { + delta: std::mem::take(delta), + current_idx: *current_idx, + groups_to_read: std::mem::take(groups_to_read), + existing_groups: taken_existing, + old_values: taken_old_values, + rowid, + read_record_state: Box::new(ReadRecord::new()), + }; + *self = next_state; + } + } EvalState::FetchData { delta, current_idx, groups_to_read, existing_groups, old_values, + rowid, read_record_state, } => { - if *current_idx >= groups_to_read.len() { - // All groups processed, compute final output - let result = - operator.merge_delta_with_existing(delta, existing_groups, old_values); - *self = EvalState::Done; - return Ok(IOResult::Done(result)); - } else { - // Get the current group to read - let (group_key_str, group_key) = &groups_to_read[*current_idx]; - - let seek_key = operator.generate_storage_key(group_key_str); - let key = SeekKey::TableRowId(seek_key); + // Get the current group to read + let (group_key_str, group_key) = &groups_to_read[*current_idx]; + // Only try to read if we have a rowid + if let Some(rowid) = rowid { + let key = SeekKey::TableRowId(*rowid); let state = return_if_io!(read_record_state.read_record( key, &operator.aggregates, - cursor + &mut cursors.table_cursor )); - - // Anything that mutates state has to happen after return_if_io! - // Unfortunately there's no good way to enforce that without turning - // this into a hot mess of mem::takes. + // Process the fetched state if let Some(state) = state { let mut old_row = group_key.clone(); old_row.extend(state.to_values(&operator.aggregates)); old_values.insert(group_key_str.clone(), old_row); existing_groups.insert(group_key_str.clone(), state.clone()); } - - // All attributes mutated in place. - *current_idx += 1; - *read_record_state = Box::new(ReadRecord::new()); + } else { + // No rowid for this group, skipping read } + // If no rowid, there's no existing state for this group + + // Move to next group + let next_idx = *current_idx + 1; + let taken_existing = std::mem::take(existing_groups); + let taken_old_values = std::mem::take(old_values); + let next_state = EvalState::FetchKey { + delta: std::mem::take(delta), + current_idx: next_idx, + groups_to_read: std::mem::take(groups_to_read), + existing_groups: taken_existing, + old_values: taken_old_values, + }; + *self = next_state; + } + EvalState::RecomputeMinMax { + delta, + existing_groups, + old_values, + recompute_state, + } => { + if operator.has_min_max() { + // Process MIN/MAX recomputation - this will update existing_groups with correct MIN/MAX + return_if_io!(recompute_state.process(existing_groups, operator, cursors)); + } + + // Now compute final output with updated MIN/MAX values + let (output_delta, computed_states) = + operator.merge_delta_with_existing(delta, existing_groups, old_values); + + *self = EvalState::Done; + return Ok(IOResult::Done((output_delta, computed_states))); } EvalState::Done => { return Ok(IOResult::Done((Delta::new(), HashMap::new()))); @@ -460,7 +653,8 @@ pub enum AggregateFunction { Count, Sum(String), Avg(String), - // MIN and MAX are not supported - see comment in compiler.rs for explanation + Min(String), + Max(String), } impl Display for AggregateFunction { @@ -469,6 +663,8 @@ impl Display for AggregateFunction { AggregateFunction::Count => write!(f, "COUNT(*)"), AggregateFunction::Sum(col) => write!(f, "SUM({col})"), AggregateFunction::Avg(col) => write!(f, "AVG({col})"), + AggregateFunction::Min(col) => write!(f, "MIN({col})"), + AggregateFunction::Max(col) => write!(f, "MAX({col})"), } } } @@ -492,8 +688,8 @@ impl AggregateFunction { AggFunc::Count | AggFunc::Count0 => Some(AggregateFunction::Count), AggFunc::Sum => input_column.map(AggregateFunction::Sum), AggFunc::Avg => input_column.map(AggregateFunction::Avg), - // MIN and MAX are not supported in incremental views - see compiler.rs - AggFunc::Min | AggFunc::Max => None, + AggFunc::Min => input_column.map(AggregateFunction::Min), + AggFunc::Max => input_column.map(AggregateFunction::Max), _ => None, // Other aggregate functions not yet supported in DBSP } } @@ -511,17 +707,25 @@ pub trait IncrementalOperator: Debug { /// /// # Arguments /// * `state` - The evaluation state (may be in progress from a previous I/O operation) - /// * `cursor` - Cursor for reading operator state from storage + /// * `cursors` - Cursors for reading operator state from storage (table and optional index) /// /// # Returns /// The output delta from the evaluation - fn eval(&mut self, state: &mut EvalState, cursor: &mut BTreeCursor) -> Result>; + fn eval( + &mut self, + state: &mut EvalState, + cursors: &mut DbspStateCursors, + ) -> Result>; /// Commit deltas to the operator's internal state and return the output /// This is called when a transaction commits, making changes permanent /// Returns the output delta (what downstream operators should see) - /// The cursor parameter is for operators that need to persist state - fn commit(&mut self, deltas: DeltaPair, cursor: &mut BTreeCursor) -> Result>; + /// The cursors parameter is for operators that need to persist state + fn commit( + &mut self, + deltas: DeltaPair, + cursors: &mut DbspStateCursors, + ) -> Result>; /// Set computation tracker fn set_tracker(&mut self, tracker: Arc>); @@ -548,7 +752,7 @@ impl IncrementalOperator for InputOperator { fn eval( &mut self, state: &mut EvalState, - _cursor: &mut BTreeCursor, + _cursors: &mut DbspStateCursors, ) -> Result> { match state { EvalState::Init { deltas } => { @@ -567,7 +771,11 @@ impl IncrementalOperator for InputOperator { } } - fn commit(&mut self, deltas: DeltaPair, _cursor: &mut BTreeCursor) -> Result> { + fn commit( + &mut self, + deltas: DeltaPair, + _cursors: &mut DbspStateCursors, + ) -> Result> { // Input operator only uses left delta, right must be empty assert!( deltas.right.is_empty(), @@ -697,7 +905,7 @@ impl IncrementalOperator for FilterOperator { fn eval( &mut self, state: &mut EvalState, - _cursor: &mut BTreeCursor, + _cursors: &mut DbspStateCursors, ) -> Result> { let delta = match state { EvalState::Init { deltas } => { @@ -733,7 +941,11 @@ impl IncrementalOperator for FilterOperator { Ok(IOResult::Done(output_delta)) } - fn commit(&mut self, deltas: DeltaPair, _cursor: &mut BTreeCursor) -> Result> { + fn commit( + &mut self, + deltas: DeltaPair, + _cursors: &mut DbspStateCursors, + ) -> Result> { // Filter operator only uses left delta, right must be empty assert!( deltas.right.is_empty(), @@ -1106,7 +1318,7 @@ impl IncrementalOperator for ProjectOperator { fn eval( &mut self, state: &mut EvalState, - _cursor: &mut BTreeCursor, + _cursors: &mut DbspStateCursors, ) -> Result> { let delta = match state { EvalState::Init { deltas } => { @@ -1138,7 +1350,11 @@ impl IncrementalOperator for ProjectOperator { Ok(IOResult::Done(output_delta)) } - fn commit(&mut self, deltas: DeltaPair, _cursor: &mut BTreeCursor) -> Result> { + fn commit( + &mut self, + deltas: DeltaPair, + _cursors: &mut DbspStateCursors, + ) -> Result> { // Project operator only uses left delta, right must be empty assert!( deltas.right.is_empty(), @@ -1168,19 +1384,32 @@ impl IncrementalOperator for ProjectOperator { /// Aggregate operator - performs incremental aggregation with GROUP BY /// Maintains running totals/counts that are updated incrementally /// +/// Information about a column that has MIN/MAX aggregations +#[derive(Debug, Clone)] +pub struct AggColumnInfo { + /// Index used for storage key generation + pub index: usize, + /// Whether this column has a MIN aggregate + pub has_min: bool, + /// Whether this column has a MAX aggregate + pub has_max: bool, +} + /// Note that the AggregateOperator essentially implements a ZSet, even /// though the ZSet structure is never used explicitly. The on-disk btree /// plays the role of the set! #[derive(Debug)] pub struct AggregateOperator { // Unique operator ID for indexing in persistent storage - operator_id: usize, + pub operator_id: usize, // GROUP BY columns group_by: Vec, - // Aggregate functions to compute - aggregates: Vec, + // Aggregate functions to compute (including MIN/MAX) + pub aggregates: Vec, // Column names from input pub input_column_names: Vec, + // Map from column name to aggregate info for quick lookup + pub column_min_max: HashMap, tracker: Option>>, // State machine for commit operation @@ -1188,7 +1417,7 @@ pub struct AggregateOperator { } /// State for a single group's aggregates -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct AggregateState { // For COUNT: just the count count: i64, @@ -1196,17 +1425,51 @@ pub struct AggregateState { sums: HashMap, // For AVG: column_name -> (sum, count) for computing average avgs: HashMap, - // MIN/MAX are not supported - they require O(n) storage overhead for handling deletions - // correctly. See comment in apply_delta() for details. + // For MIN: column_name -> minimum value + pub mins: HashMap, + // For MAX: column_name -> maximum value + pub maxs: HashMap, +} + +/// Serialize a Value using SQLite's serial type format +/// This is used for MIN/MAX values that need to be stored in a compact, sortable format +pub fn serialize_value(value: &Value, blob: &mut Vec) { + let serial_type = crate::types::SerialType::from(value); + let serial_type_u64: u64 = serial_type.into(); + crate::storage::sqlite3_ondisk::write_varint_to_vec(serial_type_u64, blob); + value.serialize_serial(blob); +} + +/// Deserialize a Value using SQLite's serial type format +/// Returns the deserialized value and the number of bytes consumed +pub fn deserialize_value(blob: &[u8]) -> Option<(Value, usize)> { + let mut cursor = 0; + + // Read the serial type + let (serial_type, varint_size) = crate::storage::sqlite3_ondisk::read_varint(blob).ok()?; + cursor += varint_size; + + let serial_type_obj = crate::types::SerialType::try_from(serial_type).ok()?; + let expected_size = serial_type_obj.size(); + + // Read the value + let (value, actual_size) = + crate::storage::sqlite3_ondisk::read_value(&blob[cursor..], serial_type_obj).ok()?; + + // Verify that the actual size matches what we expected from the serial type + if actual_size != expected_size { + return None; // Data corruption - size mismatch + } + + cursor += actual_size; + + // Convert RefValue to Value + Some((value.to_owned(), cursor)) } impl AggregateState { - fn new() -> Self { - Self { - count: 0, - sums: HashMap::new(), - avgs: HashMap::new(), - } + pub fn new() -> Self { + Self::default() } // Serialize the aggregate state to a binary blob including group key values @@ -1268,6 +1531,24 @@ impl AggregateState { AggregateFunction::Count => { // Count is already written above } + AggregateFunction::Min(col_name) => { + // Write whether we have a MIN value (1 byte) + if let Some(min_val) = self.mins.get(col_name) { + blob.push(1u8); // Has value + serialize_value(min_val, &mut blob); + } else { + blob.push(0u8); // No value + } + } + AggregateFunction::Max(col_name) => { + // Write whether we have a MAX value (1 byte) + if let Some(max_val) = self.maxs.get(col_name) { + blob.push(1u8); // Has value + serialize_value(max_val, &mut blob); + } else { + blob.push(0u8); // No value + } + } } } @@ -1355,6 +1636,28 @@ impl AggregateState { AggregateFunction::Count => { // Count was already read above } + AggregateFunction::Min(col_name) => { + // Read whether we have a MIN value + let has_value = *blob.get(cursor)?; + cursor += 1; + + if has_value == 1 { + let (min_value, bytes_consumed) = deserialize_value(&blob[cursor..])?; + cursor += bytes_consumed; + state.mins.insert(col_name.clone(), min_value); + } + } + AggregateFunction::Max(col_name) => { + // Read whether we have a MAX value + let has_value = *blob.get(cursor)?; + cursor += 1; + + if has_value == 1 { + let (max_value, bytes_consumed) = deserialize_value(&blob[cursor..])?; + cursor += bytes_consumed; + state.maxs.insert(col_name.clone(), max_value); + } + } } } @@ -1406,12 +1709,38 @@ impl AggregateState { } } } + AggregateFunction::Min(_col_name) | AggregateFunction::Max(_col_name) => { + // MIN/MAX cannot be handled incrementally in apply_delta because: + // + // 1. For insertions: We can't just keep the minimum/maximum value. + // We need to track ALL values to handle future deletions correctly. + // + // 2. For deletions (retractions): If we delete the current MIN/MAX, + // we need to find the next best value, which requires knowing all + // other values in the group. + // + // Example: Consider MIN(price) with values [10, 20, 30] + // - Current MIN = 10 + // - Delete 10 (weight = -1) + // - New MIN should be 20, but we can't determine this without + // having tracked all values [20, 30] + // + // Therefore, MIN/MAX processing is handled separately: + // - All input values are persisted to the index via persist_min_max() + // - When aggregates have MIN/MAX, we unconditionally transition to + // the RecomputeMinMax state machine (see EvalState::RecomputeMinMax) + // - RecomputeMinMax checks if the current MIN/MAX was deleted, and if so, + // scans the index to find the new MIN/MAX from remaining values + // + // This ensures correctness for incremental computation at the cost of + // additional I/O for MIN/MAX operations. + } } } } /// Convert aggregate state to output values - fn to_values(&self, aggregates: &[AggregateFunction]) -> Vec { + pub fn to_values(&self, aggregates: &[AggregateFunction]) -> Vec { let mut result = Vec::new(); for agg in aggregates { @@ -1439,6 +1768,14 @@ impl AggregateState { result.push(Value::Null); } } + AggregateFunction::Min(col_name) => { + // Return the MIN value from our state + result.push(self.mins.get(col_name).cloned().unwrap_or(Value::Null)); + } + AggregateFunction::Max(col_name) => { + // Return the MAX value from our state + result.push(self.maxs.get(col_name).cloned().unwrap_or(Value::Null)); + } } } @@ -1453,20 +1790,69 @@ impl AggregateOperator { aggregates: Vec, input_column_names: Vec, ) -> Self { + // Build map of column names to their MIN/MAX info with indices + let mut column_min_max = HashMap::new(); + let mut column_indices = HashMap::new(); + let mut current_index = 0; + + // First pass: assign indices to unique MIN/MAX columns + for agg in &aggregates { + match agg { + AggregateFunction::Min(col) | AggregateFunction::Max(col) => { + column_indices.entry(col.clone()).or_insert_with(|| { + let idx = current_index; + current_index += 1; + idx + }); + } + _ => {} + } + } + + // Second pass: build the column info map + for agg in &aggregates { + match agg { + AggregateFunction::Min(col) => { + let index = *column_indices.get(col).unwrap(); + let entry = column_min_max.entry(col.clone()).or_insert(AggColumnInfo { + index, + has_min: false, + has_max: false, + }); + entry.has_min = true; + } + AggregateFunction::Max(col) => { + let index = *column_indices.get(col).unwrap(); + let entry = column_min_max.entry(col.clone()).or_insert(AggColumnInfo { + index, + has_min: false, + has_max: false, + }); + entry.has_max = true; + } + _ => {} + } + } + Self { operator_id, group_by, aggregates, input_column_names, + column_min_max, tracker: None, commit_state: AggregateCommitState::Idle, } } + pub fn has_min_max(&self) -> bool { + !self.column_min_max.is_empty() + } + fn eval_internal( &mut self, state: &mut EvalState, - cursor: &mut BTreeCursor, + cursors: &mut DbspStateCursors, ) -> Result> { match state { EvalState::Uninitialized => { @@ -1493,7 +1879,9 @@ impl AggregateOperator { } state.advance(groups_to_read); } - EvalState::FetchData { .. } => { + EvalState::FetchKey { .. } + | EvalState::FetchData { .. } + | EvalState::RecomputeMinMax { .. } => { // Already in progress, continue processing on process_delta below. } EvalState::Done => { @@ -1502,7 +1890,7 @@ impl AggregateOperator { } // Process the delta through the state machine - let result = return_if_io!(state.process_delta(self, cursor)); + let result = return_if_io!(state.process_delta(self, cursors)); Ok(IOResult::Done(result)) } @@ -1525,9 +1913,7 @@ impl AggregateOperator { let group_key = self.extract_group_key(&row.values); let group_key_str = Self::group_key_to_string(&group_key); - let state = existing_groups - .entry(group_key_str.clone()) - .or_insert_with(AggregateState::new); + let state = existing_groups.entry(group_key_str.clone()).or_default(); temp_keys.insert(group_key_str.clone(), group_key.clone()); @@ -1561,15 +1947,58 @@ impl AggregateOperator { if state.count > 0 { // Build output row: group_by columns + aggregate values let mut output_values = group_key.clone(); - output_values.extend(state.to_values(&self.aggregates)); + let aggregate_values = state.to_values(&self.aggregates); + output_values.extend(aggregate_values); - let output_row = HashableRow::new(result_key, output_values); + let output_row = HashableRow::new(result_key, output_values.clone()); output_delta.changes.push((output_row, 1)); } } (output_delta, final_states) } + /// Extract MIN/MAX values from delta changes for persistence to index + fn extract_min_max_deltas(&self, delta: &Delta) -> MinMaxDeltas { + let mut min_max_deltas: MinMaxDeltas = HashMap::new(); + + for (row, weight) in &delta.changes { + let group_key = self.extract_group_key(&row.values); + let group_key_str = Self::group_key_to_string(&group_key); + + for agg in &self.aggregates { + match agg { + AggregateFunction::Min(col_name) | AggregateFunction::Max(col_name) => { + if let Some(idx) = + self.input_column_names.iter().position(|c| c == col_name) + { + if let Some(val) = row.values.get(idx) { + // Skip NULL values - they don't participate in MIN/MAX + if val == &Value::Null { + continue; + } + // Create a HashableRow with just this value + // Use 0 as rowid since we only care about the value for comparison + let hashable_value = HashableRow::new(0, vec![val.clone()]); + let key = (col_name.clone(), hashable_value); + + let group_entry = + min_max_deltas.entry(group_key_str.clone()).or_default(); + + let value_entry = group_entry.entry(key).or_insert(0); + + // Accumulate the weight + *value_entry += weight; + } + } + } + _ => {} // Ignore non-MIN/MAX aggregates + } + } + } + + min_max_deltas + } + pub fn set_tracker(&mut self, tracker: Arc>) { self.tracker = Some(tracker); } @@ -1577,7 +2006,7 @@ impl AggregateOperator { /// Generate a rowid for a group /// For no GROUP BY: always returns 0 /// For GROUP BY: returns a hash of the group key string - fn generate_group_rowid(&self, group_key_str: &str) -> i64 { + pub fn generate_group_rowid(&self, group_key_str: &str) -> i64 { if self.group_by.is_empty() { 0 } else { @@ -1595,7 +2024,7 @@ impl AggregateOperator { } /// Extract group key values from a row - fn extract_group_key(&self, values: &[Value]) -> Vec { + pub fn extract_group_key(&self, values: &[Value]) -> Vec { let mut key = Vec::new(); for group_col in &self.group_by { @@ -1614,7 +2043,7 @@ impl AggregateOperator { } /// Convert group key to string for indexing (since Value doesn't implement Hash) - fn group_key_to_string(key: &[Value]) -> String { + pub fn group_key_to_string(key: &[Value]) -> String { key.iter() .map(|v| format!("{v:?}")) .collect::>() @@ -1636,18 +2065,26 @@ impl AggregateOperator { } impl IncrementalOperator for AggregateOperator { - fn eval(&mut self, state: &mut EvalState, cursor: &mut BTreeCursor) -> Result> { - let (delta, _) = return_if_io!(self.eval_internal(state, cursor)); + fn eval( + &mut self, + state: &mut EvalState, + cursors: &mut DbspStateCursors, + ) -> Result> { + let (delta, _) = return_if_io!(self.eval_internal(state, cursors)); Ok(IOResult::Done(delta)) } - fn commit(&mut self, deltas: DeltaPair, cursor: &mut BTreeCursor) -> Result> { + fn commit( + &mut self, + mut deltas: DeltaPair, + cursors: &mut DbspStateCursors, + ) -> Result> { // Aggregate operator only uses left delta, right must be empty assert!( deltas.right.is_empty(), "AggregateOperator expects right delta to be empty in commit" ); - let delta = deltas.left; + let delta = std::mem::take(&mut deltas.left); loop { // Note: because we std::mem::replace here (without it, the borrow checker goes nuts, // because we call self.eval_interval, which requires a mutable borrow), we have to @@ -1663,16 +2100,27 @@ impl IncrementalOperator for AggregateOperator { self.commit_state = AggregateCommitState::Eval { eval_state }; } AggregateCommitState::Eval { ref mut eval_state } => { + // Extract input delta before eval for MIN/MAX processing + let input_delta = eval_state.extract_delta(); + + // Extract MIN/MAX deltas before any I/O operations + let min_max_deltas = self.extract_min_max_deltas(&input_delta); + + // Create a new eval state with the same delta + *eval_state = EvalState::from_delta(input_delta.clone()); + let (output_delta, computed_states) = return_and_restore_if_io!( &mut self.commit_state, state, - self.eval_internal(eval_state, cursor) + self.eval_internal(eval_state, cursors) ); + self.commit_state = AggregateCommitState::PersistDelta { delta: output_delta, computed_states, current_idx: 0, write_row: WriteRow::new(), + min_max_deltas, // Store for later use }; } AggregateCommitState::PersistDelta { @@ -1680,57 +2128,92 @@ impl IncrementalOperator for AggregateOperator { computed_states, current_idx, write_row, + min_max_deltas, } => { let states_vec: Vec<_> = computed_states.iter().collect(); if *current_idx >= states_vec.len() { - self.commit_state = AggregateCommitState::Done { + // Use the min_max_deltas we extracted earlier from the input delta + self.commit_state = AggregateCommitState::PersistMinMax { delta: delta.clone(), + min_max_persist_state: MinMaxPersistState::new(min_max_deltas.clone()), }; } else { let (group_key_str, (group_key, agg_state)) = states_vec[*current_idx]; - let seek_key = self.seek_key_from_str(group_key_str); + // Build the key components for the new table structure + // For regular aggregates, use column_index=0 and AGG_TYPE_REGULAR + let operator_storage_id = + generate_storage_id(self.operator_id, 0, AGG_TYPE_REGULAR); + let zset_id = self.generate_group_rowid(group_key_str); + let element_id = 0i64; // Determine weight: -1 to delete (cancels existing weight=1), 1 to insert/update let weight = if agg_state.count == 0 { -1 } else { 1 }; // Serialize the aggregate state with group key (even for deletion, we need a row) let state_blob = agg_state.to_blob(&self.aggregates, group_key); - let blob_row = HashableRow::new(0, vec![Value::Blob(state_blob)]); + let blob_value = Value::Blob(state_blob); - // Build the aggregate storage format: [key, blob, weight] - let seek_key_clone = seek_key.clone(); - let blob_value = blob_row.values[0].clone(); - let build_fn = move |final_weight: isize| -> Vec { - let key_i64 = match seek_key_clone.clone() { - SeekKey::TableRowId(id) => id, - _ => panic!("Expected TableRowId"), - }; - vec![ - Value::Integer(key_i64), - blob_value.clone(), // The blob with serialized state - Value::Integer(final_weight as i64), - ] - }; + // Build the aggregate storage format: [operator_id, zset_id, element_id, value, weight] + let operator_id_val = Value::Integer(operator_storage_id); + let zset_id_val = Value::Integer(zset_id); + let element_id_val = Value::Integer(element_id); + let blob_val = blob_value.clone(); + + // Create index key - the first 3 columns of our primary key + let index_key = vec![ + operator_id_val.clone(), + zset_id_val.clone(), + element_id_val.clone(), + ]; + + // Record values (without weight) + let record_values = + vec![operator_id_val, zset_id_val, element_id_val, blob_val]; return_and_restore_if_io!( &mut self.commit_state, state, - write_row.write_row(cursor, seek_key, build_fn, weight) + write_row.write_row(cursors, index_key, record_values, weight) ); let delta = std::mem::take(delta); let computed_states = std::mem::take(computed_states); + let min_max_deltas = std::mem::take(min_max_deltas); self.commit_state = AggregateCommitState::PersistDelta { delta, computed_states, current_idx: *current_idx + 1, write_row: WriteRow::new(), // Reset for next write + min_max_deltas, }; } } + AggregateCommitState::PersistMinMax { + delta, + min_max_persist_state, + } => { + if !self.has_min_max() { + let delta = std::mem::take(delta); + self.commit_state = AggregateCommitState::Done { delta }; + } else { + return_and_restore_if_io!( + &mut self.commit_state, + state, + min_max_persist_state.persist_min_max( + self.operator_id, + &self.column_min_max, + cursors, + |group_key_str| self.generate_group_rowid(group_key_str) + ) + ); + + let delta = std::mem::take(delta); + self.commit_state = AggregateCommitState::Done { delta }; + } + } AggregateCommitState::Done { delta } => { self.commit_state = AggregateCommitState::Idle; let delta = std::mem::take(delta); @@ -1755,8 +2238,8 @@ mod tests { use crate::{Database, MemoryIO, IO}; use std::sync::{Arc, Mutex}; - /// Create a test pager for operator tests - fn create_test_pager() -> (std::rc::Rc, usize) { + /// Create a test pager for operator tests with both table and index + fn create_test_pager() -> (std::rc::Rc, usize, usize) { let io: Arc = Arc::new(MemoryIO::new()); let db = Database::open_file(io.clone(), ":memory:", false, false).unwrap(); let conn = db.connect().unwrap(); @@ -1766,14 +2249,21 @@ mod tests { // Allocate page 1 first (database header) let _ = pager.io.block(|| pager.allocate_page1()); - // Properly create a BTree for aggregate state using the pager API - let root_page_id = pager + // Create a BTree for the table + let table_root_page_id = pager .io .block(|| pager.btree_create(&CreateBTreeFlags::new_table())) - .expect("Failed to create BTree for aggregate state") + .expect("Failed to create BTree for aggregate state table") as usize; - (pager, root_page_id) + // Create a BTree for the index + let index_root_page_id = pager + .io + .block(|| pager.btree_create(&CreateBTreeFlags::new_index())) + .expect("Failed to create BTree for aggregate state index") + as usize; + + (pager, table_root_page_id, index_root_page_id) } /// Read the current state from the BTree (for testing) @@ -1781,23 +2271,23 @@ mod tests { fn get_current_state_from_btree( agg: &AggregateOperator, pager: &std::rc::Rc, - cursor: &mut BTreeCursor, + cursors: &mut DbspStateCursors, ) -> Delta { let mut result = Delta::new(); // Rewind to start of table - pager.io.block(|| cursor.rewind()).unwrap(); + pager.io.block(|| cursors.table_cursor.rewind()).unwrap(); loop { // Check if cursor is empty (no more rows) - if cursor.is_empty() { + if cursors.table_cursor.is_empty() { break; } // Get the record at this position let record = pager .io - .block(|| cursor.record()) + .block(|| cursors.table_cursor.record()) .unwrap() .unwrap() .to_owned(); @@ -1805,18 +2295,19 @@ mod tests { let values_ref = record.get_values(); let values: Vec = values_ref.into_iter().map(|x| x.to_owned()).collect(); - // Check if this record belongs to our operator - if let Some(Value::Integer(key)) = values.first() { - let operator_part = (key >> 32) as usize; + // Parse the 5-column structure: operator_id, zset_id, element_id, value, weight + if let Some(Value::Integer(op_id)) = values.first() { + // For regular aggregates, use column_index=0 and AGG_TYPE_REGULAR + let expected_op_id = generate_storage_id(agg.operator_id, 0, AGG_TYPE_REGULAR); // Skip if not our operator - if operator_part != agg.operator_id { - pager.io.block(|| cursor.next()).unwrap(); + if *op_id != expected_op_id { + pager.io.block(|| cursors.table_cursor.next()).unwrap(); continue; } - // Get the blob data - if let Some(Value::Blob(blob)) = values.get(1) { + // Get the blob data from column 3 (value column) + if let Some(Value::Blob(blob)) = values.get(3) { // Deserialize the state if let Some((state, group_key)) = AggregateState::from_blob(blob, &agg.aggregates) @@ -1836,7 +2327,7 @@ mod tests { } } - pager.io.block(|| cursor.next()).unwrap(); + pager.io.block(|| cursors.table_cursor.next()).unwrap(); } result.consolidate(); @@ -1871,8 +2362,14 @@ mod tests { // and an insertion (+1) of the new value. // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); // Create an aggregate operator for SUM(age) with no GROUP BY let mut agg = AggregateOperator::new( @@ -1912,11 +2409,11 @@ mod tests { // Initialize with initial data pager .io - .block(|| agg.commit((&initial_delta).into(), &mut cursor)) + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) .unwrap(); // Verify initial state: SUM(age) = 25 + 30 + 35 = 90 - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state.changes.len(), 1, "Should have one aggregate row"); let (row, weight) = &state.changes[0]; assert_eq!(*weight, 1, "Aggregate row should have weight 1"); @@ -1936,7 +2433,7 @@ mod tests { // Process the incremental update let output_delta = pager .io - .block(|| agg.commit((&update_delta).into(), &mut cursor)) + .block(|| agg.commit((&update_delta).into(), &mut cursors)) .unwrap(); // CRITICAL: The output delta should contain TWO changes: @@ -1985,8 +2482,14 @@ mod tests { // Create an aggregate operator for SUM(score) GROUP BY team // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2033,11 +2536,11 @@ mod tests { // Initialize with initial data pager .io - .block(|| agg.commit((&initial_delta).into(), &mut cursor)) + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) .unwrap(); // Verify initial state: red team = 30, blue team = 15 - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state.changes.len(), 2, "Should have two groups"); // Find the red and blue team aggregates @@ -2079,7 +2582,7 @@ mod tests { // Process the incremental update let output_delta = pager .io - .block(|| agg.commit((&update_delta).into(), &mut cursor)) + .block(|| agg.commit((&update_delta).into(), &mut cursors)) .unwrap(); // Should have 2 changes: retraction of old red team sum, insertion of new red team sum @@ -2130,8 +2633,14 @@ mod tests { let tracker = Arc::new(Mutex::new(ComputationTracker::new())); // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); // Create COUNT(*) GROUP BY category let mut agg = AggregateOperator::new( @@ -2161,7 +2670,7 @@ mod tests { } pager .io - .block(|| agg.commit((&initial).into(), &mut cursor)) + .block(|| agg.commit((&initial).into(), &mut cursors)) .unwrap(); // Reset tracker for delta processing @@ -2180,13 +2689,13 @@ mod tests { pager .io - .block(|| agg.commit((&delta).into(), &mut cursor)) + .block(|| agg.commit((&delta).into(), &mut cursors)) .unwrap(); assert_eq!(tracker.lock().unwrap().aggregation_updates, 1); // Check the final state - cat_0 should now have count 11 - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); let cat_0 = final_state .changes .iter() @@ -2205,8 +2714,14 @@ mod tests { // Create SUM(amount) GROUP BY product // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2248,11 +2763,11 @@ mod tests { ); pager .io - .block(|| agg.commit((&initial).into(), &mut cursor)) + .block(|| agg.commit((&initial).into(), &mut cursors)) .unwrap(); // Check initial state: Widget=250, Gadget=200 - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); let widget_sum = state .changes .iter() @@ -2277,13 +2792,13 @@ mod tests { pager .io - .block(|| agg.commit((&delta).into(), &mut cursor)) + .block(|| agg.commit((&delta).into(), &mut cursors)) .unwrap(); assert_eq!(tracker.lock().unwrap().aggregation_updates, 1); // Check final state - Widget should now be 300 (250 + 50) - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); let widget = final_state .changes .iter() @@ -2296,8 +2811,14 @@ mod tests { fn test_count_and_sum_together() { // Test the example from DBSP_ROADMAP: COUNT(*) and SUM(amount) GROUP BY user_id // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2329,13 +2850,13 @@ mod tests { ); pager .io - .block(|| agg.commit((&initial).into(), &mut cursor)) + .block(|| agg.commit((&initial).into(), &mut cursors)) .unwrap(); // Check initial state // User 1: count=2, sum=300 // User 2: count=1, sum=150 - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state.changes.len(), 2); let user1 = state @@ -2364,11 +2885,11 @@ mod tests { ); pager .io - .block(|| agg.commit((&delta).into(), &mut cursor)) + .block(|| agg.commit((&delta).into(), &mut cursors)) .unwrap(); // Check final state - user 1 should have updated count and sum - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); let user1 = final_state .changes .iter() @@ -2382,8 +2903,14 @@ mod tests { fn test_avg_maintains_sum_and_count() { // Test AVG aggregation // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2424,13 +2951,13 @@ mod tests { ); pager .io - .block(|| agg.commit((&initial).into(), &mut cursor)) + .block(|| agg.commit((&initial).into(), &mut cursors)) .unwrap(); // Check initial averages // Category A: avg = (10 + 20) / 2 = 15 // Category B: avg = 30 / 1 = 30 - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); let cat_a = state .changes .iter() @@ -2459,11 +2986,11 @@ mod tests { ); pager .io - .block(|| agg.commit((&delta).into(), &mut cursor)) + .block(|| agg.commit((&delta).into(), &mut cursors)) .unwrap(); // Check final state - Category A avg should now be (10 + 20 + 30) / 3 = 20 - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); let cat_a = final_state .changes .iter() @@ -2476,8 +3003,14 @@ mod tests { fn test_delete_updates_aggregates() { // Test that deletes (negative weights) properly update aggregates // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2513,11 +3046,11 @@ mod tests { ); pager .io - .block(|| agg.commit((&initial).into(), &mut cursor)) + .block(|| agg.commit((&initial).into(), &mut cursors)) .unwrap(); // Check initial state: count=2, sum=300 - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert!(!state.changes.is_empty()); let (row, _weight) = &state.changes[0]; assert_eq!(row.values[1], Value::Integer(2)); // count @@ -2536,11 +3069,11 @@ mod tests { pager .io - .block(|| agg.commit((&delta).into(), &mut cursor)) + .block(|| agg.commit((&delta).into(), &mut cursors)) .unwrap(); // Check final state - should update to count=1, sum=200 - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); let cat_a = final_state .changes .iter() @@ -2557,8 +3090,14 @@ mod tests { let input_columns = vec!["category".to_string(), "value".to_string()]; // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2574,11 +3113,11 @@ mod tests { init_data.insert(3, vec![Value::Text("B".into()), Value::Integer(30)]); pager .io - .block(|| agg.commit((&init_data).into(), &mut cursor)) + .block(|| agg.commit((&init_data).into(), &mut cursors)) .unwrap(); // Check initial counts - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state.changes.len(), 2); // Find group A and B @@ -2602,14 +3141,14 @@ mod tests { let output = pager .io - .block(|| agg.commit((&delete_delta).into(), &mut cursor)) + .block(|| agg.commit((&delete_delta).into(), &mut cursors)) .unwrap(); // Should emit retraction for old count and insertion for new count assert_eq!(output.changes.len(), 2); // Check final state - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); let group_a_final = final_state .changes .iter() @@ -2623,13 +3162,13 @@ mod tests { let output_b = pager .io - .block(|| agg.commit((&delete_all_b).into(), &mut cursor)) + .block(|| agg.commit((&delete_all_b).into(), &mut cursors)) .unwrap(); assert_eq!(output_b.changes.len(), 1); // Only retraction, no new row assert_eq!(output_b.changes[0].1, -1); // Retraction // Final state should not have group B - let final_state2 = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state2 = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(final_state2.changes.len(), 1); // Only group A remains assert_eq!(final_state2.changes[0].0.values[0], Value::Text("A".into())); } @@ -2641,8 +3180,14 @@ mod tests { let input_columns = vec!["category".to_string(), "value".to_string()]; // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2659,11 +3204,11 @@ mod tests { init_data.insert(4, vec![Value::Text("B".into()), Value::Integer(15)]); pager .io - .block(|| agg.commit((&init_data).into(), &mut cursor)) + .block(|| agg.commit((&init_data).into(), &mut cursors)) .unwrap(); // Check initial sums - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); let group_a = state .changes .iter() @@ -2684,11 +3229,11 @@ mod tests { pager .io - .block(|| agg.commit((&delete_delta).into(), &mut cursor)) + .block(|| agg.commit((&delete_delta).into(), &mut cursors)) .unwrap(); // Check updated sum - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); let group_a = state .changes .iter() @@ -2703,11 +3248,11 @@ mod tests { pager .io - .block(|| agg.commit((&delete_all_b).into(), &mut cursor)) + .block(|| agg.commit((&delete_all_b).into(), &mut cursors)) .unwrap(); // Group B should be gone - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(final_state.changes.len(), 1); // Only group A remains assert_eq!(final_state.changes[0].0.values[0], Value::Text("A".into())); } @@ -2719,8 +3264,14 @@ mod tests { let input_columns = vec!["category".to_string(), "value".to_string()]; // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2736,11 +3287,11 @@ mod tests { init_data.insert(3, vec![Value::Text("A".into()), Value::Integer(30)]); pager .io - .block(|| agg.commit((&init_data).into(), &mut cursor)) + .block(|| agg.commit((&init_data).into(), &mut cursors)) .unwrap(); // Check initial average - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state.changes.len(), 1); assert_eq!(state.changes[0].0.values[1], Value::Float(20.0)); // AVG = (10+20+30)/3 = 20 @@ -2750,11 +3301,11 @@ mod tests { pager .io - .block(|| agg.commit((&delete_delta).into(), &mut cursor)) + .block(|| agg.commit((&delete_delta).into(), &mut cursors)) .unwrap(); // Check updated average - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state.changes[0].0.values[1], Value::Float(20.0)); // AVG = (10+30)/2 = 20 (same!) // Delete another to change the average @@ -2763,10 +3314,10 @@ mod tests { pager .io - .block(|| agg.commit((&delete_another).into(), &mut cursor)) + .block(|| agg.commit((&delete_another).into(), &mut cursors)) .unwrap(); - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state.changes[0].0.values[1], Value::Float(10.0)); // AVG = 10/1 = 10 } @@ -2782,8 +3333,14 @@ mod tests { let input_columns = vec!["category".to_string(), "value".to_string()]; // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -2799,11 +3356,11 @@ mod tests { init_data.insert(3, vec![Value::Text("B".into()), Value::Integer(50)]); pager .io - .block(|| agg.commit((&init_data).into(), &mut cursor)) + .block(|| agg.commit((&init_data).into(), &mut cursors)) .unwrap(); // Check initial state - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); let group_a = state .changes .iter() @@ -2820,11 +3377,11 @@ mod tests { pager .io - .block(|| agg.commit((&delete_delta).into(), &mut cursor)) + .block(|| agg.commit((&delete_delta).into(), &mut cursors)) .unwrap(); // Check all aggregates updated correctly - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); let group_a = state .changes .iter() @@ -2841,10 +3398,10 @@ mod tests { pager .io - .block(|| agg.commit((&insert_delta).into(), &mut cursor)) + .block(|| agg.commit((&insert_delta).into(), &mut cursors)) .unwrap(); - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); let group_a = state .changes .iter() @@ -2862,8 +3419,14 @@ mod tests { // the operator should properly consolidate the state // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut filter = FilterOperator::new( FilterPredicate::GreaterThan { @@ -2878,7 +3441,7 @@ mod tests { init_data.insert(3, vec![Value::Integer(3), Value::Integer(3)]); let state = pager .io - .block(|| filter.commit((&init_data).into(), &mut cursor)) + .block(|| filter.commit((&init_data).into(), &mut cursors)) .unwrap(); // Check initial state @@ -2897,7 +3460,7 @@ mod tests { let output = pager .io - .block(|| filter.commit((&update_delta).into(), &mut cursor)) + .block(|| filter.commit((&update_delta).into(), &mut cursors)) .unwrap(); // The output delta should have both changes (both pass the filter b > 2) @@ -2918,8 +3481,14 @@ mod tests { #[test] fn test_filter_eval_with_uncommitted() { // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut filter = FilterOperator::new( FilterPredicate::GreaterThan { @@ -2949,7 +3518,7 @@ mod tests { ); let state = pager .io - .block(|| filter.commit((&init_data).into(), &mut cursor)) + .block(|| filter.commit((&init_data).into(), &mut cursors)) .unwrap(); // Verify initial state (only Alice passes filter) @@ -2979,7 +3548,7 @@ mod tests { let mut eval_state = uncommitted.clone().into(); let result = pager .io - .block(|| filter.eval(&mut eval_state, &mut cursor)) + .block(|| filter.eval(&mut eval_state, &mut cursors)) .unwrap(); assert_eq!( result.changes.len(), @@ -2991,7 +3560,7 @@ mod tests { // Now commit the changes let state = pager .io - .block(|| filter.commit((&uncommitted).into(), &mut cursor)) + .block(|| filter.commit((&uncommitted).into(), &mut cursors)) .unwrap(); // State should now include Charlie (who passes filter) @@ -3006,8 +3575,14 @@ mod tests { fn test_aggregate_eval_with_uncommitted_preserves_state() { // This is the critical test - aggregations must not modify internal state during eval // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -3051,11 +3626,11 @@ mod tests { ); pager .io - .block(|| agg.commit((&init_data).into(), &mut cursor)) + .block(|| agg.commit((&init_data).into(), &mut cursors)) .unwrap(); // Check initial state: A -> (count=2, sum=300), B -> (count=1, sum=150) - let initial_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let initial_state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(initial_state.changes.len(), 2); // Store initial state for comparison @@ -3090,7 +3665,7 @@ mod tests { let mut eval_state = uncommitted.clone().into(); let result = pager .io - .block(|| agg.eval(&mut eval_state, &mut cursor)) + .block(|| agg.eval(&mut eval_state, &mut cursors)) .unwrap(); // Result should contain updates for A and new group C @@ -3099,7 +3674,7 @@ mod tests { assert!(!result.changes.is_empty(), "Should have aggregate changes"); // CRITICAL: Verify internal state hasn't changed - let state_after_eval = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state_after_eval = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!( state_after_eval.changes.len(), 2, @@ -3125,11 +3700,11 @@ mod tests { // Now commit the changes pager .io - .block(|| agg.commit((&uncommitted).into(), &mut cursor)) + .block(|| agg.commit((&uncommitted).into(), &mut cursors)) .unwrap(); // State should now be updated - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(final_state.changes.len(), 3, "Should now have A, B, and C"); let a_final = final_state @@ -3170,8 +3745,14 @@ mod tests { // Test that calling eval multiple times with different uncommitted data // doesn't pollute the internal state // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -3189,11 +3770,11 @@ mod tests { init_data.insert(2, vec![Value::Integer(2), Value::Integer(200)]); pager .io - .block(|| agg.commit((&init_data).into(), &mut cursor)) + .block(|| agg.commit((&init_data).into(), &mut cursors)) .unwrap(); // Initial state: count=2, sum=300 - let initial_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let initial_state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(initial_state.changes.len(), 1); assert_eq!(initial_state.changes[0].0.values[0], Value::Integer(2)); assert_eq!(initial_state.changes[0].0.values[1], Value::Float(300.0)); @@ -3204,11 +3785,11 @@ mod tests { let mut eval_state1 = uncommitted1.clone().into(); let _ = pager .io - .block(|| agg.eval(&mut eval_state1, &mut cursor)) + .block(|| agg.eval(&mut eval_state1, &mut cursors)) .unwrap(); // State should be unchanged - let state1 = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state1 = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state1.changes[0].0.values[0], Value::Integer(2)); assert_eq!(state1.changes[0].0.values[1], Value::Float(300.0)); @@ -3219,11 +3800,11 @@ mod tests { let mut eval_state2 = uncommitted2.clone().into(); let _ = pager .io - .block(|| agg.eval(&mut eval_state2, &mut cursor)) + .block(|| agg.eval(&mut eval_state2, &mut cursors)) .unwrap(); // State should STILL be unchanged - let state2 = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state2 = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state2.changes[0].0.values[0], Value::Integer(2)); assert_eq!(state2.changes[0].0.values[1], Value::Float(300.0)); @@ -3233,11 +3814,11 @@ mod tests { let mut eval_state3 = uncommitted3.clone().into(); let _ = pager .io - .block(|| agg.eval(&mut eval_state3, &mut cursor)) + .block(|| agg.eval(&mut eval_state3, &mut cursors)) .unwrap(); // State should STILL be unchanged - let state3 = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state3 = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state3.changes[0].0.values[0], Value::Integer(2)); assert_eq!(state3.changes[0].0.values[1], Value::Float(300.0)); } @@ -3246,8 +3827,14 @@ mod tests { fn test_aggregate_eval_with_mixed_committed_and_uncommitted() { // Test eval with both committed delta and uncommitted changes // Create a persistent pager for the test - let (pager, root_page_id) = create_test_pager(); - let mut cursor = BTreeCursor::new_table(None, pager.clone(), root_page_id, 10); + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + // Create index cursor with proper index definition for DBSP state table + let index_def = create_dbsp_state_index(index_root_page_id); + // Index has 4 columns: operator_id, zset_id, element_id, rowid + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); let mut agg = AggregateOperator::new( 1, // operator_id for testing @@ -3262,7 +3849,7 @@ mod tests { init_data.insert(2, vec![Value::Integer(2), Value::Text("Y".into())]); pager .io - .block(|| agg.commit((&init_data).into(), &mut cursor)) + .block(|| agg.commit((&init_data).into(), &mut cursors)) .unwrap(); // Create a committed delta (to be processed) @@ -3280,7 +3867,7 @@ mod tests { let mut eval_state = combined.clone().into(); let result = pager .io - .block(|| agg.eval(&mut eval_state, &mut cursor)) + .block(|| agg.eval(&mut eval_state, &mut cursors)) .unwrap(); // Result should reflect changes from both @@ -3334,17 +3921,17 @@ mod tests { assert_eq!(sorted_changes[4].1, 1); // insertion only (no retraction as it's new); // But internal state should be unchanged - let state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let state = get_current_state_from_btree(&agg, &pager, &mut cursors); assert_eq!(state.changes.len(), 2, "Should still have only X and Y"); // Now commit only the committed_delta pager .io - .block(|| agg.commit((&committed_delta).into(), &mut cursor)) + .block(|| agg.commit((&committed_delta).into(), &mut cursors)) .unwrap(); // State should now have X count=2, Y count=1 - let final_state = get_current_state_from_btree(&agg, &pager, &mut cursor); + let final_state = get_current_state_from_btree(&agg, &pager, &mut cursors); let x = final_state .changes .iter() @@ -3352,4 +3939,962 @@ mod tests { .unwrap(); assert_eq!(x.0.values[1], Value::Integer(2)); } + + #[test] + fn test_min_max_basic() { + // Test basic MIN/MAX functionality + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("price".to_string()), + AggregateFunction::Max("price".to_string()), + ], + vec!["id".to_string(), "name".to_string(), "price".to_string()], + ); + + // Initial data + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("Apple".into()), + Value::Float(1.50), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Banana".into()), + Value::Float(0.75), + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + initial_delta.insert( + 4, + vec![ + Value::Integer(4), + Value::Text("Grape".into()), + Value::Float(3.50), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Verify MIN and MAX + assert_eq!(result.changes.len(), 1); + let (row, weight) = &result.changes[0]; + assert_eq!(*weight, 1); + assert_eq!(row.values[0], Value::Float(0.75)); // MIN + assert_eq!(row.values[1], Value::Float(3.50)); // MAX + } + + #[test] + fn test_min_max_deletion_updates_min() { + // Test that deleting the MIN value updates to the next lowest + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("price".to_string()), + AggregateFunction::Max("price".to_string()), + ], + vec!["id".to_string(), "name".to_string(), "price".to_string()], + ); + + // Initial data + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("Apple".into()), + Value::Float(1.50), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Banana".into()), + Value::Float(0.75), + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + initial_delta.insert( + 4, + vec![ + Value::Integer(4), + Value::Text("Grape".into()), + Value::Float(3.50), + ], + ); + + pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Delete the MIN value (Banana at 0.75) + let mut delete_delta = Delta::new(); + delete_delta.delete( + 2, + vec![ + Value::Integer(2), + Value::Text("Banana".into()), + Value::Float(0.75), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&delete_delta).into(), &mut cursors)) + .unwrap(); + + // Should emit retraction of old values and new values + assert_eq!(result.changes.len(), 2); + + // Find the retraction (weight = -1) + let retraction = result.changes.iter().find(|(_, w)| *w == -1).unwrap(); + assert_eq!(retraction.0.values[0], Value::Float(0.75)); // Old MIN + assert_eq!(retraction.0.values[1], Value::Float(3.50)); // Old MAX + + // Find the new values (weight = 1) + let new_values = result.changes.iter().find(|(_, w)| *w == 1).unwrap(); + assert_eq!(new_values.0.values[0], Value::Float(1.50)); // New MIN (Apple) + assert_eq!(new_values.0.values[1], Value::Float(3.50)); // MAX unchanged + } + + #[test] + fn test_min_max_deletion_updates_max() { + // Test that deleting the MAX value updates to the next highest + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("price".to_string()), + AggregateFunction::Max("price".to_string()), + ], + vec!["id".to_string(), "name".to_string(), "price".to_string()], + ); + + // Initial data + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("Apple".into()), + Value::Float(1.50), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Banana".into()), + Value::Float(0.75), + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + initial_delta.insert( + 4, + vec![ + Value::Integer(4), + Value::Text("Grape".into()), + Value::Float(3.50), + ], + ); + + pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Delete the MAX value (Grape at 3.50) + let mut delete_delta = Delta::new(); + delete_delta.delete( + 4, + vec![ + Value::Integer(4), + Value::Text("Grape".into()), + Value::Float(3.50), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&delete_delta).into(), &mut cursors)) + .unwrap(); + + // Should emit retraction of old values and new values + assert_eq!(result.changes.len(), 2); + + // Find the retraction (weight = -1) + let retraction = result.changes.iter().find(|(_, w)| *w == -1).unwrap(); + assert_eq!(retraction.0.values[0], Value::Float(0.75)); // Old MIN + assert_eq!(retraction.0.values[1], Value::Float(3.50)); // Old MAX + + // Find the new values (weight = 1) + let new_values = result.changes.iter().find(|(_, w)| *w == 1).unwrap(); + assert_eq!(new_values.0.values[0], Value::Float(0.75)); // MIN unchanged + assert_eq!(new_values.0.values[1], Value::Float(2.00)); // New MAX (Orange) + } + + #[test] + fn test_min_max_insertion_updates_min() { + // Test that inserting a new MIN value updates the aggregate + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("price".to_string()), + AggregateFunction::Max("price".to_string()), + ], + vec!["id".to_string(), "name".to_string(), "price".to_string()], + ); + + // Initial data + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("Apple".into()), + Value::Float(1.50), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("Grape".into()), + Value::Float(3.50), + ], + ); + + pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Insert a new MIN value + let mut insert_delta = Delta::new(); + insert_delta.insert( + 4, + vec![ + Value::Integer(4), + Value::Text("Lemon".into()), + Value::Float(0.50), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&insert_delta).into(), &mut cursors)) + .unwrap(); + + // Should emit retraction of old values and new values + assert_eq!(result.changes.len(), 2); + + // Find the retraction (weight = -1) + let retraction = result.changes.iter().find(|(_, w)| *w == -1).unwrap(); + assert_eq!(retraction.0.values[0], Value::Float(1.50)); // Old MIN + assert_eq!(retraction.0.values[1], Value::Float(3.50)); // Old MAX + + // Find the new values (weight = 1) + let new_values = result.changes.iter().find(|(_, w)| *w == 1).unwrap(); + assert_eq!(new_values.0.values[0], Value::Float(0.50)); // New MIN (Lemon) + assert_eq!(new_values.0.values[1], Value::Float(3.50)); // MAX unchanged + } + + #[test] + fn test_min_max_insertion_updates_max() { + // Test that inserting a new MAX value updates the aggregate + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("price".to_string()), + AggregateFunction::Max("price".to_string()), + ], + vec!["id".to_string(), "name".to_string(), "price".to_string()], + ); + + // Initial data + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("Apple".into()), + Value::Float(1.50), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("Grape".into()), + Value::Float(3.50), + ], + ); + + pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Insert a new MAX value + let mut insert_delta = Delta::new(); + insert_delta.insert( + 4, + vec![ + Value::Integer(4), + Value::Text("Melon".into()), + Value::Float(5.00), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&insert_delta).into(), &mut cursors)) + .unwrap(); + + // Should emit retraction of old values and new values + assert_eq!(result.changes.len(), 2); + + // Find the retraction (weight = -1) + let retraction = result.changes.iter().find(|(_, w)| *w == -1).unwrap(); + assert_eq!(retraction.0.values[0], Value::Float(1.50)); // Old MIN + assert_eq!(retraction.0.values[1], Value::Float(3.50)); // Old MAX + + // Find the new values (weight = 1) + let new_values = result.changes.iter().find(|(_, w)| *w == 1).unwrap(); + assert_eq!(new_values.0.values[0], Value::Float(1.50)); // MIN unchanged + assert_eq!(new_values.0.values[1], Value::Float(5.00)); // New MAX (Melon) + } + + #[test] + fn test_min_max_update_changes_min() { + // Test that updating a row to become the new MIN updates the aggregate + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("price".to_string()), + AggregateFunction::Max("price".to_string()), + ], + vec!["id".to_string(), "name".to_string(), "price".to_string()], + ); + + // Initial data + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("Apple".into()), + Value::Float(1.50), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("Grape".into()), + Value::Float(3.50), + ], + ); + + pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Update Orange price to be the new MIN (update = delete + insert) + let mut update_delta = Delta::new(); + update_delta.delete( + 2, + vec![ + Value::Integer(2), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + update_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Orange".into()), + Value::Float(0.25), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&update_delta).into(), &mut cursors)) + .unwrap(); + + // Should emit retraction of old values and new values + assert_eq!(result.changes.len(), 2); + + // Find the retraction (weight = -1) + let retraction = result.changes.iter().find(|(_, w)| *w == -1).unwrap(); + assert_eq!(retraction.0.values[0], Value::Float(1.50)); // Old MIN + assert_eq!(retraction.0.values[1], Value::Float(3.50)); // Old MAX + + // Find the new values (weight = 1) + let new_values = result.changes.iter().find(|(_, w)| *w == 1).unwrap(); + assert_eq!(new_values.0.values[0], Value::Float(0.25)); // New MIN (updated Orange) + assert_eq!(new_values.0.values[1], Value::Float(3.50)); // MAX unchanged + } + + #[test] + fn test_min_max_with_group_by() { + // Test MIN/MAX with GROUP BY + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec!["category".to_string()], // GROUP BY category + vec![ + AggregateFunction::Min("price".to_string()), + AggregateFunction::Max("price".to_string()), + ], + vec![ + "id".to_string(), + "category".to_string(), + "name".to_string(), + "price".to_string(), + ], + ); + + // Initial data with two categories + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("fruit".into()), + Value::Text("Apple".into()), + Value::Float(1.50), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("fruit".into()), + Value::Text("Banana".into()), + Value::Float(0.75), + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("fruit".into()), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + initial_delta.insert( + 4, + vec![ + Value::Integer(4), + Value::Text("veggie".into()), + Value::Text("Carrot".into()), + Value::Float(0.50), + ], + ); + initial_delta.insert( + 5, + vec![ + Value::Integer(5), + Value::Text("veggie".into()), + Value::Text("Lettuce".into()), + Value::Float(1.25), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Should have two groups + assert_eq!(result.changes.len(), 2); + + // Find fruit group + let fruit = result + .changes + .iter() + .find(|(row, _)| row.values[0] == Value::Text("fruit".into())) + .unwrap(); + assert_eq!(fruit.1, 1); // weight + assert_eq!(fruit.0.values[1], Value::Float(0.75)); // MIN (Banana) + assert_eq!(fruit.0.values[2], Value::Float(2.00)); // MAX (Orange) + + // Find veggie group + let veggie = result + .changes + .iter() + .find(|(row, _)| row.values[0] == Value::Text("veggie".into())) + .unwrap(); + assert_eq!(veggie.1, 1); // weight + assert_eq!(veggie.0.values[1], Value::Float(0.50)); // MIN (Carrot) + assert_eq!(veggie.0.values[2], Value::Float(1.25)); // MAX (Lettuce) + } + + #[test] + fn test_min_max_with_nulls() { + // Test that NULL values are ignored in MIN/MAX + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("price".to_string()), + AggregateFunction::Max("price".to_string()), + ], + vec!["id".to_string(), "name".to_string(), "price".to_string()], + ); + + // Initial data with NULL values + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("Apple".into()), + Value::Float(1.50), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Unknown1".into()), + Value::Null, + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("Orange".into()), + Value::Float(2.00), + ], + ); + initial_delta.insert( + 4, + vec![ + Value::Integer(4), + Value::Text("Unknown2".into()), + Value::Null, + ], + ); + initial_delta.insert( + 5, + vec![ + Value::Integer(5), + Value::Text("Grape".into()), + Value::Float(3.50), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Verify MIN and MAX ignore NULLs + assert_eq!(result.changes.len(), 1); + let (row, weight) = &result.changes[0]; + assert_eq!(*weight, 1); + assert_eq!(row.values[0], Value::Float(1.50)); // MIN (Apple, ignoring NULLs) + assert_eq!(row.values[1], Value::Float(3.50)); // MAX (Grape, ignoring NULLs) + } + + #[test] + fn test_min_max_integer_values() { + // Test MIN/MAX with integer values instead of floats + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("score".to_string()), + AggregateFunction::Max("score".to_string()), + ], + vec!["id".to_string(), "name".to_string(), "score".to_string()], + ); + + // Initial data with integer scores + let mut initial_delta = Delta::new(); + initial_delta.insert( + 1, + vec![ + Value::Integer(1), + Value::Text("Alice".into()), + Value::Integer(85), + ], + ); + initial_delta.insert( + 2, + vec![ + Value::Integer(2), + Value::Text("Bob".into()), + Value::Integer(92), + ], + ); + initial_delta.insert( + 3, + vec![ + Value::Integer(3), + Value::Text("Carol".into()), + Value::Integer(78), + ], + ); + initial_delta.insert( + 4, + vec![ + Value::Integer(4), + Value::Text("Dave".into()), + Value::Integer(95), + ], + ); + + let result = pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Verify MIN and MAX with integers + assert_eq!(result.changes.len(), 1); + let (row, weight) = &result.changes[0]; + assert_eq!(*weight, 1); + assert_eq!(row.values[0], Value::Integer(78)); // MIN (Carol) + assert_eq!(row.values[1], Value::Integer(95)); // MAX (Dave) + } + + #[test] + fn test_min_max_text_values() { + // Test MIN/MAX with text values (alphabetical ordering) + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("name".to_string()), + AggregateFunction::Max("name".to_string()), + ], + vec!["id".to_string(), "name".to_string()], + ); + + // Initial data with text values + let mut initial_delta = Delta::new(); + initial_delta.insert(1, vec![Value::Integer(1), Value::Text("Charlie".into())]); + initial_delta.insert(2, vec![Value::Integer(2), Value::Text("Alice".into())]); + initial_delta.insert(3, vec![Value::Integer(3), Value::Text("Bob".into())]); + initial_delta.insert(4, vec![Value::Integer(4), Value::Text("David".into())]); + + let result = pager + .io + .block(|| agg.commit((&initial_delta).into(), &mut cursors)) + .unwrap(); + + // Verify MIN and MAX with text (alphabetical) + assert_eq!(result.changes.len(), 1); + let (row, weight) = &result.changes[0]; + assert_eq!(*weight, 1); + assert_eq!(row.values[0], Value::Text("Alice".into())); // MIN alphabetically + assert_eq!(row.values[1], Value::Text("David".into())); // MAX alphabetically + } + + #[test] + fn test_min_max_with_other_aggregates() { + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Count, + AggregateFunction::Sum("value".to_string()), + AggregateFunction::Min("value".to_string()), + AggregateFunction::Max("value".to_string()), + AggregateFunction::Avg("value".to_string()), + ], + vec!["id".to_string(), "value".to_string()], + ); + + // Initial data + let mut delta = Delta::new(); + delta.insert(1, vec![Value::Integer(1), Value::Integer(10)]); + delta.insert(2, vec![Value::Integer(2), Value::Integer(5)]); + delta.insert(3, vec![Value::Integer(3), Value::Integer(15)]); + delta.insert(4, vec![Value::Integer(4), Value::Integer(20)]); + + let result = pager + .io + .block(|| agg.commit((&delta).into(), &mut cursors)) + .unwrap(); + + assert_eq!(result.changes.len(), 1); + let (row, weight) = &result.changes[0]; + assert_eq!(*weight, 1); + assert_eq!(row.values[0], Value::Integer(4)); // COUNT + assert_eq!(row.values[1], Value::Integer(50)); // SUM + assert_eq!(row.values[2], Value::Integer(5)); // MIN + assert_eq!(row.values[3], Value::Integer(20)); // MAX + assert_eq!(row.values[4], Value::Float(12.5)); // AVG (50/4) + + // Delete the MIN value + let mut delta2 = Delta::new(); + delta2.delete(2, vec![Value::Integer(2), Value::Integer(5)]); + + let result2 = pager + .io + .block(|| agg.commit((&delta2).into(), &mut cursors)) + .unwrap(); + + assert_eq!(result2.changes.len(), 2); + let (row_del, weight_del) = &result2.changes[0]; + assert_eq!(*weight_del, -1); + assert_eq!(row_del.values[0], Value::Integer(4)); // Old COUNT + assert_eq!(row_del.values[1], Value::Integer(50)); // Old SUM + assert_eq!(row_del.values[2], Value::Integer(5)); // Old MIN + assert_eq!(row_del.values[3], Value::Integer(20)); // Old MAX + assert_eq!(row_del.values[4], Value::Float(12.5)); // Old AVG + + let (row_ins, weight_ins) = &result2.changes[1]; + assert_eq!(*weight_ins, 1); + assert_eq!(row_ins.values[0], Value::Integer(3)); // New COUNT + assert_eq!(row_ins.values[1], Value::Integer(45)); // New SUM + assert_eq!(row_ins.values[2], Value::Integer(10)); // New MIN + assert_eq!(row_ins.values[3], Value::Integer(20)); // MAX unchanged + assert_eq!(row_ins.values[4], Value::Float(15.0)); // New AVG (45/3) + + // Now delete the MAX value + let mut delta3 = Delta::new(); + delta3.delete(4, vec![Value::Integer(4), Value::Integer(20)]); + + let result3 = pager + .io + .block(|| agg.commit((&delta3).into(), &mut cursors)) + .unwrap(); + + assert_eq!(result3.changes.len(), 2); + let (row_del2, weight_del2) = &result3.changes[0]; + assert_eq!(*weight_del2, -1); + assert_eq!(row_del2.values[3], Value::Integer(20)); // Old MAX + + let (row_ins2, weight_ins2) = &result3.changes[1]; + assert_eq!(*weight_ins2, 1); + assert_eq!(row_ins2.values[0], Value::Integer(2)); // COUNT + assert_eq!(row_ins2.values[1], Value::Integer(25)); // SUM + assert_eq!(row_ins2.values[2], Value::Integer(10)); // MIN unchanged + assert_eq!(row_ins2.values[3], Value::Integer(15)); // New MAX + assert_eq!(row_ins2.values[4], Value::Float(12.5)); // AVG (25/2) + } + + #[test] + fn test_min_max_multiple_columns() { + let (pager, table_root_page_id, index_root_page_id) = create_test_pager(); + let table_cursor = BTreeCursor::new_table(None, pager.clone(), table_root_page_id, 5); + let index_def = create_dbsp_state_index(index_root_page_id); + let index_cursor = + BTreeCursor::new_index(None, pager.clone(), index_root_page_id, &index_def, 4); + let mut cursors = DbspStateCursors::new(table_cursor, index_cursor); + + let mut agg = AggregateOperator::new( + 1, // operator_id + vec![], // No GROUP BY + vec![ + AggregateFunction::Min("col1".to_string()), + AggregateFunction::Max("col2".to_string()), + AggregateFunction::Min("col3".to_string()), + ], + vec!["col1".to_string(), "col2".to_string(), "col3".to_string()], + ); + + // Initial data + let mut delta = Delta::new(); + delta.insert( + 1, + vec![ + Value::Integer(10), + Value::Integer(100), + Value::Integer(1000), + ], + ); + delta.insert( + 2, + vec![Value::Integer(5), Value::Integer(200), Value::Integer(2000)], + ); + delta.insert( + 3, + vec![Value::Integer(15), Value::Integer(150), Value::Integer(500)], + ); + + let result = pager + .io + .block(|| agg.commit((&delta).into(), &mut cursors)) + .unwrap(); + + assert_eq!(result.changes.len(), 1); + let (row, weight) = &result.changes[0]; + assert_eq!(*weight, 1); + assert_eq!(row.values[0], Value::Integer(5)); // MIN(col1) + assert_eq!(row.values[1], Value::Integer(200)); // MAX(col2) + assert_eq!(row.values[2], Value::Integer(500)); // MIN(col3) + + // Delete the row with MIN(col1) and MAX(col2) + let mut delta2 = Delta::new(); + delta2.delete( + 2, + vec![Value::Integer(5), Value::Integer(200), Value::Integer(2000)], + ); + + let result2 = pager + .io + .block(|| agg.commit((&delta2).into(), &mut cursors)) + .unwrap(); + + assert_eq!(result2.changes.len(), 2); + // Should emit delete of old state and insert of new state + let (row_del, weight_del) = &result2.changes[0]; + assert_eq!(*weight_del, -1); + assert_eq!(row_del.values[0], Value::Integer(5)); // Old MIN(col1) + assert_eq!(row_del.values[1], Value::Integer(200)); // Old MAX(col2) + assert_eq!(row_del.values[2], Value::Integer(500)); // Old MIN(col3) + + let (row_ins, weight_ins) = &result2.changes[1]; + assert_eq!(*weight_ins, 1); + assert_eq!(row_ins.values[0], Value::Integer(10)); // New MIN(col1) + assert_eq!(row_ins.values[1], Value::Integer(150)); // New MAX(col2) + assert_eq!(row_ins.values[2], Value::Integer(500)); // MIN(col3) unchanged + } } diff --git a/core/incremental/persistence.rs b/core/incremental/persistence.rs index 381b406aa..eca26cd7c 100644 --- a/core/incremental/persistence.rs +++ b/core/incremental/persistence.rs @@ -1,7 +1,12 @@ -use crate::incremental::operator::{AggregateFunction, AggregateState}; +use crate::incremental::dbsp::HashableRow; +use crate::incremental::operator::{ + generate_storage_id, AggColumnInfo, AggregateFunction, AggregateOperator, AggregateState, + DbspStateCursors, MinMaxDeltas, AGG_TYPE_MINMAX, +}; use crate::storage::btree::{BTreeCursor, BTreeKey}; -use crate::types::{IOResult, SeekKey, SeekOp, SeekResult}; -use crate::{return_if_io, Result, Value}; +use crate::types::{IOResult, ImmutableRecord, RefValue, SeekKey, SeekOp, SeekResult}; +use crate::{return_if_io, LimboError, Result, Value}; +use std::collections::{HashMap, HashSet}; #[derive(Debug, Default)] pub enum ReadRecord { @@ -32,21 +37,22 @@ impl ReadRecord { } else { let record = return_if_io!(cursor.record()); let r = record.ok_or_else(|| { - crate::LimboError::InternalError(format!( + LimboError::InternalError(format!( "Found key {key:?} in aggregate storage but could not read record" )) })?; let values = r.get_values(); - let blob = values[1].to_owned(); + // The blob is in column 3: operator_id, zset_id, element_id, value, weight + let blob = values[3].to_owned(); let (state, _group_key) = match blob { Value::Blob(blob) => AggregateState::from_blob(&blob, aggregates) .ok_or_else(|| { - crate::LimboError::InternalError(format!( + LimboError::InternalError(format!( "Cannot deserialize aggregate state {blob:?}", )) }), - _ => Err(crate::LimboError::ParseError( + _ => Err(LimboError::ParseError( "Value in aggregator not blob".to_string(), )), }?; @@ -63,8 +69,22 @@ impl ReadRecord { pub enum WriteRow { #[default] GetRecord, - Delete, - Insert { + Delete { + rowid: i64, + }, + DeleteIndex, + ComputeNewRowId { + final_weight: isize, + }, + InsertNew { + rowid: i64, + final_weight: isize, + }, + InsertIndex { + rowid: i64, + }, + UpdateExisting { + rowid: i64, final_weight: isize, }, Done, @@ -75,97 +95,193 @@ impl WriteRow { Self::default() } - /// Write a row with weight management. + /// Write a row with weight management using index for lookups. /// /// # Arguments - /// * `cursor` - BTree cursor for the storage - /// * `key` - The key to seek (TableRowId) - /// * `build_record` - Function that builds the record values to insert. - /// Takes the final_weight and returns the complete record values. + /// * `cursors` - DBSP state cursors (table and index) + /// * `index_key` - The key to seek in the index + /// * `record_values` - The record values (without weight) to insert /// * `weight` - The weight delta to apply - pub fn write_row( + pub fn write_row( &mut self, - cursor: &mut BTreeCursor, - key: SeekKey, - build_record: F, + cursors: &mut DbspStateCursors, + index_key: Vec, + record_values: Vec, weight: isize, - ) -> Result> - where - F: Fn(isize) -> Vec, - { + ) -> Result> { loop { match self { WriteRow::GetRecord => { - let res = return_if_io!(cursor.seek(key.clone(), SeekOp::GE { eq_only: true })); + // First, seek in the index to find if the row exists + let index_values = index_key.clone(); + let index_record = + ImmutableRecord::from_values(&index_values, index_values.len()); + + let res = return_if_io!(cursors.index_cursor.seek( + SeekKey::IndexKey(&index_record), + SeekOp::GE { eq_only: true } + )); + if !matches!(res, SeekResult::Found) { - *self = WriteRow::Insert { + // Row doesn't exist, we'll insert a new one + *self = WriteRow::ComputeNewRowId { final_weight: weight, }; } else { - let existing_record = return_if_io!(cursor.record()); + // Found in index, get the rowid it points to + let rowid = return_if_io!(cursors.index_cursor.rowid()); + let rowid = rowid.ok_or_else(|| { + LimboError::InternalError( + "Index cursor does not have a valid rowid".to_string(), + ) + })?; + + // Now seek in the table using the rowid + let table_res = return_if_io!(cursors + .table_cursor + .seek(SeekKey::TableRowId(rowid), SeekOp::GE { eq_only: true })); + + if !matches!(table_res, SeekResult::Found) { + return Err(LimboError::InternalError( + "Index points to non-existent table row".to_string(), + )); + } + + let existing_record = return_if_io!(cursors.table_cursor.record()); let r = existing_record.ok_or_else(|| { - crate::LimboError::InternalError(format!( - "Found key {key:?} in storage but could not read record" - )) + LimboError::InternalError( + "Found rowid in table but could not read record".to_string(), + ) })?; let values = r.get_values(); - // Weight is always the last value - let existing_weight = match values.last() { + // Weight is always the last value (column 4 in our 5-column structure) + let existing_weight = match values.get(4) { Some(val) => match val.to_owned() { Value::Integer(w) => w as isize, _ => { - return Err(crate::LimboError::InternalError(format!( - "Invalid weight value in storage for key {key:?}" - ))) + return Err(LimboError::InternalError( + "Invalid weight value in storage".to_string(), + )) } }, None => { - return Err(crate::LimboError::InternalError(format!( - "No weight value found in storage for key {key:?}" - ))) + return Err(LimboError::InternalError( + "No weight value found in storage".to_string(), + )) } }; let final_weight = existing_weight + weight; if final_weight <= 0 { - *self = WriteRow::Delete + // Store index_key for later deletion of index entry + *self = WriteRow::Delete { rowid } } else { - *self = WriteRow::Insert { final_weight } + // Store the rowid for update + *self = WriteRow::UpdateExisting { + rowid, + final_weight, + } } } } - WriteRow::Delete => { + WriteRow::Delete { rowid } => { + // Seek to the row and delete it + return_if_io!(cursors + .table_cursor + .seek(SeekKey::TableRowId(*rowid), SeekOp::GE { eq_only: true })); + + // Transition to DeleteIndex to also delete the index entry + *self = WriteRow::DeleteIndex; + return_if_io!(cursors.table_cursor.delete()); + } + WriteRow::DeleteIndex => { // Mark as Done before delete to avoid retry on I/O *self = WriteRow::Done; - return_if_io!(cursor.delete()); + return_if_io!(cursors.index_cursor.delete()); } - WriteRow::Insert { final_weight } => { - return_if_io!(cursor.seek(key.clone(), SeekOp::GE { eq_only: true })); - - // Extract the row ID from the key - let key_i64 = match key { - SeekKey::TableRowId(id) => id, - _ => { - return Err(crate::LimboError::InternalError( - "Expected TableRowId for storage".to_string(), - )) + WriteRow::ComputeNewRowId { final_weight } => { + // Find the last rowid to compute the next one + return_if_io!(cursors.table_cursor.last()); + let rowid = if cursors.table_cursor.is_empty() { + 1 + } else { + match return_if_io!(cursors.table_cursor.rowid()) { + Some(id) => id + 1, + None => { + return Err(LimboError::InternalError( + "Table cursor has rows but no valid rowid".to_string(), + )) + } } }; - // Build the record values using the provided function - let record_values = build_record(*final_weight); + // Transition to InsertNew with the computed rowid + *self = WriteRow::InsertNew { + rowid, + final_weight: *final_weight, + }; + } + WriteRow::InsertNew { + rowid, + final_weight, + } => { + let rowid_val = *rowid; + let final_weight_val = *final_weight; + + // Seek to where we want to insert + // The insert will position the cursor correctly + return_if_io!(cursors.table_cursor.seek( + SeekKey::TableRowId(rowid_val), + SeekOp::GE { eq_only: false } + )); + + // Build the complete record with weight + // Use the function parameter record_values directly + let mut complete_record = record_values.clone(); + complete_record.push(Value::Integer(final_weight_val as i64)); // Create an ImmutableRecord from the values - let immutable_record = crate::types::ImmutableRecord::from_values( - &record_values, - record_values.len(), - ); - let btree_key = BTreeKey::new_table_rowid(key_i64, Some(&immutable_record)); + let immutable_record = + ImmutableRecord::from_values(&complete_record, complete_record.len()); + let btree_key = BTreeKey::new_table_rowid(rowid_val, Some(&immutable_record)); + + // Transition to InsertIndex state after table insertion + *self = WriteRow::InsertIndex { rowid: rowid_val }; + return_if_io!(cursors.table_cursor.insert(&btree_key)); + } + WriteRow::InsertIndex { rowid } => { + // For has_rowid indexes, we need to append the rowid to the index key + // Use the function parameter index_key directly + let mut index_values = index_key.clone(); + index_values.push(Value::Integer(*rowid)); + + // Create the index record with the rowid appended + let index_record = + ImmutableRecord::from_values(&index_values, index_values.len()); + let index_btree_key = BTreeKey::new_index_key(&index_record); + + // Mark as Done before index insert to avoid retry on I/O + *self = WriteRow::Done; + return_if_io!(cursors.index_cursor.insert(&index_btree_key)); + } + WriteRow::UpdateExisting { + rowid, + final_weight, + } => { + // Build the complete record with weight + let mut complete_record = record_values.clone(); + complete_record.push(Value::Integer(*final_weight as i64)); + + // Create an ImmutableRecord from the values + let immutable_record = + ImmutableRecord::from_values(&complete_record, complete_record.len()); + let btree_key = BTreeKey::new_table_rowid(*rowid, Some(&immutable_record)); // Mark as Done before insert to avoid retry on I/O *self = WriteRow::Done; - return_if_io!(cursor.insert(&btree_key)); + // BTree insert with existing key will replace the old value + return_if_io!(cursors.table_cursor.insert(&btree_key)); } WriteRow::Done => { return Ok(IOResult::Done(())); @@ -174,3 +290,672 @@ impl WriteRow { } } } + +/// State machine for recomputing MIN/MAX values after deletion +#[derive(Debug)] +pub enum RecomputeMinMax { + ProcessElements { + /// Current column being processed + current_column_idx: usize, + /// Columns to process (combined MIN and MAX) + columns_to_process: Vec<(String, String, bool)>, // (group_key, column_name, is_min) + /// MIN/MAX deltas for checking values and weights + min_max_deltas: MinMaxDeltas, + }, + Scan { + /// Columns still to process + columns_to_process: Vec<(String, String, bool)>, + /// Current index in columns_to_process (will resume from here) + current_column_idx: usize, + /// MIN/MAX deltas for checking values and weights + min_max_deltas: MinMaxDeltas, + /// Current group key being processed + group_key: String, + /// Current column name being processed + column_name: String, + /// Whether we're looking for MIN (true) or MAX (false) + is_min: bool, + /// The scan state machine for finding the new MIN/MAX + scan_state: Box, + }, + Done, +} + +impl RecomputeMinMax { + pub fn new( + min_max_deltas: MinMaxDeltas, + existing_groups: &HashMap, + operator: &AggregateOperator, + ) -> Self { + let mut groups_to_check: HashSet<(String, String, bool)> = HashSet::new(); + + // Remember the min_max_deltas are essentially just the only column that is affected by + // this min/max, in delta (actually ZSet - consolidated delta) format. This makes it easier + // for us to consume it in here. + // + // The most challenging case is the case where there is a retraction, since we need to go + // back to the index. + for (group_key_str, values) in &min_max_deltas { + for ((col_name, hashable_row), weight) in values { + let col_info = operator.column_min_max.get(col_name); + + let value = &hashable_row.values[0]; + + if *weight < 0 { + // Deletion detected - check if it's the current MIN/MAX + if let Some(state) = existing_groups.get(group_key_str) { + // Check for MIN + if let Some(current_min) = state.mins.get(col_name) { + if current_min == value { + groups_to_check.insert(( + group_key_str.clone(), + col_name.clone(), + true, + )); + } + } + // Check for MAX + if let Some(current_max) = state.maxs.get(col_name) { + if current_max == value { + groups_to_check.insert(( + group_key_str.clone(), + col_name.clone(), + false, + )); + } + } + } + } else if *weight > 0 { + // If it is not found in the existing groups, then we only need to care + // about this if this is a new record being inserted + if let Some(info) = col_info { + if info.has_min { + groups_to_check.insert((group_key_str.clone(), col_name.clone(), true)); + } + if info.has_max { + groups_to_check.insert(( + group_key_str.clone(), + col_name.clone(), + false, + )); + } + } + } + } + } + + if groups_to_check.is_empty() { + // No recomputation or initialization needed + Self::Done + } else { + // Convert HashSet to Vec for indexed processing + let groups_to_check_vec: Vec<_> = groups_to_check.into_iter().collect(); + Self::ProcessElements { + current_column_idx: 0, + columns_to_process: groups_to_check_vec, + min_max_deltas, + } + } + } + + pub fn process( + &mut self, + existing_groups: &mut HashMap, + operator: &AggregateOperator, + cursors: &mut DbspStateCursors, + ) -> Result> { + loop { + match self { + RecomputeMinMax::ProcessElements { + current_column_idx, + columns_to_process, + min_max_deltas, + } => { + if *current_column_idx >= columns_to_process.len() { + *self = RecomputeMinMax::Done; + return Ok(IOResult::Done(())); + } + + let (group_key, column_name, is_min) = + columns_to_process[*current_column_idx].clone(); + + // Get column index from pre-computed info + let column_index = operator + .column_min_max + .get(&column_name) + .map(|info| info.index) + .unwrap(); // Should always exist since we're processing known columns + + // Get current value from existing state + let current_value = existing_groups.get(&group_key).and_then(|state| { + if is_min { + state.mins.get(&column_name).cloned() + } else { + state.maxs.get(&column_name).cloned() + } + }); + + // Create storage keys for index lookup + let storage_id = + generate_storage_id(operator.operator_id, column_index, AGG_TYPE_MINMAX); + let zset_id = operator.generate_group_rowid(&group_key); + + // Get the values for this group from min_max_deltas + let group_values = min_max_deltas.get(&group_key).cloned().unwrap_or_default(); + + let columns_to_process = std::mem::take(columns_to_process); + let min_max_deltas = std::mem::take(min_max_deltas); + + let scan_state = if is_min { + Box::new(ScanState::new_for_min( + current_value, + group_key.clone(), + column_name.clone(), + storage_id, + zset_id, + group_values, + )) + } else { + Box::new(ScanState::new_for_max( + current_value, + group_key.clone(), + column_name.clone(), + storage_id, + zset_id, + group_values, + )) + }; + + *self = RecomputeMinMax::Scan { + columns_to_process, + current_column_idx: *current_column_idx, + min_max_deltas, + group_key, + column_name, + is_min, + scan_state, + }; + } + RecomputeMinMax::Scan { + columns_to_process, + current_column_idx, + min_max_deltas, + group_key, + column_name, + is_min, + scan_state, + } => { + // Find new value using the scan state machine + let new_value = return_if_io!(scan_state.find_new_value(cursors)); + + // Update the state with new value (create if doesn't exist) + let state = existing_groups.entry(group_key.clone()).or_default(); + + if *is_min { + if let Some(min_val) = new_value { + state.mins.insert(column_name.clone(), min_val); + } else { + state.mins.remove(column_name); + } + } else if let Some(max_val) = new_value { + state.maxs.insert(column_name.clone(), max_val); + } else { + state.maxs.remove(column_name); + } + + // Move to next column + let min_max_deltas = std::mem::take(min_max_deltas); + let columns_to_process = std::mem::take(columns_to_process); + *self = RecomputeMinMax::ProcessElements { + current_column_idx: *current_column_idx + 1, + columns_to_process, + min_max_deltas, + }; + } + RecomputeMinMax::Done => { + return Ok(IOResult::Done(())); + } + } + } + } +} + +/// State machine for scanning through the index to find new MIN/MAX values +#[derive(Debug)] +pub enum ScanState { + CheckCandidate { + /// Current candidate value for MIN/MAX + candidate: Option, + /// Group key being processed + group_key: String, + /// Column name being processed + column_name: String, + /// Storage ID for the index seek + storage_id: i64, + /// ZSet ID for the group + zset_id: i64, + /// Group values from MinMaxDeltas: (column_name, HashableRow) -> weight + group_values: HashMap<(String, HashableRow), isize>, + /// Whether we're looking for MIN (true) or MAX (false) + is_min: bool, + }, + FetchNextCandidate { + /// Current candidate to seek past + current_candidate: Value, + /// Group key being processed + group_key: String, + /// Column name being processed + column_name: String, + /// Storage ID for the index seek + storage_id: i64, + /// ZSet ID for the group + zset_id: i64, + /// Group values from MinMaxDeltas: (column_name, HashableRow) -> weight + group_values: HashMap<(String, HashableRow), isize>, + /// Whether we're looking for MIN (true) or MAX (false) + is_min: bool, + }, + Done { + /// The final MIN/MAX value found + result: Option, + }, +} + +impl ScanState { + pub fn new_for_min( + current_min: Option, + group_key: String, + column_name: String, + storage_id: i64, + zset_id: i64, + group_values: HashMap<(String, HashableRow), isize>, + ) -> Self { + Self::CheckCandidate { + candidate: current_min, + group_key, + column_name, + storage_id, + zset_id, + group_values, + is_min: true, + } + } + + // Extract a new candidate from the index. It is possible that, when searching, + // we end up going into a different operator altogether. That means we have + // exhausted this operator (or group) entirely, and no good candidate was found + fn extract_new_candidate( + cursors: &mut DbspStateCursors, + index_record: &ImmutableRecord, + seek_op: SeekOp, + storage_id: i64, + zset_id: i64, + ) -> Result>> { + let seek_result = return_if_io!(cursors + .index_cursor + .seek(SeekKey::IndexKey(index_record), seek_op)); + if !matches!(seek_result, SeekResult::Found) { + return Ok(IOResult::Done(None)); + } + + let record = return_if_io!(cursors.index_cursor.record()).ok_or_else(|| { + LimboError::InternalError( + "Record found on the cursor, but could not be read".to_string(), + ) + })?; + + let values = record.get_values(); + if values.len() < 3 { + return Ok(IOResult::Done(None)); + } + + let Some(rec_storage_id) = values.first() else { + return Ok(IOResult::Done(None)); + }; + let Some(rec_zset_id) = values.get(1) else { + return Ok(IOResult::Done(None)); + }; + + // Check if we're still in the same group + if let (RefValue::Integer(rec_sid), RefValue::Integer(rec_zid)) = + (rec_storage_id, rec_zset_id) + { + if *rec_sid != storage_id || *rec_zid != zset_id { + return Ok(IOResult::Done(None)); + } + } else { + return Ok(IOResult::Done(None)); + } + + // Get the value (3rd element) + Ok(IOResult::Done(values.get(2).map(|v| v.to_owned()))) + } + + pub fn new_for_max( + current_max: Option, + group_key: String, + column_name: String, + storage_id: i64, + zset_id: i64, + group_values: HashMap<(String, HashableRow), isize>, + ) -> Self { + Self::CheckCandidate { + candidate: current_max, + group_key, + column_name, + storage_id, + zset_id, + group_values, + is_min: false, + } + } + + pub fn find_new_value( + &mut self, + cursors: &mut DbspStateCursors, + ) -> Result>> { + loop { + match self { + ScanState::CheckCandidate { + candidate, + group_key, + column_name, + storage_id, + zset_id, + group_values, + is_min, + } => { + // First, check if we have a candidate + if let Some(cand_val) = candidate { + // Check if the candidate is retracted (weight <= 0) + // Create a HashableRow to look up the weight + let hashable_cand = HashableRow::new(0, vec![cand_val.clone()]); + let key = (column_name.clone(), hashable_cand); + let is_retracted = + group_values.get(&key).is_some_and(|weight| *weight <= 0); + + if is_retracted { + // Candidate is retracted, need to fetch next from index + *self = ScanState::FetchNextCandidate { + current_candidate: cand_val.clone(), + group_key: std::mem::take(group_key), + column_name: std::mem::take(column_name), + storage_id: *storage_id, + zset_id: *zset_id, + group_values: std::mem::take(group_values), + is_min: *is_min, + }; + continue; + } + } + + // Candidate is valid or we have no candidate + // Now find the best value from insertions in group_values + let mut best_from_zset = None; + for ((col, hashable_val), weight) in group_values.iter() { + if col == column_name && *weight > 0 { + let value = &hashable_val.values[0]; + // Skip NULL values - they don't participate in MIN/MAX + if value == &Value::Null { + continue; + } + // This is an insertion for our column + if let Some(ref current_best) = best_from_zset { + if *is_min { + if value.cmp(current_best) == std::cmp::Ordering::Less { + best_from_zset = Some(value.clone()); + } + } else if value.cmp(current_best) == std::cmp::Ordering::Greater { + best_from_zset = Some(value.clone()); + } + } else { + best_from_zset = Some(value.clone()); + } + } + } + + // Compare candidate with best from ZSet, filtering out NULLs + let result = match (&candidate, &best_from_zset) { + (Some(cand), Some(zset_val)) if cand != &Value::Null => { + if *is_min { + if zset_val.cmp(cand) == std::cmp::Ordering::Less { + Some(zset_val.clone()) + } else { + Some(cand.clone()) + } + } else if zset_val.cmp(cand) == std::cmp::Ordering::Greater { + Some(zset_val.clone()) + } else { + Some(cand.clone()) + } + } + (Some(cand), None) if cand != &Value::Null => Some(cand.clone()), + (None, Some(zset_val)) => Some(zset_val.clone()), + (Some(cand), Some(_)) if cand == &Value::Null => best_from_zset, + _ => None, + }; + + *self = ScanState::Done { result }; + } + + ScanState::FetchNextCandidate { + current_candidate, + group_key, + column_name, + storage_id, + zset_id, + group_values, + is_min, + } => { + // Seek to the next value in the index + let index_key = vec![ + Value::Integer(*storage_id), + Value::Integer(*zset_id), + current_candidate.clone(), + ]; + let index_record = ImmutableRecord::from_values(&index_key, index_key.len()); + + let seek_op = if *is_min { + SeekOp::GT // For MIN, seek greater than current + } else { + SeekOp::LT // For MAX, seek less than current + }; + + let new_candidate = return_if_io!(Self::extract_new_candidate( + cursors, + &index_record, + seek_op, + *storage_id, + *zset_id + )); + + *self = ScanState::CheckCandidate { + candidate: new_candidate, + group_key: std::mem::take(group_key), + column_name: std::mem::take(column_name), + storage_id: *storage_id, + zset_id: *zset_id, + group_values: std::mem::take(group_values), + is_min: *is_min, + }; + } + + ScanState::Done { result } => { + return Ok(IOResult::Done(result.clone())); + } + } + } + } +} + +/// State machine for persisting Min/Max values to storage +#[derive(Debug)] +pub enum MinMaxPersistState { + Init { + min_max_deltas: MinMaxDeltas, + group_keys: Vec, + }, + ProcessGroup { + min_max_deltas: MinMaxDeltas, + group_keys: Vec, + group_idx: usize, + value_idx: usize, + }, + WriteValue { + min_max_deltas: MinMaxDeltas, + group_keys: Vec, + group_idx: usize, + value_idx: usize, + value: Value, + column_name: String, + weight: isize, + write_row: WriteRow, + }, + Done, +} + +impl MinMaxPersistState { + pub fn new(min_max_deltas: MinMaxDeltas) -> Self { + let group_keys: Vec = min_max_deltas.keys().cloned().collect(); + Self::Init { + min_max_deltas, + group_keys, + } + } + + pub fn persist_min_max( + &mut self, + operator_id: usize, + column_min_max: &HashMap, + cursors: &mut DbspStateCursors, + generate_group_rowid: impl Fn(&str) -> i64, + ) -> Result> { + loop { + match self { + MinMaxPersistState::Init { + min_max_deltas, + group_keys, + } => { + let min_max_deltas = std::mem::take(min_max_deltas); + let group_keys = std::mem::take(group_keys); + *self = MinMaxPersistState::ProcessGroup { + min_max_deltas, + group_keys, + group_idx: 0, + value_idx: 0, + }; + } + MinMaxPersistState::ProcessGroup { + min_max_deltas, + group_keys, + group_idx, + value_idx, + } => { + // Check if we're past all groups + if *group_idx >= group_keys.len() { + *self = MinMaxPersistState::Done; + continue; + } + + let group_key_str = &group_keys[*group_idx]; + let values = &min_max_deltas[group_key_str]; // This should always exist + + // Convert HashMap to Vec for indexed access + let values_vec: Vec<_> = values.iter().collect(); + + // Check if we have more values in current group + if *value_idx >= values_vec.len() { + *group_idx += 1; + *value_idx = 0; + // Continue to check if we're past all groups now + continue; + } + + // Process current value and extract what we need before taking ownership + let ((column_name, hashable_row), weight) = values_vec[*value_idx]; + let column_name = column_name.clone(); + let value = hashable_row.values[0].clone(); // Extract the Value from HashableRow + let weight = *weight; + + let min_max_deltas = std::mem::take(min_max_deltas); + let group_keys = std::mem::take(group_keys); + *self = MinMaxPersistState::WriteValue { + min_max_deltas, + group_keys, + group_idx: *group_idx, + value_idx: *value_idx, + column_name, + value, + weight, + write_row: WriteRow::new(), + }; + } + MinMaxPersistState::WriteValue { + min_max_deltas, + group_keys, + group_idx, + value_idx, + value, + column_name, + weight, + write_row, + } => { + // Should have exited in the previous state + assert!(*group_idx < group_keys.len()); + + let group_key_str = &group_keys[*group_idx]; + + // Get the column index from the pre-computed map + let column_info = column_min_max + .get(&*column_name) + .expect("Column should exist in column_min_max map"); + let column_index = column_info.index; + + // Build the key components for MinMax storage using new encoding + let storage_id = + generate_storage_id(operator_id, column_index, AGG_TYPE_MINMAX); + let zset_id = generate_group_rowid(group_key_str); + + // element_id is the actual value for Min/Max + let element_id_val = value.clone(); + + // Create index key + let index_key = vec![ + Value::Integer(storage_id), + Value::Integer(zset_id), + element_id_val.clone(), + ]; + + // Record values (operator_id, zset_id, element_id, unused_placeholder) + // For MIN/MAX, the element_id IS the value, so we use NULL for the 4th column + let record_values = vec![ + Value::Integer(storage_id), + Value::Integer(zset_id), + element_id_val.clone(), + Value::Null, // Placeholder - not used for MIN/MAX + ]; + + return_if_io!(write_row.write_row( + cursors, + index_key.clone(), + record_values, + *weight + )); + + // Move to next value + let min_max_deltas = std::mem::take(min_max_deltas); + let group_keys = std::mem::take(group_keys); + *self = MinMaxPersistState::ProcessGroup { + min_max_deltas, + group_keys, + group_idx: *group_idx, + value_idx: *value_idx + 1, + }; + } + MinMaxPersistState::Done => { + return Ok(IOResult::Done(())); + } + } + } + } +} diff --git a/core/incremental/view.rs b/core/incremental/view.rs index bdcabd7c9..591e95e38 100644 --- a/core/incremental/view.rs +++ b/core/incremental/view.rs @@ -206,6 +206,7 @@ impl IncrementalView { schema: &Schema, main_data_root: usize, internal_state_root: usize, + internal_state_index_root: usize, ) -> Result { // Build the logical plan from the SELECT statement let mut builder = LogicalPlanBuilder::new(schema); @@ -214,7 +215,11 @@ impl IncrementalView { let logical_plan = builder.build_statement(&stmt)?; // Compile the logical plan to a DBSP circuit with the storage roots - let compiler = DbspCompiler::new(main_data_root, internal_state_root); + let compiler = DbspCompiler::new( + main_data_root, + internal_state_root, + internal_state_index_root, + ); let circuit = compiler.compile(&logical_plan)?; Ok(circuit) @@ -271,6 +276,7 @@ impl IncrementalView { schema: &Schema, main_data_root: usize, internal_state_root: usize, + internal_state_index_root: usize, ) -> Result { let mut parser = Parser::new(sql.as_bytes()); let cmd = parser.next_cmd()?; @@ -287,6 +293,7 @@ impl IncrementalView { schema, main_data_root, internal_state_root, + internal_state_index_root, ), _ => Err(LimboError::ParseError(format!( "View is not a CREATE MATERIALIZED VIEW statement: {sql}" @@ -300,6 +307,7 @@ impl IncrementalView { schema: &Schema, main_data_root: usize, internal_state_root: usize, + internal_state_index_root: usize, ) -> Result { let name = view_name.name.as_str().to_string(); @@ -327,6 +335,7 @@ impl IncrementalView { schema, main_data_root, internal_state_root, + internal_state_index_root, ) } @@ -340,13 +349,19 @@ impl IncrementalView { schema: &Schema, main_data_root: usize, internal_state_root: usize, + internal_state_index_root: usize, ) -> Result { // Create the tracker that will be shared by all operators let tracker = Arc::new(Mutex::new(ComputationTracker::new())); // Compile the SELECT statement into a DBSP circuit - let circuit = - Self::try_compile_circuit(&select_stmt, schema, main_data_root, internal_state_root)?; + let circuit = Self::try_compile_circuit( + &select_stmt, + schema, + main_data_root, + internal_state_root, + internal_state_index_root, + )?; Ok(Self { name, diff --git a/core/io/clock.rs b/core/io/clock.rs index d0bdfa009..d522ac278 100644 --- a/core/io/clock.rs +++ b/core/io/clock.rs @@ -6,6 +6,10 @@ pub struct Instant { pub micros: u32, } +const NSEC_PER_SEC: u64 = 1_000_000_000; +const NANOS_PER_MICRO: u32 = 1_000; +const MICROS_PER_SEC: u32 = NSEC_PER_SEC as u32 / NANOS_PER_MICRO; + impl Instant { pub fn to_system_time(self) -> SystemTime { if self.secs >= 0 { @@ -24,6 +28,35 @@ impl Instant { } } } + + pub fn checked_add_duration(&self, other: &Duration) -> Option { + let mut secs = self.secs.checked_add_unsigned(other.as_secs())?; + + // Micros calculations can't overflow because micros are <1B which fit + // in a u32. + let mut micros = other.subsec_micros() + self.micros; + if micros >= MICROS_PER_SEC { + micros -= MICROS_PER_SEC; + secs = secs.checked_add(1)?; + } + + Some(Self { secs, micros }) + } + + pub fn checked_sub_duration(&self, other: &Duration) -> Option { + let mut secs = self.secs.checked_sub_unsigned(other.as_secs())?; + + // Similar to above, micros can't overflow. + let mut micros = self.micros as i32 - other.subsec_micros() as i32; + if micros < 0 { + micros += MICROS_PER_SEC as i32; + secs = secs.checked_sub(1)?; + } + Some(Self { + secs, + micros: micros as u32, + }) + } } impl From> for Instant { @@ -35,6 +68,22 @@ impl From> for Instant { } } +impl std::ops::Add for Instant { + type Output = Instant; + + fn add(self, rhs: Duration) -> Self::Output { + self.checked_add_duration(&rhs).unwrap() + } +} + +impl std::ops::Sub for Instant { + type Output = Instant; + + fn sub(self, rhs: Duration) -> Self::Output { + self.checked_sub_duration(&rhs).unwrap() + } +} + pub trait Clock { fn now(&self) -> Instant; } diff --git a/core/lib.rs b/core/lib.rs index 9c803d41f..00ff043c4 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -40,7 +40,6 @@ pub mod numeric; #[cfg(not(feature = "fuzz"))] mod numeric; -use crate::incremental::view::AllViewsTxState; use crate::storage::checksum::CHECKSUM_REQUIRED_RESERVED_BYTES; use crate::storage::encryption::CipherMode; use crate::translate::pragma::TURSO_CDC_DEFAULT_TABLE_NAME; @@ -50,6 +49,7 @@ use crate::types::{WalFrameInfo, WalState}; use crate::util::{OpenMode, OpenOptions}; use crate::vdbe::metrics::ConnectionMetrics; use crate::vtab::VirtualTable; +use crate::{incremental::view::AllViewsTxState, translate::emitter::TransactionMode}; use core::str; pub use error::{CompletionError, LimboError}; pub use io::clock::{Clock, Instant}; @@ -75,6 +75,7 @@ use std::{ atomic::{AtomicUsize, Ordering}, Arc, LazyLock, Mutex, Weak, }, + time::Duration, }; #[cfg(feature = "fs")] use storage::database::DatabaseFile; @@ -497,7 +498,6 @@ impl Database { ), database_schemas: RefCell::new(std::collections::HashMap::new()), auto_commit: Cell::new(true), - mv_transactions: RefCell::new(Vec::new()), transaction_state: Cell::new(TransactionState::None), last_insert_rowid: Cell::new(0), last_change: Cell::new(0), @@ -511,7 +511,7 @@ impl Database { closed: Cell::new(false), attached_databases: RefCell::new(DatabaseCatalog::new()), query_only: Cell::new(false), - mv_tx_id: Cell::new(None), + mv_tx: Cell::new(None), view_transaction_states: AllViewsTxState::new(), metrics: RefCell::new(ConnectionMetrics::new()), is_nested_stmt: Cell::new(false), @@ -519,6 +519,7 @@ impl Database { encryption_cipher_mode: Cell::new(None), sync_mode: Cell::new(SyncMode::Full), data_sync_retry: Cell::new(false), + busy_timeout: Cell::new(None), }); self.n_connections .fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -978,8 +979,6 @@ pub struct Connection { database_schemas: RefCell>>, /// Whether to automatically commit transaction auto_commit: Cell, - /// Transactions that are in progress. - mv_transactions: RefCell>, transaction_state: Cell, last_insert_rowid: Cell, last_change: Cell, @@ -998,7 +997,7 @@ pub struct Connection { /// Attached databases attached_databases: RefCell, query_only: Cell, - pub(crate) mv_tx_id: Cell>, + pub(crate) mv_tx: Cell>, /// Per-connection view transaction states for uncommitted changes. This represents /// one entry per view that was touched in the transaction. @@ -1012,6 +1011,8 @@ pub struct Connection { encryption_cipher_mode: Cell>, sync_mode: Cell, data_sync_retry: Cell, + /// User defined max accumulated Busy timeout duration + busy_timeout: Cell>, } impl Drop for Connection { @@ -2158,6 +2159,83 @@ impl Connection { } pager.set_encryption_context(cipher_mode, key) } + + /// Sets maximum total accumuated timeout. If the duration is None or Zero, we unset the busy handler for this Connection + /// + /// This api defers slighty from: https://www.sqlite.org/c3ref/busy_timeout.html + /// + /// Instead of sleeping for linear amount of time specified by the user, + /// we will sleep in phases, until the the total amount of time is reached. + /// This means we first sleep of 1ms, then if we still return busy, we sleep for 2 ms, and repeat until a maximum of 100 ms per phase. + /// + /// Example: + /// 1. Set duration to 5ms + /// 2. Step through query -> returns Busy -> sleep/yield for 1 ms + /// 3. Step through query -> returns Busy -> sleep/yield for 2 ms + /// 4. Step through query -> returns Busy -> sleep/yield for 2 ms (totaling 5 ms of sleep) + /// 5. Step through query -> returns Busy -> return Busy to user + /// + /// This slight api change demonstrated a better throughtput in `perf/throughput/turso` benchmark + pub fn busy_timeout(&self, mut duration: Option) { + duration = duration.filter(|duration| !duration.is_zero()); + self.busy_timeout.set(duration); + } +} + +#[derive(Debug, Default)] +struct BusyTimeout { + /// Busy timeout instant + timeout: Option, + /// Max duration of timeout set by Connection + max_duration: Duration, + /// Accumulated duration for busy timeout + /// + /// It will be decremented until it reaches 0, then after that no timeout will be emitted + accum_duration: Duration, + iteration: usize, +} + +impl BusyTimeout { + const DELAYS: [std::time::Duration; 12] = [ + Duration::from_millis(1), + Duration::from_millis(2), + Duration::from_millis(5), + Duration::from_millis(10), + Duration::from_millis(15), + Duration::from_millis(20), + Duration::from_millis(25), + Duration::from_millis(25), + Duration::from_millis(25), + Duration::from_millis(50), + Duration::from_millis(50), + Duration::from_millis(100), + ]; + + pub fn new(duration: std::time::Duration) -> Self { + Self { + timeout: None, + max_duration: duration, + iteration: 0, + accum_duration: duration, + } + } + + pub fn initiate_timeout(&mut self, now: Instant) { + self.timeout = Self::DELAYS.get(self.iteration).and_then(|delay| { + if self.accum_duration.is_zero() { + None + } else { + let new_timeout = now + (*delay).min(self.accum_duration); + self.accum_duration = self.accum_duration.saturating_sub(*delay); + Some(new_timeout) + } + }); + self.iteration = if self.iteration < Self::DELAYS.len() - 1 { + self.iteration + 1 + } else { + self.iteration + }; + } } pub struct Statement { @@ -2173,6 +2251,8 @@ pub struct Statement { query_mode: QueryMode, /// Flag to show if the statement was busy busy: bool, + /// Busy timeout instant + busy_timeout: Option, } impl Statement { @@ -2197,6 +2277,7 @@ impl Statement { accesses_db, query_mode, busy: false, + busy_timeout: None, } } pub fn get_query_mode(&self) -> QueryMode { @@ -2207,8 +2288,8 @@ impl Statement { self.program.n_change.get() } - pub fn set_mv_tx_id(&mut self, mv_tx_id: Option) { - self.program.connection.mv_tx_id.set(mv_tx_id); + pub fn set_mv_tx(&mut self, mv_tx: Option<(u64, TransactionMode)>) { + self.program.connection.mv_tx.set(mv_tx); } pub fn interrupt(&mut self) { @@ -2216,7 +2297,19 @@ impl Statement { } pub fn step(&mut self) -> Result { - let res = if !self.accesses_db { + if let Some(busy_timeout) = self.busy_timeout.as_mut() { + if let Some(timeout) = busy_timeout.timeout { + let now = self.pager.io.now(); + + if now < timeout { + // Yield the query as the timeout has not been reached yet + return Ok(StepResult::IO); + } + // Timeout ended now continue to query execution + } + } + + let mut res = if !self.accesses_db { self.program.step( &mut self.state, self.mv_store.clone(), @@ -2257,6 +2350,18 @@ impl Statement { self.busy = true; } + if matches!(res, Ok(StepResult::Busy)) { + self.check_if_busy_handler_set(); + if let Some(busy_timeout) = self.busy_timeout.as_mut() { + busy_timeout.initiate_timeout(self.pager.io.now()); + if busy_timeout.timeout.is_some() { + // Yield instead of busy, as now we will try to wait for the timeout + // before continuing execution + res = Ok(StepResult::IO); + } + } + } + res } @@ -2427,6 +2532,7 @@ impl Statement { pub fn _reset(&mut self, max_registers: Option, max_cursors: Option) { self.state.reset(max_registers, max_cursors); self.busy = false; + self.check_if_busy_handler_set(); } pub fn row(&self) -> Option<&Row> { @@ -2440,6 +2546,30 @@ impl Statement { pub fn is_busy(&self) -> bool { self.busy } + + /// Checks if the busy handler is set in the connection and sets the handler if needed + fn check_if_busy_handler_set(&mut self) { + let conn_busy_timeout = self + .program + .connection + .busy_timeout + .get() + .map(BusyTimeout::new); + if self.busy_timeout.is_none() { + self.busy_timeout = conn_busy_timeout; + return; + } + if let Some(conn_busy_timeout) = conn_busy_timeout { + let busy_timeout = self + .busy_timeout + .as_mut() + .expect("busy timeout was checked for None above"); + // User changed max duration, so clear previous handler and set a new one + if busy_timeout.max_duration != conn_busy_timeout.max_duration { + *busy_timeout = conn_busy_timeout; + } + } + } } pub type Row = vdbe::Row; diff --git a/core/mvcc/cursor.rs b/core/mvcc/cursor.rs index d9454cd7e..6ffade31a 100644 --- a/core/mvcc/cursor.rs +++ b/core/mvcc/cursor.rs @@ -46,14 +46,20 @@ impl MvccLazyCursor { /// Sets the cursor to the inserted row. pub fn insert(&mut self, row: Row) -> Result<()> { self.current_pos = CursorPosition::Loaded(row.id); - self.db.insert(self.tx_id, row).inspect_err(|_| { - self.current_pos = CursorPosition::BeforeFirst; - })?; + if self.db.read(self.tx_id, row.id)?.is_some() { + self.db.update(self.tx_id, row).inspect_err(|_| { + self.current_pos = CursorPosition::BeforeFirst; + })?; + } else { + self.db.insert(self.tx_id, row).inspect_err(|_| { + self.current_pos = CursorPosition::BeforeFirst; + })?; + } Ok(()) } - pub fn delete(&mut self, rowid: RowID, pager: Rc) -> Result<()> { - self.db.delete(self.tx_id, rowid, pager)?; + pub fn delete(&mut self, rowid: RowID) -> Result<()> { + self.db.delete(self.tx_id, rowid)?; Ok(()) } diff --git a/core/mvcc/database/mod.rs b/core/mvcc/database/mod.rs index 08548a94b..62a7a3b11 100644 --- a/core/mvcc/database/mod.rs +++ b/core/mvcc/database/mod.rs @@ -12,6 +12,7 @@ use crate::storage::sqlite3_ondisk::DatabaseHeader; use crate::storage::wal::TursoRwLock; use crate::types::IOResult; use crate::types::ImmutableRecord; +use crate::types::SeekResult; use crate::Completion; use crate::IOExt; use crate::LimboError; @@ -27,6 +28,8 @@ use std::ops::Bound; use std::rc::Rc; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; +use tracing::instrument; +use tracing::Level; #[cfg(test)] pub mod tests; @@ -141,20 +144,28 @@ impl std::fmt::Display for Transaction { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { write!( f, - "{{ state: {}, id: {}, begin_ts: {}, write_set: {:?}, read_set: {:?}", + "{{ state: {}, id: {}, begin_ts: {}, write_set: [", self.state.load(), self.tx_id, self.begin_ts, - // FIXME: I'm sorry, we obviously shouldn't be cloning here. - self.write_set - .iter() - .map(|v| *v.value()) - .collect::>(), - self.read_set - .iter() - .map(|v| *v.value()) - .collect::>() - ) + )?; + + for (i, v) in self.write_set.iter().enumerate() { + if i > 0 { + write!(f, ", ")? + } + write!(f, "{:?}", *v.value())?; + } + + write!(f, "], read_set: [")?; + for (i, v) in self.read_set.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{:?}", *v.value())?; + } + + write!(f, "] }}") } } @@ -380,7 +391,7 @@ impl StateTransition for CommitStateMachine { type Context = MvStore; type SMResult = (); - #[tracing::instrument(fields(state = ?self.state), skip(self, mvcc_store))] + #[tracing::instrument(fields(state = ?self.state), skip(self, mvcc_store), level = Level::DEBUG)] fn step(&mut self, mvcc_store: &Self::Context) -> Result> { match self.state { CommitState::Initial => { @@ -476,12 +487,26 @@ impl StateTransition for CommitStateMachine { only if TE commits. """ */ - tx.state.store(TransactionState::Committed(end_ts)); tracing::trace!("commit_tx(tx_id={})", self.tx_id); self.write_set .extend(tx.write_set.iter().map(|v| *v.value())); self.write_set .sort_by(|a, b| a.table_id.cmp(&b.table_id).then(a.row_id.cmp(&b.row_id))); + if self.write_set.is_empty() { + tx.state.store(TransactionState::Committed(end_ts)); + if mvcc_store.is_exclusive_tx(&self.tx_id) { + mvcc_store.release_exclusive_tx(&self.tx_id); + self.commit_coordinator.pager_commit_lock.unlock(); + // FIXME: this function isnt re-entrant + self.pager + .io + .block(|| self.pager.end_tx(false, &self.connection))?; + } else { + self.pager.end_read_tx()?; + } + self.finalize(mvcc_store)?; + return Ok(TransitionResult::Done(())); + } self.state = CommitState::BeginPagerTxn { end_ts }; Ok(TransitionResult::Continue) } @@ -501,6 +526,9 @@ impl StateTransition for CommitStateMachine { requires_seek: true, }; return Ok(TransitionResult::Continue); + } else if mvcc_store.has_exclusive_tx() { + // There is an exclusive transaction holding the write lock. We must abort. + return Err(LimboError::WriteWriteConflict); } // Currently txns are queued without any heuristics whasoever. This is important because // we need to ensure writes to disk happen sequentially. @@ -535,9 +563,27 @@ impl StateTransition for CommitStateMachine { })?; } } + // We started a pager read transaction at the beginning of the MV transaction, because + // any reads we do from the database file and WAL must uphold snapshot isolation. + // However, now we must end and immediately restart the read transaction before committing. + // This is because other transactions may have committed writes to the DB file or WAL, + // and our pager must read in those changes when applying our writes; otherwise we would overwrite + // the changes from the previous committed transactions. + // + // Note that this would be incredibly unsafe in the regular transaction model, but in MVCC we trust + // the MV-store to uphold the guarantee that no write-write conflicts happened. + self.pager.end_read_tx().expect("end_read_tx cannot fail"); + let result = self.pager.begin_read_tx()?; + if let crate::result::LimboResult::Busy = result { + // We cannot obtain a WAL read lock due to contention, so we must abort. + self.commit_coordinator.pager_commit_lock.unlock(); + return Err(LimboError::WriteWriteConflict); + } let result = self.pager.io.block(|| self.pager.begin_write_tx())?; if let crate::result::LimboResult::Busy = result { - panic!("Pager write transaction busy, in mvcc this should never happen"); + // There is a non-CONCURRENT transaction holding the write lock. We must abort. + self.commit_coordinator.pager_commit_lock.unlock(); + return Err(LimboError::WriteWriteConflict); } self.state = CommitState::WriteRow { end_ts, @@ -558,8 +604,10 @@ impl StateTransition for CommitStateMachine { let id = &self.write_set[write_set_index]; if let Some(row_versions) = mvcc_store.rows.get(id) { let row_versions = row_versions.value().read(); - // Find rows that were written by this transaction - for row_version in row_versions.iter() { + // Find rows that were written by this transaction. + // Hekaton uses oldest-to-newest order for row versions, so we reverse iterate to find the newest one + // this transaction changed. + for row_version in row_versions.iter().rev() { if let TxTimestampOrID::TxID(row_tx_id) = row_version.begin { if row_tx_id == self.tx_id { let cursor = if let Some(cursor) = self.cursors.get(&id.table_id) { @@ -709,6 +757,9 @@ impl StateTransition for CommitStateMachine { } CommitState::Commit { end_ts } => { let mut log_record = LogRecord::new(end_ts); + let tx = mvcc_store.txs.get(&self.tx_id).unwrap(); + let tx_unlocked = tx.value(); + tx_unlocked.state.store(TransactionState::Committed(end_ts)); for id in &self.write_set { if let Some(row_versions) = mvcc_store.rows.get(id) { let mut row_versions = row_versions.value().write(); @@ -778,7 +829,7 @@ impl StateTransition for WriteRowStateMachine { type Context = (); type SMResult = (); - #[tracing::instrument(fields(state = ?self.state), skip(self, _context))] + #[tracing::instrument(fields(state = ?self.state), skip(self, _context), level = Level::DEBUG)] fn step(&mut self, _context: &Self::Context) -> Result> { use crate::types::{IOResult, SeekKey, SeekOp}; @@ -881,7 +932,13 @@ impl StateTransition for DeleteRowStateMachine { .write() .seek(seek_key, SeekOp::GE { eq_only: true })? { - IOResult::Done(_) => { + IOResult::Done(seek_res) => { + if seek_res == SeekResult::NotFound { + crate::bail_corrupt_error!( + "MVCC delete: rowid {} not found", + self.rowid.row_id + ); + } self.state = DeleteRowState::Delete; Ok(TransitionResult::Continue) } @@ -1028,9 +1085,9 @@ impl MvStore { /// # Returns /// /// Returns `true` if the row was successfully updated, and `false` otherwise. - pub fn update(&self, tx_id: TxID, row: Row, pager: Rc) -> Result { + pub fn update(&self, tx_id: TxID, row: Row) -> Result { tracing::trace!("update(tx_id={}, row.id={:?})", tx_id, row.id); - if !self.delete(tx_id, row.id, pager)? { + if !self.delete(tx_id, row.id)? { return Ok(false); } self.insert(tx_id, row)?; @@ -1039,9 +1096,9 @@ impl MvStore { /// Inserts a row in the database with new values, previously deleting /// any old data if it existed. Bails on a delete error, e.g. write-write conflict. - pub fn upsert(&self, tx_id: TxID, row: Row, pager: Rc) -> Result<()> { + pub fn upsert(&self, tx_id: TxID, row: Row) -> Result<()> { tracing::trace!("upsert(tx_id={}, row.id={:?})", tx_id, row.id); - self.delete(tx_id, row.id, pager)?; + self.delete(tx_id, row.id)?; self.insert(tx_id, row) } @@ -1059,7 +1116,7 @@ impl MvStore { /// /// Returns `true` if the row was successfully deleted, and `false` otherwise. /// - pub fn delete(&self, tx_id: TxID, id: RowID, pager: Rc) -> Result { + pub fn delete(&self, tx_id: TxID, id: RowID) -> Result { tracing::trace!("delete(tx_id={}, id={:?})", tx_id, id); let row_versions_opt = self.rows.get(&id); if let Some(ref row_versions) = row_versions_opt { @@ -1079,7 +1136,6 @@ impl MvStore { if is_write_write_conflict(&self.txs, tx, rv) { drop(row_versions); drop(row_versions_opt); - self.rollback_tx(tx_id, pager); return Err(LimboError::WriteWriteConflict); } @@ -1248,19 +1304,51 @@ impl MvStore { /// /// This is used for IMMEDIATE and EXCLUSIVE transaction types where we need /// to ensure exclusive write access as per SQLite semantics. - pub fn begin_exclusive_tx(&self, pager: Rc) -> Result> { - let tx_id = self.get_tx_id(); + pub fn begin_exclusive_tx( + &self, + pager: Rc, + maybe_existing_tx_id: Option, + ) -> Result> { + self._begin_exclusive_tx(pager, false, maybe_existing_tx_id) + } + + /// Upgrades a read transaction to an exclusive write transaction. + /// + /// This is used for IMMEDIATE and EXCLUSIVE transaction types where we need + /// to ensure exclusive write access as per SQLite semantics. + pub fn upgrade_to_exclusive_tx( + &self, + pager: Rc, + maybe_existing_tx_id: Option, + ) -> Result> { + self._begin_exclusive_tx(pager, true, maybe_existing_tx_id) + } + + /// Begins an exclusive write transaction that prevents concurrent writes. + /// + /// This is used for IMMEDIATE and EXCLUSIVE transaction types where we need + /// to ensure exclusive write access as per SQLite semantics. + #[instrument(skip_all, level = Level::DEBUG)] + fn _begin_exclusive_tx( + &self, + pager: Rc, + is_upgrade_from_read: bool, + maybe_existing_tx_id: Option, + ) -> Result> { + let tx_id = maybe_existing_tx_id.unwrap_or_else(|| self.get_tx_id()); let begin_ts = self.get_timestamp(); self.acquire_exclusive_tx(&tx_id)?; // Try to acquire the pager read lock - match pager.begin_read_tx()? { - LimboResult::Busy => { - self.release_exclusive_tx(&tx_id); - return Err(LimboError::Busy); + if !is_upgrade_from_read { + match pager.begin_read_tx()? { + LimboResult::Busy => { + self.release_exclusive_tx(&tx_id); + return Err(LimboError::Busy); + } + LimboResult::Ok => {} } - LimboResult::Ok => {} } let locked = self.commit_coordinator.pager_commit_lock.write(); if !locked { @@ -1273,7 +1361,15 @@ impl MvStore { LimboResult::Busy => { tracing::debug!("begin_exclusive_tx: tx_id={} failed with Busy", tx_id); // Failed to get pager lock - release our exclusive lock - panic!("begin_exclusive_tx: tx_id={tx_id} failed with Busy, this should never happen as we were able to lock mvcc exclusive write lock"); + self.commit_coordinator.pager_commit_lock.unlock(); + self.release_exclusive_tx(&tx_id); + if maybe_existing_tx_id.is_none() { + // If we were upgrading an existing non-CONCURRENT mvcc transaction to write, we don't end the read tx on Busy. + // But if we were beginning a completely new non-CONCURRENT mvcc transaction, we do end it because the next time the connection + // attempts to do something, it will open a new read tx, which will fail if we don't end this one here. + pager.end_read_tx()?; + } + return Err(LimboError::Busy); } LimboResult::Ok => { let tx = Transaction::new(tx_id, begin_ts); @@ -1294,7 +1390,7 @@ impl MvStore { /// This function starts a new transaction in the database and returns a `TxID` value /// that you can use to perform operations within the transaction. All changes made within the /// transaction are isolated from other transactions until you commit the transaction. - pub fn begin_tx(&self, pager: Rc) -> TxID { + pub fn begin_tx(&self, pager: Rc) -> Result { let tx_id = self.get_tx_id(); let begin_ts = self.get_timestamp(); let tx = Transaction::new(tx_id, begin_ts); @@ -1303,8 +1399,11 @@ impl MvStore { // TODO: we need to tie a pager's read transaction to a transaction ID, so that future refactors to read // pages from WAL/DB read from a consistent state to maintiain snapshot isolation. - pager.begin_read_tx().unwrap(); - tx_id + let result = pager.begin_read_tx()?; + if let crate::result::LimboResult::Busy = result { + return Err(LimboError::Busy); + } + Ok(tx_id) } /// Commits a transaction with the specified transaction ID. @@ -1322,7 +1421,6 @@ impl MvStore { pager: Rc, connection: &Arc, ) -> Result>> { - tracing::trace!("commit_tx(tx_id={})", tx_id); let state_machine: StateMachine> = StateMachine::>::new(CommitStateMachine::new( CommitState::Initial, @@ -1343,21 +1441,39 @@ impl MvStore { /// # Arguments /// /// * `tx_id` - The ID of the transaction to abort. - pub fn rollback_tx(&self, tx_id: TxID, pager: Rc) { + pub fn rollback_tx( + &self, + tx_id: TxID, + pager: Rc, + connection: &Connection, + ) -> Result<()> { let tx_unlocked = self.txs.get(&tx_id).unwrap(); let tx = tx_unlocked.value(); - assert_eq!(tx.state, TransactionState::Active); + connection.mv_tx.set(None); + assert!(tx.state == TransactionState::Active || tx.state == TransactionState::Preparing); tx.state.store(TransactionState::Aborted); tracing::trace!("abort(tx_id={})", tx_id); let write_set: Vec = tx.write_set.iter().map(|v| *v.value()).collect(); - if self.is_exclusive_tx(&tx_id) { + let pager_rollback_done = if self.is_exclusive_tx(&tx_id) { + self.commit_coordinator.pager_commit_lock.unlock(); self.release_exclusive_tx(&tx_id); - } + pager.io.block(|| pager.end_tx(true, connection))?; + true + } else { + false + }; for ref id in write_set { if let Some(row_versions) = self.rows.get(id) { let mut row_versions = row_versions.value().write(); + for rv in row_versions.iter_mut() { + if rv.end == Some(TxTimestampOrID::TxID(tx_id)) { + // undo deletions by this transaction + rv.end = None; + } + } + // remove insertions by this transaction row_versions.retain(|rv| rv.begin != TxTimestampOrID::TxID(tx_id)); if row_versions.is_empty() { self.rows.remove(id); @@ -1368,10 +1484,14 @@ impl MvStore { let tx = tx_unlocked.value(); tx.state.store(TransactionState::Terminated); tracing::trace!("terminate(tx_id={})", tx_id); - pager.end_read_tx().unwrap(); + if !pager_rollback_done { + pager.end_read_tx()?; + } // FIXME: verify that we can already remove the transaction here! // Maybe it's fine for snapshot isolation, but too early for serializable? self.txs.remove(&tx_id); + + Ok(()) } /// Returns true if the given transaction is the exclusive transaction. @@ -1379,6 +1499,11 @@ impl MvStore { self.exclusive_tx.read().as_ref() == Some(tx_id) } + /// Returns true if there is an exclusive transaction ongoing. + fn has_exclusive_tx(&self) -> bool { + self.exclusive_tx.read().is_some() + } + /// Acquires the exclusive transaction lock to the given transaction ID. fn acquire_exclusive_tx(&self, tx_id: &TxID) -> Result<()> { let mut exclusive_tx = self.exclusive_tx.write(); @@ -1505,8 +1630,8 @@ impl MvStore { // we can either switch to a tree-like structure, or at least use partition_point() // which performs a binary search for the insertion point. let mut position = 0_usize; - for (i, v) in versions.iter().rev().enumerate() { - if self.get_begin_timestamp(&v.begin) < self.get_begin_timestamp(&row_version.begin) { + for (i, v) in versions.iter().enumerate().rev() { + if self.get_begin_timestamp(&v.begin) <= self.get_begin_timestamp(&row_version.begin) { position = i + 1; break; } @@ -1734,7 +1859,9 @@ fn is_end_visible( match row_version.end { Some(TxTimestampOrID::Timestamp(rv_end_ts)) => current_tx.begin_ts < rv_end_ts, Some(TxTimestampOrID::TxID(rv_end)) => { - let other_tx = txs.get(&rv_end).unwrap(); + let other_tx = txs + .get(&rv_end) + .unwrap_or_else(|| panic!("Transaction {rv_end} not found")); let other_tx = other_tx.value(); let visible = match other_tx.state.load() { // V's sharp mind discovered an issue with the hekaton paper which basically states that a diff --git a/core/mvcc/database/tests.rs b/core/mvcc/database/tests.rs index a348d89d0..9ff6f2416 100644 --- a/core/mvcc/database/tests.rs +++ b/core/mvcc/database/tests.rs @@ -95,7 +95,10 @@ pub(crate) fn generate_simple_string_row(table_id: u64, id: i64, data: &str) -> fn test_insert_read() { let db = MvccTestDb::new(); - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let tx1_row = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db @@ -112,7 +115,10 @@ fn test_insert_read() { assert_eq!(tx1_row, row); commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); - let tx2 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx2 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let row = db .mvcc_store .read( @@ -130,7 +136,10 @@ fn test_insert_read() { #[test] fn test_read_nonexistent() { let db = MvccTestDb::new(); - let tx = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let row = db.mvcc_store.read( tx, RowID { @@ -145,7 +154,10 @@ fn test_read_nonexistent() { fn test_delete() { let db = MvccTestDb::new(); - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let tx1_row = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db @@ -167,7 +179,6 @@ fn test_delete() { table_id: 1, row_id: 1, }, - db.conn.pager.borrow().clone(), ) .unwrap(); let row = db @@ -183,7 +194,10 @@ fn test_delete() { assert!(row.is_none()); commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); - let tx2 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx2 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let row = db .mvcc_store .read( @@ -200,7 +214,10 @@ fn test_delete() { #[test] fn test_delete_nonexistent() { let db = MvccTestDb::new(); - let tx = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); assert!(!db .mvcc_store .delete( @@ -209,7 +226,6 @@ fn test_delete_nonexistent() { table_id: 1, row_id: 1 }, - db.conn.pager.borrow().clone(), ) .unwrap()); } @@ -217,7 +233,10 @@ fn test_delete_nonexistent() { #[test] fn test_commit() { let db = MvccTestDb::new(); - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let tx1_row = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db @@ -233,9 +252,7 @@ fn test_commit() { .unwrap(); assert_eq!(tx1_row, row); let tx1_updated_row = generate_simple_string_row(1, 1, "World"); - db.mvcc_store - .update(tx1, tx1_updated_row.clone(), db.conn.pager.borrow().clone()) - .unwrap(); + db.mvcc_store.update(tx1, tx1_updated_row.clone()).unwrap(); let row = db .mvcc_store .read( @@ -250,7 +267,10 @@ fn test_commit() { assert_eq!(tx1_updated_row, row); commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); - let tx2 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx2 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let row = db .mvcc_store .read( @@ -270,7 +290,10 @@ fn test_commit() { #[test] fn test_rollback() { let db = MvccTestDb::new(); - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let row1 = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx1, row1.clone()).unwrap(); let row2 = db @@ -286,9 +309,7 @@ fn test_rollback() { .unwrap(); assert_eq!(row1, row2); let row3 = generate_simple_string_row(1, 1, "World"); - db.mvcc_store - .update(tx1, row3.clone(), db.conn.pager.borrow().clone()) - .unwrap(); + db.mvcc_store.update(tx1, row3.clone()).unwrap(); let row4 = db .mvcc_store .read( @@ -302,8 +323,12 @@ fn test_rollback() { .unwrap(); assert_eq!(row3, row4); db.mvcc_store - .rollback_tx(tx1, db.conn.pager.borrow().clone()); - let tx2 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + .rollback_tx(tx1, db.conn.pager.borrow().clone(), &db.conn) + .unwrap(); + let tx2 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let row5 = db .mvcc_store .read( @@ -322,7 +347,10 @@ fn test_dirty_write() { let db = MvccTestDb::new(); // T1 inserts a row with ID 1, but does not commit. - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let tx1_row = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db @@ -340,12 +368,12 @@ fn test_dirty_write() { let conn2 = db.db.connect().unwrap(); // T2 attempts to delete row with ID 1, but fails because T1 has not committed. - let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); - let tx2_row = generate_simple_string_row(1, 1, "World"); - assert!(!db + let tx2 = db .mvcc_store - .update(tx2, tx2_row, conn2.pager.borrow().clone()) - .unwrap()); + .begin_tx(conn2.pager.borrow().clone()) + .unwrap(); + let tx2_row = generate_simple_string_row(1, 1, "World"); + assert!(!db.mvcc_store.update(tx2, tx2_row).unwrap()); let row = db .mvcc_store @@ -366,13 +394,19 @@ fn test_dirty_read() { let db = MvccTestDb::new(); // T1 inserts a row with ID 1, but does not commit. - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let row1 = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx1, row1).unwrap(); // T2 attempts to read row with ID 1, but doesn't see one because T1 has not committed. let conn2 = db.db.connect().unwrap(); - let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); + let tx2 = db + .mvcc_store + .begin_tx(conn2.pager.borrow().clone()) + .unwrap(); let row2 = db .mvcc_store .read( @@ -391,14 +425,20 @@ fn test_dirty_read_deleted() { let db = MvccTestDb::new(); // T1 inserts a row with ID 1 and commits. - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let tx1_row = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); // T2 deletes row with ID 1, but does not commit. let conn2 = db.db.connect().unwrap(); - let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); + let tx2 = db + .mvcc_store + .begin_tx(conn2.pager.borrow().clone()) + .unwrap(); assert!(db .mvcc_store .delete( @@ -407,13 +447,15 @@ fn test_dirty_read_deleted() { table_id: 1, row_id: 1 }, - conn2.pager.borrow().clone(), ) .unwrap()); // T3 reads row with ID 1, but doesn't see the delete because T2 hasn't committed. let conn3 = db.db.connect().unwrap(); - let tx3 = db.mvcc_store.begin_tx(conn3.pager.borrow().clone()); + let tx3 = db + .mvcc_store + .begin_tx(conn3.pager.borrow().clone()) + .unwrap(); let row = db .mvcc_store .read( @@ -433,7 +475,10 @@ fn test_fuzzy_read() { let db = MvccTestDb::new(); // T1 inserts a row with ID 1 and commits. - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let tx1_row = generate_simple_string_row(1, 1, "First"); db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db @@ -452,7 +497,10 @@ fn test_fuzzy_read() { // T2 reads the row with ID 1 within an active transaction. let conn2 = db.db.connect().unwrap(); - let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); + let tx2 = db + .mvcc_store + .begin_tx(conn2.pager.borrow().clone()) + .unwrap(); let row = db .mvcc_store .read( @@ -468,11 +516,12 @@ fn test_fuzzy_read() { // T3 updates the row and commits. let conn3 = db.db.connect().unwrap(); - let tx3 = db.mvcc_store.begin_tx(conn3.pager.borrow().clone()); - let tx3_row = generate_simple_string_row(1, 1, "Second"); - db.mvcc_store - .update(tx3, tx3_row, conn3.pager.borrow().clone()) + let tx3 = db + .mvcc_store + .begin_tx(conn3.pager.borrow().clone()) .unwrap(); + let tx3_row = generate_simple_string_row(1, 1, "Second"); + db.mvcc_store.update(tx3, tx3_row).unwrap(); commit_tx(db.mvcc_store.clone(), &conn3, tx3).unwrap(); // T2 still reads the same version of the row as before. @@ -492,9 +541,7 @@ fn test_fuzzy_read() { // T2 tries to update the row, but fails because T3 has already committed an update to the row, // so T2 trying to write would violate snapshot isolation if it succeeded. let tx2_newrow = generate_simple_string_row(1, 1, "Third"); - let update_result = db - .mvcc_store - .update(tx2, tx2_newrow, conn2.pager.borrow().clone()); + let update_result = db.mvcc_store.update(tx2, tx2_newrow); assert!(matches!(update_result, Err(LimboError::WriteWriteConflict))); } @@ -503,7 +550,10 @@ fn test_lost_update() { let db = MvccTestDb::new(); // T1 inserts a row with ID 1 and commits. - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let tx1_row = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); let row = db @@ -522,22 +572,28 @@ fn test_lost_update() { // T2 attempts to update row ID 1 within an active transaction. let conn2 = db.db.connect().unwrap(); - let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); - let tx2_row = generate_simple_string_row(1, 1, "World"); - assert!(db + let tx2 = db .mvcc_store - .update(tx2, tx2_row.clone(), conn2.pager.borrow().clone()) - .unwrap()); + .begin_tx(conn2.pager.borrow().clone()) + .unwrap(); + let tx2_row = generate_simple_string_row(1, 1, "World"); + assert!(db.mvcc_store.update(tx2, tx2_row.clone()).unwrap()); // T3 also attempts to update row ID 1 within an active transaction. let conn3 = db.db.connect().unwrap(); - let tx3 = db.mvcc_store.begin_tx(conn3.pager.borrow().clone()); + let tx3 = db + .mvcc_store + .begin_tx(conn3.pager.borrow().clone()) + .unwrap(); let tx3_row = generate_simple_string_row(1, 1, "Hello, world!"); assert!(matches!( - db.mvcc_store - .update(tx3, tx3_row, conn3.pager.borrow().clone(),), + db.mvcc_store.update(tx3, tx3_row), Err(LimboError::WriteWriteConflict) )); + // hack: in the actual tursodb database we rollback the mvcc tx ourselves, so manually roll it back here + db.mvcc_store + .rollback_tx(tx3, conn3.pager.borrow().clone(), &conn3) + .unwrap(); commit_tx(db.mvcc_store.clone(), &conn2, tx2).unwrap(); assert!(matches!( @@ -546,7 +602,10 @@ fn test_lost_update() { )); let conn4 = db.db.connect().unwrap(); - let tx4 = db.mvcc_store.begin_tx(conn4.pager.borrow().clone()); + let tx4 = db + .mvcc_store + .begin_tx(conn4.pager.borrow().clone()) + .unwrap(); let row = db .mvcc_store .read( @@ -568,19 +627,22 @@ fn test_committed_visibility() { let db = MvccTestDb::new(); // let's add $10 to my account since I like money - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let tx1_row = generate_simple_string_row(1, 1, "10"); db.mvcc_store.insert(tx1, tx1_row.clone()).unwrap(); commit_tx(db.mvcc_store.clone(), &db.conn, tx1).unwrap(); // but I like more money, so let me try adding $10 more let conn2 = db.db.connect().unwrap(); - let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); - let tx2_row = generate_simple_string_row(1, 1, "20"); - assert!(db + let tx2 = db .mvcc_store - .update(tx2, tx2_row.clone(), conn2.pager.borrow().clone()) - .unwrap()); + .begin_tx(conn2.pager.borrow().clone()) + .unwrap(); + let tx2_row = generate_simple_string_row(1, 1, "20"); + assert!(db.mvcc_store.update(tx2, tx2_row.clone()).unwrap()); let row = db .mvcc_store .read( @@ -596,7 +658,10 @@ fn test_committed_visibility() { // can I check how much money I have? let conn3 = db.db.connect().unwrap(); - let tx3 = db.mvcc_store.begin_tx(conn3.pager.borrow().clone()); + let tx3 = db + .mvcc_store + .begin_tx(conn3.pager.borrow().clone()) + .unwrap(); let row = db .mvcc_store .read( @@ -616,10 +681,16 @@ fn test_committed_visibility() { fn test_future_row() { let db = MvccTestDb::new(); - let tx1 = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx1 = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let conn2 = db.db.connect().unwrap(); - let tx2 = db.mvcc_store.begin_tx(conn2.pager.borrow().clone()); + let tx2 = db + .mvcc_store + .begin_tx(conn2.pager.borrow().clone()) + .unwrap(); let tx2_row = generate_simple_string_row(1, 1, "Hello"); db.mvcc_store.insert(tx2, tx2_row).unwrap(); @@ -663,7 +734,10 @@ use crate::{MemoryIO, Statement}; fn setup_test_db() -> (MvccTestDb, u64) { let db = MvccTestDb::new(); - let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx_id = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let table_id = 1; let test_rows = [ @@ -683,13 +757,19 @@ fn setup_test_db() -> (MvccTestDb, u64) { commit_tx(db.mvcc_store.clone(), &db.conn, tx_id).unwrap(); - let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx_id = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); (db, tx_id) } fn setup_lazy_db(initial_keys: &[i64]) -> (MvccTestDb, u64) { let db = MvccTestDb::new(); - let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx_id = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let table_id = 1; for i in initial_keys { @@ -702,7 +782,10 @@ fn setup_lazy_db(initial_keys: &[i64]) -> (MvccTestDb, u64) { commit_tx(db.mvcc_store.clone(), &db.conn, tx_id).unwrap(); - let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx_id = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); (db, tx_id) } @@ -866,10 +949,13 @@ fn test_cursor_with_empty_table() { { // FIXME: force page 1 initialization let pager = db.conn.pager.borrow().clone(); - let tx_id = db.mvcc_store.begin_tx(pager.clone()); + let tx_id = db.mvcc_store.begin_tx(pager.clone()).unwrap(); commit_tx(db.mvcc_store.clone(), &db.conn, tx_id).unwrap(); } - let tx_id = db.mvcc_store.begin_tx(db.conn.pager.borrow().clone()); + let tx_id = db + .mvcc_store + .begin_tx(db.conn.pager.borrow().clone()) + .unwrap(); let table_id = 1; // Empty table // Test LazyScanCursor with empty table @@ -1092,7 +1178,7 @@ fn test_restart() { { let conn = db.connect(); let mvcc_store = db.get_mvcc_store(); - let tx_id = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let tx_id = mvcc_store.begin_tx(conn.pager.borrow().clone()).unwrap(); let row = generate_simple_string_row(1, 1, "foo"); mvcc_store.insert(tx_id, row).unwrap(); @@ -1104,13 +1190,13 @@ fn test_restart() { { let conn = db.connect(); let mvcc_store = db.get_mvcc_store(); - let tx_id = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let tx_id = mvcc_store.begin_tx(conn.pager.borrow().clone()).unwrap(); let row = generate_simple_string_row(1, 2, "bar"); mvcc_store.insert(tx_id, row).unwrap(); commit_tx(mvcc_store.clone(), &conn, tx_id).unwrap(); - let tx_id = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let tx_id = mvcc_store.begin_tx(conn.pager.borrow().clone()).unwrap(); let row = mvcc_store.read(tx_id, RowID::new(1, 2)).unwrap().unwrap(); let record = get_record_value(&row); match record.get_value(0).unwrap() { @@ -1381,3 +1467,30 @@ fn test_batch_writes() { } println!("start: {start} end: {end}"); } + +#[test] +fn transaction_display() { + let state = AtomicTransactionState::from(TransactionState::Preparing); + let tx_id = 42; + let begin_ts = 20250914; + + let write_set = SkipSet::new(); + write_set.insert(RowID::new(1, 11)); + write_set.insert(RowID::new(1, 13)); + + let read_set = SkipSet::new(); + read_set.insert(RowID::new(2, 17)); + read_set.insert(RowID::new(2, 19)); + + let tx = Transaction { + state, + tx_id, + begin_ts, + write_set, + read_set, + }; + + let expected = "{ state: Preparing, id: 42, begin_ts: 20250914, write_set: [RowID { table_id: 1, row_id: 11 }, RowID { table_id: 1, row_id: 13 }], read_set: [RowID { table_id: 2, row_id: 17 }, RowID { table_id: 2, row_id: 19 }] }"; + let output = format!("{tx}"); + assert_eq!(output, expected); +} diff --git a/core/mvcc/mod.rs b/core/mvcc/mod.rs index 8216faa82..26a01cfbb 100644 --- a/core/mvcc/mod.rs +++ b/core/mvcc/mod.rs @@ -65,7 +65,7 @@ mod tests { let conn = db.get_db().connect().unwrap(); let mvcc_store = db.get_db().mv_store.as_ref().unwrap().clone(); for _ in 0..iterations { - let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()).unwrap(); let id = IDS.fetch_add(1, Ordering::SeqCst); let id = RowID { table_id: 1, @@ -74,7 +74,7 @@ mod tests { let row = generate_simple_string_row(1, id.row_id, "Hello"); mvcc_store.insert(tx, row.clone()).unwrap(); commit_tx_no_conn(&db, tx, &conn).unwrap(); - let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()).unwrap(); let committed_row = mvcc_store.read(tx, id).unwrap(); commit_tx_no_conn(&db, tx, &conn).unwrap(); assert_eq!(committed_row, Some(row)); @@ -86,7 +86,7 @@ mod tests { let conn = db.get_db().connect().unwrap(); let mvcc_store = db.get_db().mv_store.as_ref().unwrap().clone(); for _ in 0..iterations { - let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()).unwrap(); let id = IDS.fetch_add(1, Ordering::SeqCst); let id = RowID { table_id: 1, @@ -95,7 +95,7 @@ mod tests { let row = generate_simple_string_row(1, id.row_id, "World"); mvcc_store.insert(tx, row.clone()).unwrap(); commit_tx_no_conn(&db, tx, &conn).unwrap(); - let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()).unwrap(); let committed_row = mvcc_store.read(tx, id).unwrap(); commit_tx_no_conn(&db, tx, &conn).unwrap(); assert_eq!(committed_row, Some(row)); @@ -127,15 +127,14 @@ mod tests { let dropped = mvcc_store.drop_unused_row_versions(); tracing::debug!("garbage collected {dropped} versions"); } - let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()); + let tx = mvcc_store.begin_tx(conn.pager.borrow().clone()).unwrap(); let id = i % 16; let id = RowID { table_id: 1, row_id: id, }; let row = generate_simple_string_row(1, id.row_id, &format!("{prefix} @{tx}")); - if let Err(e) = mvcc_store.upsert(tx, row.clone(), conn.pager.borrow().clone()) - { + if let Err(e) = mvcc_store.upsert(tx, row.clone()) { tracing::trace!("upsert failed: {e}"); failed_upserts += 1; continue; diff --git a/core/schema.rs b/core/schema.rs index b849586aa..cb2817d77 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -306,6 +306,8 @@ impl Schema { // Store DBSP state table root pages: view_name -> dbsp_state_root_page let mut dbsp_state_roots: HashMap = HashMap::new(); + // Store DBSP state table index root pages: view_name -> dbsp_state_index_root_page + let mut dbsp_state_index_roots: HashMap = HashMap::new(); // Store materialized view info (SQL and root page) for later creation let mut materialized_view_info: HashMap = HashMap::new(); @@ -357,6 +359,7 @@ impl Schema { &mut from_sql_indexes, &mut automatic_indices, &mut dbsp_state_roots, + &mut dbsp_state_index_roots, &mut materialized_view_info, )?; drop(record_cursor); @@ -369,7 +372,11 @@ impl Schema { self.populate_indices(from_sql_indexes, automatic_indices)?; - self.populate_materialized_views(materialized_view_info, dbsp_state_roots)?; + self.populate_materialized_views( + materialized_view_info, + dbsp_state_roots, + dbsp_state_index_roots, + )?; Ok(()) } @@ -492,6 +499,7 @@ impl Schema { &mut self, materialized_view_info: std::collections::HashMap, dbsp_state_roots: std::collections::HashMap, + dbsp_state_index_roots: std::collections::HashMap, ) -> Result<()> { for (view_name, (sql, main_root)) in materialized_view_info { // Look up the DBSP state root for this view - must exist for materialized views @@ -501,9 +509,17 @@ impl Schema { )) })?; - // Create the IncrementalView with both root pages - let incremental_view = - IncrementalView::from_sql(&sql, self, main_root, *dbsp_state_root)?; + // Look up the DBSP state index root (may not exist for older schemas) + let dbsp_state_index_root = + dbsp_state_index_roots.get(&view_name).copied().unwrap_or(0); + // Create the IncrementalView with all root pages + let incremental_view = IncrementalView::from_sql( + &sql, + self, + main_root, + *dbsp_state_root, + dbsp_state_index_root, + )?; let referenced_tables = incremental_view.get_referenced_table_names(); // Create a BTreeTable for the materialized view @@ -539,6 +555,7 @@ impl Schema { from_sql_indexes: &mut Vec, automatic_indices: &mut std::collections::HashMap>, dbsp_state_roots: &mut std::collections::HashMap, + dbsp_state_index_roots: &mut std::collections::HashMap, materialized_view_info: &mut std::collections::HashMap, ) -> Result<()> { match ty { @@ -593,12 +610,23 @@ impl Schema { // index|sqlite_autoindex_foo_1|foo|3| let index_name = name.to_string(); let table_name = table_name.to_string(); - match automatic_indices.entry(table_name) { - std::collections::hash_map::Entry::Vacant(e) => { - e.insert(vec![(index_name, root_page as usize)]); - } - std::collections::hash_map::Entry::Occupied(mut e) => { - e.get_mut().push((index_name, root_page as usize)); + + // Check if this is an index for a DBSP state table + if table_name.starts_with(DBSP_TABLE_PREFIX) { + // Extract the view name from __turso_internal_dbsp_state_ + let view_name = table_name + .strip_prefix(DBSP_TABLE_PREFIX) + .unwrap() + .to_string(); + dbsp_state_index_roots.insert(view_name, root_page as usize); + } else { + match automatic_indices.entry(table_name) { + std::collections::hash_map::Entry::Vacant(e) => { + e.insert(vec![(index_name, root_page as usize)]); + } + std::collections::hash_map::Entry::Occupied(mut e) => { + e.get_mut().push((index_name, root_page as usize)); + } } } } diff --git a/core/storage/btree.rs b/core/storage/btree.rs index dd94cdf45..2afbe6708 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -2157,7 +2157,7 @@ impl BTreeCursor { (cmp, found) } - #[instrument(skip_all, level = Level::INFO)] + #[instrument(skip_all, level = Level::DEBUG)] pub fn move_to(&mut self, key: SeekKey<'_>, cmp: SeekOp) -> Result> { turso_assert!( self.mv_cursor.is_none(), @@ -3542,17 +3542,20 @@ impl BTreeCursor { usable_space, )?; let overflow_cell_count_after = parent_contents.overflow_cells.len(); - let divider_cell_is_overflow_cell = - overflow_cell_count_after > overflow_cell_count_before; #[cfg(debug_assertions)] - BTreeCursor::validate_balance_non_root_divider_cell_insertion( - balance_info, - parent_contents, - divider_cell_insert_idx_in_parent, - divider_cell_is_overflow_cell, - page, - usable_space, - ); + { + let divider_cell_is_overflow_cell = + overflow_cell_count_after > overflow_cell_count_before; + + BTreeCursor::validate_balance_non_root_divider_cell_insertion( + balance_info, + parent_contents, + divider_cell_insert_idx_in_parent, + divider_cell_is_overflow_cell, + page, + usable_space, + ); + } } tracing::debug!( "balance_non_root(parent_overflow={})", @@ -4625,7 +4628,7 @@ impl BTreeCursor { } }; let row = crate::mvcc::database::Row::new(row_id, record_buf, num_columns); - mv_cursor.borrow_mut().insert(row).unwrap(); + mv_cursor.borrow_mut().insert(row)?; } None => todo!("Support mvcc inserts with index btrees"), }, @@ -4655,7 +4658,7 @@ impl BTreeCursor { pub fn delete(&mut self) -> Result> { if let Some(mv_cursor) = &self.mv_cursor { let rowid = mv_cursor.borrow_mut().current_row_id().unwrap(); - mv_cursor.borrow_mut().delete(rowid, self.pager.clone())?; + mv_cursor.borrow_mut().delete(rowid)?; return Ok(IOResult::Done(())); } diff --git a/core/storage/pager.rs b/core/storage/pager.rs index 5fff92f93..1ce9e689b 100644 --- a/core/storage/pager.rs +++ b/core/storage/pager.rs @@ -552,13 +552,8 @@ enum AllocatePage1State { #[derive(Debug, Clone)] enum FreePageState { Start, - AddToTrunk { - page: Arc, - trunk_page: Option>, - }, - NewTrunk { - page: Arc, - }, + AddToTrunk { page: Arc }, + NewTrunk { page: Arc }, } impl Pager { @@ -1741,25 +1736,19 @@ impl Pager { let trunk_page_id = header.freelist_trunk_page.get(); if trunk_page_id != 0 { - *state = FreePageState::AddToTrunk { - page, - trunk_page: None, - }; + *state = FreePageState::AddToTrunk { page }; } else { *state = FreePageState::NewTrunk { page }; } } - FreePageState::AddToTrunk { page, trunk_page } => { + FreePageState::AddToTrunk { page } => { let trunk_page_id = header.freelist_trunk_page.get(); - if trunk_page.is_none() { - // Add as leaf to current trunk - let (page, c) = self.read_page(trunk_page_id as usize)?; - trunk_page.replace(page); - if let Some(c) = c { + let (trunk_page, c) = self.read_page(trunk_page_id as usize)?; + if let Some(c) = c { + if !c.is_completed() { io_yield_one!(c); } } - let trunk_page = trunk_page.as_ref().unwrap(); turso_assert!(trunk_page.is_loaded(), "trunk_page should be loaded"); let trunk_page_contents = trunk_page.get_contents(); @@ -1775,7 +1764,7 @@ impl Pager { trunk_page.get().id == trunk_page_id as usize, "trunk page has unexpected id" ); - self.add_dirty(trunk_page); + self.add_dirty(&trunk_page); trunk_page_contents.write_u32_no_offset( TRUNK_PAGE_LEAF_COUNT_OFFSET, diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 96dc77e29..9f6e17966 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -927,7 +927,7 @@ pub fn begin_read_page( db_file.read_page(page_idx, io_ctx, c) } -#[instrument(skip_all, level = Level::INFO)] +#[instrument(skip_all, level = Level::DEBUG)] pub fn finish_read_page(page_idx: usize, buffer_ref: Arc, page: PageRef) { tracing::trace!("finish_read_page(page_idx = {page_idx})"); let pos = if page_idx == DatabaseHeader::PAGE_ID { diff --git a/core/storage/wal.rs b/core/storage/wal.rs index 637216caa..988245b28 100644 --- a/core/storage/wal.rs +++ b/core/storage/wal.rs @@ -1099,14 +1099,27 @@ impl Wal for WalFile { let epoch = shared_file.read().epoch.load(Ordering::Acquire); frame.set_wal_tag(frame_id, epoch); }); - let shared = self.get_shared(); - assert!( - shared.enabled.load(Ordering::Relaxed), - "WAL must be enabled" - ); - let file = shared.file.as_ref().unwrap(); + let file = { + let shared = self.get_shared(); + assert!( + shared.enabled.load(Ordering::Relaxed), + "WAL must be enabled" + ); + // important not to hold shared lock beyond this point to avoid deadlock scenario where: + // thread 1: takes readlock here, passes reference to shared.file to begin_read_wal_frame + // thread 2: tries to acquire write lock elsewhere + // thread 1: tries to re-acquire read lock in the completion (see 'complete' above) + // + // this causes a deadlock due to the locking policy in parking_lot: + // from https://docs.rs/parking_lot/latest/parking_lot/type.RwLock.html: + // "This lock uses a task-fair locking policy which avoids both reader and writer starvation. + // This means that readers trying to acquire the lock will block even if the lock is unlocked + // when there are writers waiting to acquire the lock. + // Because of this, attempts to recursively acquire a read lock within a single thread may result in a deadlock." + shared.file.as_ref().unwrap().clone() + }; begin_read_wal_frame( - file, + &file, offset + WAL_FRAME_HEADER_SIZE as u64, buffer_pool, complete, diff --git a/core/translate/alter.rs b/core/translate/alter.rs index 6a764808d..e3ecf43ea 100644 --- a/core/translate/alter.rs +++ b/core/translate/alter.rs @@ -198,6 +198,7 @@ pub fn translate_alter_table( } } + // TODO: All quoted ids will be quoted with `[]`, we should store some info from the parsed AST btree.columns.push(column.clone()); let sql = btree.to_sql(); diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index cad89e0ef..8b1e5a176 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -185,7 +185,7 @@ pub enum OperationMode { DELETE, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] /// Sqlite always considers Read transactions implicit pub enum TransactionMode { None, diff --git a/core/translate/view.rs b/core/translate/view.rs index afcef3331..f89f29817 100644 --- a/core/translate/view.rs +++ b/core/translate/view.rs @@ -2,7 +2,7 @@ use crate::schema::{Schema, DBSP_TABLE_PREFIX}; use crate::storage::pager::CreateBTreeFlags; use crate::translate::emitter::Resolver; use crate::translate::schema::{emit_schema_entry, SchemaEntryType, SQLITE_TABLEID}; -use crate::util::normalize_ident; +use crate::util::{normalize_ident, PRIMARY_KEY_AUTOMATIC_INDEX_NAME_PREFIX}; use crate::vdbe::builder::{CursorType, ProgramBuilder}; use crate::vdbe::insn::{CmpInsFlags, Cookie, Insn, RegisterOrLiteral}; use crate::{Connection, Result, SymbolTable}; @@ -141,7 +141,20 @@ pub fn translate_create_materialized_view( // Add the DBSP state table to sqlite_master (required for materialized views) let dbsp_table_name = format!("{DBSP_TABLE_PREFIX}{normalized_view_name}"); - let dbsp_sql = format!("CREATE TABLE {dbsp_table_name} (key INTEGER PRIMARY KEY, state BLOB)"); + // The element_id column uses SQLite's dynamic typing system to store different value types: + // - For hash-based operators (joins, filters): stores INTEGER hash values or rowids + // - For future MIN/MAX operators: stores the actual values being compared (INTEGER, REAL, TEXT, BLOB) + // SQLite's type affinity and sorting rules ensure correct ordering within each operator's data + let dbsp_sql = format!( + "CREATE TABLE {dbsp_table_name} (\ + operator_id INTEGER NOT NULL, \ + zset_id INTEGER NOT NULL, \ + element_id NOT NULL, \ + value BLOB, \ + weight INTEGER NOT NULL, \ + PRIMARY KEY (operator_id, zset_id, element_id)\ + )" + ); emit_schema_entry( &mut program, @@ -155,11 +168,37 @@ pub fn translate_create_materialized_view( Some(dbsp_sql), )?; + // Create automatic primary key index for the DBSP table + // Since the table has PRIMARY KEY (operator_id, zset_id, element_id), we need an index + let dbsp_index_root_reg = program.alloc_register(); + program.emit_insn(Insn::CreateBtree { + db: 0, + root: dbsp_index_root_reg, + flags: CreateBTreeFlags::new_index(), + }); + + // Register the index in sqlite_schema + let dbsp_index_name = format!( + "{}{}_1", + PRIMARY_KEY_AUTOMATIC_INDEX_NAME_PREFIX, &dbsp_table_name + ); + emit_schema_entry( + &mut program, + &resolver, + sqlite_schema_cursor_id, + None, // cdc_table_cursor_id + SchemaEntryType::Index, + &dbsp_index_name, + &dbsp_table_name, + dbsp_index_root_reg, + None, // Automatic indexes don't store SQL + )?; + // Parse schema to load the new view and DBSP state table program.emit_insn(Insn::ParseSchema { db: sqlite_schema_cursor_id, where_clause: Some(format!( - "name = '{normalized_view_name}' OR name = '{dbsp_table_name}'" + "name = '{normalized_view_name}' OR name = '{dbsp_table_name}' OR name = '{dbsp_index_name}'" )), }); diff --git a/core/util.rs b/core/util.rs index a40ea12d4..1c4217a66 100644 --- a/core/util.rs +++ b/core/util.rs @@ -1,6 +1,7 @@ #![allow(unused)] use crate::incremental::view::IncrementalView; use crate::numeric::StrToF64; +use crate::translate::emitter::TransactionMode; use crate::translate::expr::WalkControl; use crate::types::IOResult; use crate::{ @@ -150,10 +151,10 @@ pub fn parse_schema_rows( mut rows: Statement, schema: &mut Schema, syms: &SymbolTable, - mv_tx_id: Option, + mv_tx: Option<(u64, TransactionMode)>, mut existing_views: HashMap>>, ) -> Result<()> { - rows.set_mv_tx_id(mv_tx_id); + rows.set_mv_tx(mv_tx); // TODO: if we IO, this unparsed indexes is lost. Will probably need some state between // IO runs let mut from_sql_indexes = Vec::with_capacity(10); @@ -162,6 +163,9 @@ pub fn parse_schema_rows( // Store DBSP state table root pages: view_name -> dbsp_state_root_page let mut dbsp_state_roots: std::collections::HashMap = std::collections::HashMap::new(); + // Store DBSP state table index root pages: view_name -> dbsp_state_index_root_page + let mut dbsp_state_index_roots: std::collections::HashMap = + std::collections::HashMap::new(); // Store materialized view info (SQL and root page) for later creation let mut materialized_view_info: std::collections::HashMap = std::collections::HashMap::new(); @@ -184,8 +188,9 @@ pub fn parse_schema_rows( &mut from_sql_indexes, &mut automatic_indices, &mut dbsp_state_roots, + &mut dbsp_state_index_roots, &mut materialized_view_info, - )?; + )? } StepResult::IO => { // TODO: How do we ensure that the I/O we submitted to @@ -199,7 +204,11 @@ pub fn parse_schema_rows( } schema.populate_indices(from_sql_indexes, automatic_indices)?; - schema.populate_materialized_views(materialized_view_info, dbsp_state_roots)?; + schema.populate_materialized_views( + materialized_view_info, + dbsp_state_roots, + dbsp_state_index_roots, + )?; Ok(()) } diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index ea51ec8e9..a6d3e5a58 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -941,8 +941,8 @@ pub fn op_open_read( let pager = program.get_pager_from_database_index(db); let (_, cursor_type) = program.cursor_ref.get(*cursor_id).unwrap(); - let mv_cursor = match program.connection.mv_tx_id.get() { - Some(tx_id) => { + let mv_cursor = match program.connection.mv_tx.get() { + Some((tx_id, _)) => { let table_id = *root_page as u64; let mv_store = mv_store.unwrap().clone(); let mv_cursor = Rc::new(RefCell::new( @@ -2156,7 +2156,7 @@ pub fn op_transaction( // In MVCC we don't have write exclusivity, therefore we just need to start a transaction if needed. // Programs can run Transaction twice, first with read flag and then with write flag. So a single txid is enough // for both. - if program.connection.mv_tx_id.get().is_none() { + if program.connection.mv_tx.get().is_none() { // We allocate the first page lazily in the first transaction. return_if_io!(pager.maybe_allocate_page1()); // TODO: when we fix MVCC enable schema cookie detection for reprepare statements @@ -2168,23 +2168,31 @@ pub fn op_transaction( // } let tx_id = match tx_mode { TransactionMode::None | TransactionMode::Read | TransactionMode::Concurrent => { - mv_store.begin_tx(pager.clone()) + mv_store.begin_tx(pager.clone())? } TransactionMode::Write => { - return_if_io!(mv_store.begin_exclusive_tx(pager.clone())) + return_if_io!(mv_store.begin_exclusive_tx(pager.clone(), None)) } }; - conn.mv_transactions.borrow_mut().push(tx_id); - program.connection.mv_tx_id.set(Some(tx_id)); - } else if updated - && matches!(new_transaction_state, TransactionState::Write { .. }) - && matches!(tx_mode, TransactionMode::Write) - { - // For MVCC with concurrent transactions, we don't need to upgrade to exclusive. - // The existing MVCC transaction can handle both reads and writes. - // We only upgrade to exclusive for IMMEDIATE/EXCLUSIVE transaction modes. - // Since we already have an MVCC transaction from BEGIN CONCURRENT, - // we can just continue using it for writes. + program.connection.mv_tx.set(Some((tx_id, *tx_mode))); + } else if updated { + // TODO: fix tx_mode in Insn::Transaction, now each statement overrides it even if there's already a CONCURRENT Tx in progress, for example + let mv_tx_mode = program.connection.mv_tx.get().unwrap().1; + let actual_tx_mode = if mv_tx_mode == TransactionMode::Concurrent { + TransactionMode::Concurrent + } else { + *tx_mode + }; + if matches!(new_transaction_state, TransactionState::Write { .. }) + && matches!(actual_tx_mode, TransactionMode::Write) + { + let (tx_id, mv_tx_mode) = program.connection.mv_tx.get().unwrap(); + if mv_tx_mode == TransactionMode::Read { + return_if_io!(mv_store.upgrade_to_exclusive_tx(pager.clone(), Some(tx_id))); + } else { + return_if_io!(mv_store.begin_exclusive_tx(pager.clone(), Some(tx_id))); + } + } } } else { if matches!(tx_mode, TransactionMode::Concurrent) { @@ -2292,14 +2300,20 @@ pub fn op_auto_commit( if *auto_commit != conn.auto_commit.get() { if *rollback { // TODO(pere): add rollback I/O logic once we implement rollback journal - return_if_io!(pager.end_tx(true, &conn)); + if let Some(mv_store) = mv_store { + if let Some((tx_id, _)) = conn.mv_tx.get() { + mv_store.rollback_tx(tx_id, pager.clone(), &conn)?; + } + } else { + return_if_io!(pager.end_tx(true, &conn)); + } conn.transaction_state.replace(TransactionState::None); conn.auto_commit.replace(true); } else { conn.auto_commit.replace(*auto_commit); } } else { - let mvcc_tx_active = program.connection.mv_tx_id.get().is_some(); + let mvcc_tx_active = program.connection.mv_tx.get().is_some(); if !mvcc_tx_active { if !*auto_commit { return Err(LimboError::TxError( @@ -6374,8 +6388,8 @@ pub fn op_open_write( CursorType::BTreeIndex(index) => Some(index), _ => None, }; - let mv_cursor = match program.connection.mv_tx_id.get() { - Some(tx_id) => { + let mv_cursor = match program.connection.mv_tx.get() { + Some((tx_id, _)) => { let table_id = root_page; let mv_store = mv_store.unwrap().clone(); let mv_cursor = Rc::new(RefCell::new( @@ -6649,7 +6663,7 @@ pub fn op_parse_schema( stmt, schema, &conn.syms.borrow(), - program.connection.mv_tx_id.get(), + program.connection.mv_tx.get(), existing_views, ) }) @@ -6664,7 +6678,7 @@ pub fn op_parse_schema( stmt, schema, &conn.syms.borrow(), - program.connection.mv_tx_id.get(), + program.connection.mv_tx.get(), existing_views, ) }) @@ -7120,8 +7134,8 @@ pub fn op_open_ephemeral( let root_page = return_if_io!(pager.btree_create(flag)); let (_, cursor_type) = program.cursor_ref.get(cursor_id).unwrap(); - let mv_cursor = match program.connection.mv_tx_id.get() { - Some(tx_id) => { + let mv_cursor = match program.connection.mv_tx.get() { + Some((tx_id, _)) => { let table_id = root_page as u64; let mv_store = mv_store.unwrap().clone(); let mv_cursor = Rc::new(RefCell::new( diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 52b7bf080..0606f2e08 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -820,17 +820,11 @@ impl Program { let auto_commit = conn.auto_commit.get(); if auto_commit { // FIXME: we don't want to commit stuff from other programs. - let mut mv_transactions = conn.mv_transactions.borrow_mut(); if matches!(program_state.commit_state, CommitState::Ready) { - assert!( - mv_transactions.len() <= 1, - "for now we only support one mv transaction in single connection, {mv_transactions:?}", - ); - if mv_transactions.is_empty() { + let Some((tx_id, _)) = conn.mv_tx.get() else { return Ok(IOResult::Done(())); - } - let tx_id = mv_transactions.first().unwrap(); - let state_machine = mv_store.commit_tx(*tx_id, pager.clone(), &conn).unwrap(); + }; + let state_machine = mv_store.commit_tx(tx_id, pager.clone(), &conn).unwrap(); program_state.commit_state = CommitState::CommitingMvcc { state_machine }; } let CommitState::CommitingMvcc { state_machine } = &mut program_state.commit_state @@ -840,10 +834,9 @@ impl Program { match self.step_end_mvcc_txn(state_machine, mv_store)? { IOResult::Done(_) => { assert!(state_machine.is_finalized()); - conn.mv_tx_id.set(None); + conn.mv_tx.set(None); conn.transaction_state.replace(TransactionState::None); program_state.commit_state = CommitState::Ready; - mv_transactions.clear(); return Ok(IOResult::Done(())); } IOResult::IO(io) => { @@ -1082,10 +1075,14 @@ pub fn handle_program_error( LimboError::TxError(_) => {} // Table locked errors, e.g. trying to checkpoint in an interactive transaction, do not cause a rollback. LimboError::TableLocked => {} + // Busy errors do not cause a rollback. + LimboError::Busy => {} _ => { if let Some(mv_store) = mv_store { - if let Some(tx_id) = connection.mv_tx_id.get() { - mv_store.rollback_tx(tx_id, pager.clone()); + if let Some((tx_id, _)) = connection.mv_tx.get() { + connection.transaction_state.replace(TransactionState::None); + connection.auto_commit.replace(true); + mv_store.rollback_tx(tx_id, pager.clone(), connection)?; } } else { pager diff --git a/extensions/core/Cargo.toml b/extensions/core/Cargo.toml index c18a08d29..ecc4581fa 100644 --- a/extensions/core/Cargo.toml +++ b/extensions/core/Cargo.toml @@ -16,4 +16,4 @@ static = [] turso_macros = { workspace = true } getrandom = "0.3.1" -chrono = "0.4.40" +chrono = { workspace = true, default-features = true } diff --git a/extensions/csv/Cargo.toml b/extensions/csv/Cargo.toml index d182deda9..235bf4d44 100644 --- a/extensions/csv/Cargo.toml +++ b/extensions/csv/Cargo.toml @@ -18,7 +18,7 @@ turso_ext = { workspace = true, features = ["static"] } csv = "1.3.1" [dev-dependencies] -tempfile = "3.19.1" +tempfile = { workspace = true } [target.'cfg(not(target_family = "wasm"))'.dependencies] mimalloc = { version = "0.1", default-features = false } diff --git a/extensions/regexp/Cargo.toml b/extensions/regexp/Cargo.toml index 699416ff7..eb2308fce 100644 --- a/extensions/regexp/Cargo.toml +++ b/extensions/regexp/Cargo.toml @@ -17,7 +17,7 @@ crate-type = ["cdylib", "lib"] [dependencies] turso_ext = { workspace = true, features = ["static"] } -regex = "1.11.1" +regex = { workspace = true } [target.'cfg(not(target_family = "wasm"))'.dependencies] mimalloc = { version = "0.1", default-features = false } diff --git a/extensions/tests/Cargo.toml b/extensions/tests/Cargo.toml index dca601b0c..2d123ed7a 100644 --- a/extensions/tests/Cargo.toml +++ b/extensions/tests/Cargo.toml @@ -14,7 +14,7 @@ crate-type = ["cdylib", "lib"] static= [ "turso_ext/static" ] [dependencies] -env_logger = "0.11.6" +env_logger = { workspace = true } lazy_static = "1.5.0" turso_ext = { workspace = true, features = ["static", "vfs"] } log = "0.4.26" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index d4768d2f6..6f9720bc8 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -15,17 +15,17 @@ default = [] serde = ["dep:serde", "bitflags/serde"] [dependencies] -bitflags = "2.0" -miette = "7.4.0" +bitflags = { workspace = true } +miette = { workspace = true } strum = { workspace = true } strum_macros = {workspace = true } serde = { workspace = true , optional = true, features = ["derive"] } -thiserror = "1.0.61" +thiserror = { workspace = true } turso_macros = { workspace = true } [dev-dependencies] -fallible-iterator = "0.3" -criterion = { version = "0.5", features = ["html_reports" ] } +fallible-iterator = { workspace = true } +criterion = { workspace = true, features = ["html_reports" ] } [target.'cfg(not(target_family = "windows"))'.dev-dependencies] pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] } diff --git a/perf/throughput/rusqlite/Cargo.lock b/perf/throughput/rusqlite/Cargo.lock deleted file mode 100644 index 2bb86369d..000000000 --- a/perf/throughput/rusqlite/Cargo.lock +++ /dev/null @@ -1,403 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - -[[package]] -name = "anstream" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" - -[[package]] -name = "anstyle-parse" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys", -] - -[[package]] -name = "bitflags" -version = "2.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" - -[[package]] -name = "cc" -version = "1.2.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" - -[[package]] -name = "clap" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" - -[[package]] -name = "colorchoice" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" - -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] -name = "find-msvc-tools" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" - -[[package]] -name = "hashbrown" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" -dependencies = [ - "hashbrown", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "libsqlite3-sys" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "proc-macro2" -version = "1.0.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "rusqlite" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "unicode-ident" -version = "1.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.53.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - -[[package]] -name = "write-throughput" -version = "0.1.0" -dependencies = [ - "clap", - "rusqlite", -] - -[[package]] -name = "zerocopy" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/perf/throughput/rusqlite/Cargo.toml b/perf/throughput/rusqlite/Cargo.toml index 1ffa56b3a..9fb484194 100644 --- a/perf/throughput/rusqlite/Cargo.toml +++ b/perf/throughput/rusqlite/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "write-throughput" +name = "write-throughput-sqlite" version = "0.1.0" edition = "2021" [[bin]] -name = "write-throughput" +name = "write-throughput-sqlite" path = "src/main.rs" [dependencies] -rusqlite = { version = "0.31", features = ["bundled"] } -clap = { version = "4.0", features = ["derive"] } \ No newline at end of file +rusqlite = { workspace = true } +clap = { workspace = true, features = ["derive"] } \ No newline at end of file diff --git a/perf/throughput/rusqlite/src/main.rs b/perf/throughput/rusqlite/src/main.rs index d13bf958a..1dd518270 100644 --- a/perf/throughput/rusqlite/src/main.rs +++ b/perf/throughput/rusqlite/src/main.rs @@ -2,7 +2,7 @@ use clap::Parser; use rusqlite::{Connection, Result}; use std::sync::{Arc, Barrier}; use std::thread; -use std::time::{Duration, Instant}; +use std::time::Instant; #[derive(Parser)] #[command(name = "write-throughput")] @@ -73,7 +73,7 @@ fn main() -> Result<()> { match handle.join() { Ok(Ok(inserts)) => total_inserts += inserts, Ok(Err(e)) => { - eprintln!("Thread error: {}", e); + eprintln!("Thread error: {e}"); return Err(e); } Err(_) => { @@ -87,9 +87,9 @@ fn main() -> Result<()> { let overall_throughput = (total_inserts as f64) / overall_elapsed.as_secs_f64(); println!("\n=== BENCHMARK RESULTS ==="); - println!("Total inserts: {}", total_inserts); + println!("Total inserts: {total_inserts}",); println!("Total time: {:.2}s", overall_elapsed.as_secs_f64()); - println!("Overall throughput: {:.2} inserts/sec", overall_throughput); + println!("Overall throughput: {overall_throughput:.2} inserts/sec"); println!("Threads: {}", args.threads); println!("Batch size: {}", args.batch_size); println!("Iterations per thread: {}", args.iterations); @@ -116,7 +116,7 @@ fn setup_database(db_path: &str) -> Result { [], )?; - println!("Database created at: {}", db_path); + println!("Database created at: {db_path}"); Ok(conn) } @@ -144,7 +144,7 @@ fn worker_thread( for i in 0..batch_size { let id = thread_id * iterations * batch_size + iteration * batch_size + i; - stmt.execute([&id.to_string(), &format!("data_{}", id)])?; + stmt.execute([&id.to_string(), &format!("data_{id}")])?; total_inserts += 1; } if think_ms > 0 { diff --git a/perf/throughput/turso/Cargo.lock b/perf/throughput/turso/Cargo.lock deleted file mode 100644 index d49ae8488..000000000 --- a/perf/throughput/turso/Cargo.lock +++ /dev/null @@ -1,2066 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common", - "generic-array", -] - -[[package]] -name = "aegis" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a1c2f54793fee13c334f70557d3bd6a029a9d453ebffd82ba571d139064da8" -dependencies = [ - "cc", - "softaes", -] - -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - -[[package]] -name = "aes-gcm" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anstream" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" - -[[package]] -name = "anstyle-parse" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" -dependencies = [ - "windows-sys 0.60.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.60.2", -] - -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - -[[package]] -name = "bitflags" -version = "2.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" - -[[package]] -name = "built" -version = "0.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" -dependencies = [ - "chrono", - "git2", -] - -[[package]] -name = "bumpalo" -version = "3.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - -[[package]] -name = "bytemuck" -version = "1.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f154e572231cb6ba2bd1176980827e3d5dc04cc183a75dea38109fbdd672d29" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" - -[[package]] -name = "cc" -version = "1.2.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" - -[[package]] -name = "cfg_block" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18758054972164c3264f7c8386f5fc6da6114cb46b619fd365d4e3b2dc3ae487" - -[[package]] -name = "chrono" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" -dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link 0.2.0", -] - -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - -[[package]] -name = "clap" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" - -[[package]] -name = "colorchoice" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-skiplist" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "rand_core 0.6.4", - "typenum", -] - -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.60.2", -] - -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "find-msvc-tools" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-executor" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi 0.14.5+wasi-0.2.4", -] - -[[package]] -name = "ghash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" -dependencies = [ - "opaque-debug", - "polyval", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "git2" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2deb07a133b1520dc1a5690e9bd08950108873d7ed5de38dcc74d3b5ebffa110" -dependencies = [ - "bitflags", - "libc", - "libgit2-sys", - "log", - "url", -] - -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "iana-time-zone" -version = "0.1.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "icu_collections" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" -dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" - -[[package]] -name = "icu_properties" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "potential_utf", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" - -[[package]] -name = "icu_provider" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" -dependencies = [ - "displaydoc", - "icu_locale_core", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921" -dependencies = [ - "equivalent", - "hashbrown", -] - -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags", - "cfg-if", - "libc", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.3", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.78" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "julian_day_converter" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2987f71b89b85c812c8484cbf0c5d7912589e77bfdc66fd3e52f760e7859f16" -dependencies = [ - "chrono", -] - -[[package]] -name = "libc" -version = "0.2.175" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" - -[[package]] -name = "libgit2-sys" -version = "0.18.2+1.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c42fe03df2bd3c53a3a9c7317ad91d80c81cd1fb0caec8d7cc4cd2bfa10c222" -dependencies = [ - "cc", - "libc", - "libz-sys", - "pkg-config", -] - -[[package]] -name = "libloading" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" -dependencies = [ - "cfg-if", - "windows-targets 0.53.3", -] - -[[package]] -name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - -[[package]] -name = "libz-sys" -version = "1.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linux-raw-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" - -[[package]] -name = "litemap" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" - -[[package]] -name = "lock_api" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" - -[[package]] -name = "memchr" -version = "2.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" - -[[package]] -name = "miette" -version = "7.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" -dependencies = [ - "cfg-if", - "miette-derive", - "unicode-width", -] - -[[package]] -name = "miette-derive" -version = "7.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" - -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - -[[package]] -name = "pack1" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6e7cd9bd638dc2c831519a0caa1c006cab771a92b1303403a8322773c5b72d6" -dependencies = [ - "bytemuck", -] - -[[package]] -name = "parking_lot" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.52.6", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "polling" -version = "3.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5bd19146350fe804f7cb2669c851c03d69da628803dab0d98018142aaa5d829" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi", - "pin-project-lite", - "rustix", - "windows-sys 0.60.2", -] - -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if", - "cpufeatures", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "potential_utf" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" -dependencies = [ - "zerovec", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.3", -] - -[[package]] -name = "redox_syscall" -version = "0.5.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex" -version = "1.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" - -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - -[[package]] -name = "rustix" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.60.2", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "serde" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" -dependencies = [ - "libc", -] - -[[package]] -name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "socket2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "softaes" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef461faaeb36c340b6c887167a9054a034f6acfc50a014ead26a02b4356b3de" - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn", -] - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "syn" -version = "2.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tempfile" -version = "3.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84fa4d11fadde498443cca10fd3ac23c951f0dc59e080e9f4b93d4df4e4eea53" -dependencies = [ - "fastrand", - "getrandom 0.3.3", - "once_cell", - "rustix", - "windows-sys 0.60.2", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" -dependencies = [ - "thiserror-impl 2.0.16", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tinystr" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tokio" -version = "1.47.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" -dependencies = [ - "backtrace", - "bytes", - "io-uring", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "slab", - "socket2", - "tokio-macros", - "windows-sys 0.59.0", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" -dependencies = [ - "once_cell", -] - -[[package]] -name = "turso" -version = "0.2.0-pre.3" -dependencies = [ - "thiserror 2.0.16", - "turso_core", -] - -[[package]] -name = "turso_core" -version = "0.2.0-pre.3" -dependencies = [ - "aegis", - "aes", - "aes-gcm", - "bitflags", - "built", - "bytemuck", - "cfg_block", - "chrono", - "crossbeam-skiplist", - "fallible-iterator", - "getrandom 0.2.16", - "hex", - "io-uring", - "julian_day_converter", - "libc", - "libloading", - "libm", - "miette", - "pack1", - "parking_lot", - "paste", - "polling", - "rand 0.8.5", - "regex", - "regex-syntax", - "rustix", - "ryu", - "strum", - "strum_macros", - "tempfile", - "thiserror 1.0.69", - "tracing", - "turso_ext", - "turso_macros", - "turso_parser", - "turso_sqlite3_parser", - "twox-hash", - "uncased", - "uuid", -] - -[[package]] -name = "turso_ext" -version = "0.2.0-pre.3" -dependencies = [ - "chrono", - "getrandom 0.3.3", - "turso_macros", -] - -[[package]] -name = "turso_macros" -version = "0.2.0-pre.3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "turso_parser" -version = "0.2.0-pre.3" -dependencies = [ - "bitflags", - "miette", - "strum", - "strum_macros", - "thiserror 1.0.69", - "turso_macros", -] - -[[package]] -name = "turso_sqlite3_parser" -version = "0.2.0-pre.3" -dependencies = [ - "bitflags", - "cc", - "fallible-iterator", - "indexmap", - "log", - "memchr", - "miette", - "smallvec", - "strum", - "strum_macros", -] - -[[package]] -name = "twox-hash" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -dependencies = [ - "rand 0.9.2", -] - -[[package]] -name = "typenum" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" - -[[package]] -name = "uncased" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" -dependencies = [ - "version_check", -] - -[[package]] -name = "unicode-ident" -version = "1.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" - -[[package]] -name = "unicode-width" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" - -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common", - "subtle", -] - -[[package]] -name = "url" -version = "2.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "uuid" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" -dependencies = [ - "getrandom 0.3.3", - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasi" -version = "0.14.5+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4" -dependencies = [ - "wasip2", -] - -[[package]] -name = "wasip2" -version = "1.0.0+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link 0.1.3", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - -[[package]] -name = "windows-link" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link 0.1.3", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link 0.1.3", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.3", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" -dependencies = [ - "windows-link 0.1.3", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - -[[package]] -name = "wit-bindgen" -version = "0.45.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" - -[[package]] -name = "write-throughput" -version = "0.1.0" -dependencies = [ - "clap", - "futures", - "tokio", - "turso", -] - -[[package]] -name = "writeable" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" - -[[package]] -name = "yoke" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerotrie" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/perf/throughput/turso/Cargo.toml b/perf/throughput/turso/Cargo.toml index 7a6eb65cf..bb87ab767 100644 --- a/perf/throughput/turso/Cargo.toml +++ b/perf/throughput/turso/Cargo.toml @@ -8,7 +8,8 @@ name = "write-throughput" path = "src/main.rs" [dependencies] -turso = { path = "../../../bindings/rust" } -clap = { version = "4.0", features = ["derive"] } -tokio = { version = "1.0", features = ["full"] } -futures = "0.3" \ No newline at end of file +turso = { workspace = true } +clap = { workspace = true, features = ["derive"] } +tokio = { workspace = true, default-features = true, features = ["full"] } +futures = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/perf/throughput/turso/src/main.rs b/perf/throughput/turso/src/main.rs index fd5c5761c..61bd35ed0 100644 --- a/perf/throughput/turso/src/main.rs +++ b/perf/throughput/turso/src/main.rs @@ -1,7 +1,7 @@ use clap::{Parser, ValueEnum}; -use std::sync::{Arc, Barrier}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Instant; +use std::sync::{Arc, Barrier}; +use std::time::{Duration, Instant}; use turso::{Builder, Database, Result}; #[derive(Debug, Clone, Copy, ValueEnum)] @@ -33,10 +33,18 @@ struct Args { help = "Per transaction think time (ms)" )] think: u64, + + #[arg( + long = "timeout", + default_value = "30000", + help = "Busy timeout in milliseconds" + )] + timeout: u64, } #[tokio::main] async fn main() -> Result<()> { + let _ = tracing_subscriber::fmt::try_init(); let args = Args::parse(); println!( @@ -58,6 +66,8 @@ async fn main() -> Result<()> { let start_barrier = Arc::new(Barrier::new(args.threads)); let mut handles = Vec::new(); + let timeout = Duration::from_millis(args.timeout); + let overall_start = Instant::now(); for thread_id in 0..args.threads { @@ -72,17 +82,18 @@ async fn main() -> Result<()> { barrier, args.mode, args.think, + timeout, )); handles.push(handle); } let mut total_inserts = 0; - for handle in handles { + for (idx, handle) in handles.into_iter().enumerate() { match handle.await { Ok(Ok(inserts)) => total_inserts += inserts, Ok(Err(e)) => { - eprintln!("Thread error: {}", e); + eprintln!("Thread error {idx}: {e}"); return Err(e); } Err(_) => { @@ -96,9 +107,9 @@ async fn main() -> Result<()> { let overall_throughput = (total_inserts as f64) / overall_elapsed.as_secs_f64(); println!("\n=== BENCHMARK RESULTS ==="); - println!("Total inserts: {}", total_inserts); + println!("Total inserts: {total_inserts}"); println!("Total time: {:.2}s", overall_elapsed.as_secs_f64()); - println!("Overall throughput: {:.2} inserts/sec", overall_throughput); + println!("Overall throughput: {overall_throughput:.2} inserts/sec"); println!("Threads: {}", args.threads); println!("Batch size: {}", args.batch_size); println!("Iterations per thread: {}", args.iterations); @@ -133,10 +144,11 @@ async fn setup_database(db_path: &str, mode: TransactionMode) -> Result, mode: TransactionMode, think_ms: u64, + timeout: Duration, ) -> Result { start_barrier.wait(); @@ -155,6 +168,7 @@ async fn worker_thread( for iteration in 0..iterations { let conn = db.connect()?; + conn.busy_timeout(Some(timeout))?; let total_inserts = Arc::clone(&total_inserts); let tx_fut = async move { let mut stmt = conn @@ -171,7 +185,7 @@ async fn worker_thread( let id = thread_id * iterations * batch_size + iteration * batch_size + i; stmt.execute(turso::params::Params::Positional(vec![ turso::Value::Integer(id as i64), - turso::Value::Text(format!("data_{}", id)), + turso::Value::Text(format!("data_{id}")), ])) .await?; total_inserts.fetch_add(1, Ordering::Relaxed); diff --git a/simulator-docker-runner/Dockerfile.simulator b/simulator-docker-runner/Dockerfile.simulator index 6c3bceb57..d819b14e1 100644 --- a/simulator-docker-runner/Dockerfile.simulator +++ b/simulator-docker-runner/Dockerfile.simulator @@ -19,6 +19,7 @@ COPY extensions ./extensions/ COPY macros ./macros/ COPY sync ./sync COPY parser ./parser/ +COPY perf/throughput/turso ./perf/throughput/turso COPY vendored ./vendored/ COPY cli ./cli/ COPY sqlite3 ./sqlite3/ @@ -43,6 +44,7 @@ COPY --from=planner /app/vendored ./vendored/ COPY --from=planner /app/extensions ./extensions/ COPY --from=planner /app/macros ./macros/ COPY --from=planner /app/parser ./parser/ +COPY --from=planner /app/perf/throughput/turso ./perf/throughput/turso COPY --from=planner /app/simulator ./simulator/ COPY --from=planner /app/packages ./packages/ COPY --from=planner /app/sql_generation ./sql_generation/ diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index 89ee9634b..9ea6d093e 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -15,27 +15,27 @@ name = "limbo_sim" path = "main.rs" [dependencies] -turso_core = { path = "../core", features = ["simulator"]} +turso_core = { workspace = true, features = ["simulator"]} rand = { workspace = true } -rand_chacha = "0.9.0" +rand_chacha = { workspace = true } log = "0.4.20" -env_logger = "0.10.1" -regex = "1.11.1" -regex-syntax = { version = "0.8.5", default-features = false, features = [ +env_logger = { workspace = true } +regex = { workspace = true } +regex-syntax = { workspace = true, default-features = false, features = [ "unicode", ] } -clap = { version = "4.5", features = ["derive"] } +clap = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } notify = "8.0.0" rusqlite.workspace = true dirs = "6.0.0" -chrono = { version = "0.4.40", features = ["serde"] } +chrono = { workspace = true, default-features = true, features = ["serde"] } tracing = { workspace = true } -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +tracing-subscriber = { workspace = true, features = ["env-filter"] } anyhow.workspace = true -hex = "0.4.3" -itertools = "0.14.0" +hex = { workspace = true } +itertools = { workspace = true } sql_generation = { workspace = true } turso_parser = { workspace = true } schemars = { workspace = true } @@ -43,4 +43,4 @@ garde = { workspace = true, features = ["derive", "serde"] } json5 = { version = "0.4.1" } strum = { workspace = true } parking_lot = { workspace = true } -indexmap = "2.10.0" +indexmap = { workspace = true } diff --git a/sql_generation/Cargo.toml b/sql_generation/Cargo.toml index 0d8b6a097..d42668237 100644 --- a/sql_generation/Cargo.toml +++ b/sql_generation/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true path = "lib.rs" [dependencies] -hex = "0.4.3" +hex = { workspace = true } serde = { workspace = true, features = ["derive"] } turso_core = { workspace = true, features = ["simulator"] } turso_parser = { workspace = true, features = ["serde"] } @@ -21,7 +21,7 @@ anyhow = { workspace = true } tracing = { workspace = true } schemars = { workspace = true } garde = { workspace = true, features = ["derive", "serde"] } -indexmap = { version = "2.11.0" } +indexmap = { workspace = true } [dev-dependencies] -rand_chacha = "0.9.0" +rand_chacha = { workspace = true } diff --git a/sqlite3/Cargo.toml b/sqlite3/Cargo.toml index 76e397141..d4bc80d34 100644 --- a/sqlite3/Cargo.toml +++ b/sqlite3/Cargo.toml @@ -22,15 +22,15 @@ doc = false crate-type = ["lib", "cdylib", "staticlib"] [dependencies] -env_logger = { version = "0.11.3", default-features = false } +env_logger = { workspace = true, default-features = false } libc = "0.2.169" -turso_core = { path = "../core", features = ["conn_raw_api"] } -tracing = "0.1.41" -tracing-appender = "0.2.3" -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +turso_core = { workspace = true, features = ["conn_raw_api"] } +tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } [dev-dependencies] -tempfile = "3.8.0" +tempfile = { workspace = true } [package.metadata.capi.header] name = "sqlite3.h" diff --git a/stress/Cargo.toml b/stress/Cargo.toml index b667e773d..2e51f003c 100644 --- a/stress/Cargo.toml +++ b/stress/Cargo.toml @@ -21,12 +21,12 @@ experimental_indexes = [] [dependencies] anarchist-readable-name-generator-lib = "0.1.0" -antithesis_sdk = "0.2.5" -clap = { version = "4.5", features = ["derive"] } -hex = "0.4" -tempfile = "3.20.0" -tokio = { version = "1.29.1", features = ["full"] } -tracing = "0.1.41" -tracing-appender = "0.2.3" -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } -turso = { path = "../bindings/rust" } +antithesis_sdk = { workspace = true } +clap = { workspace = true, features = ["derive"] } +hex = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +turso = { workspace = true } diff --git a/stress/main.rs b/stress/main.rs index 8178e8e13..0282c393d 100644 --- a/stress/main.rs +++ b/stress/main.rs @@ -488,6 +488,8 @@ async fn main() -> Result<(), Box> { let plan = plan.clone(); let conn = db.lock().await.connect()?; + conn.execute("PRAGMA data_sync_retry = 1", ()).await?; + // Apply each DDL statement individually for stmt in &plan.ddl_statements { if opts.verbose { diff --git a/sync/engine/Cargo.toml b/sync/engine/Cargo.toml index 229c60714..89b20b406 100644 --- a/sync/engine/Cargo.toml +++ b/sync/engine/Cargo.toml @@ -22,11 +22,11 @@ roaring = "0.11.2" [dev-dependencies] ctor = "0.4.2" -tempfile = "3.20.0" -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } -tokio = { version = "1.46.1", features = ["macros", "rt-multi-thread", "test-util"] } +tempfile = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } uuid = "1.17.0" -rand = "0.9.2" -rand_chacha = "0.9.0" +rand = { workspace = true } +rand_chacha = { workspace = true } turso = { workspace = true, features = ["conn_raw_api"] } -futures = "0.3.31" +futures = { workspace = true } diff --git a/sync/engine/src/database_sync_engine.rs b/sync/engine/src/database_sync_engine.rs index 7a6607d94..7176d7ede 100644 --- a/sync/engine/src/database_sync_engine.rs +++ b/sync/engine/src/database_sync_engine.rs @@ -246,12 +246,17 @@ impl DatabaseSyncEngine

{ let main_conn = connect_untracked(&self.main_tape)?; let change_id = self.meta().last_pushed_change_id_hint; let last_pull_unix_time = self.meta().last_pull_unix_time; + let revision = self.meta().synced_revision.clone().map(|x| match x { + DatabasePullRevision::Legacy { + generation, + synced_frame_no, + } => format!("generation={generation},synced_frame_no={synced_frame_no:?}"), + DatabasePullRevision::V1 { revision } => revision, + }); let last_push_unix_time = self.meta().last_push_unix_time; let revert_wal_path = &self.revert_db_wal_path; - let revert_wal_file = self - .io - .open_file(revert_wal_path, OpenFlags::all(), false)?; - let revert_wal_size = revert_wal_file.size()?; + let revert_wal_file = self.io.try_open(revert_wal_path)?; + let revert_wal_size = revert_wal_file.map(|f| f.size()).transpose()?.unwrap_or(0); let main_wal_frames = main_conn.wal_state()?.max_frame; let main_wal_size = if main_wal_frames == 0 { 0 @@ -264,6 +269,7 @@ impl DatabaseSyncEngine

{ revert_wal_size, last_pull_unix_time, last_push_unix_time, + revision, }) } @@ -416,7 +422,6 @@ impl DatabaseSyncEngine

{ &mut self, coro: &Coro, remote_changes: DbChangesStatus, - now: turso_core::Instant, ) -> Result<()> { assert!(remote_changes.file_slot.is_some(), "file_slot must be set"); let changes_file = remote_changes.file_slot.as_ref().unwrap().value.clone(); @@ -436,7 +441,7 @@ impl DatabaseSyncEngine

{ m.revert_since_wal_watermark = revert_since_wal_watermark; m.synced_revision = Some(remote_changes.revision); m.last_pushed_change_id_hint = 0; - m.last_pull_unix_time = now.secs; + m.last_pull_unix_time = remote_changes.time.secs; }) .await?; Ok(()) @@ -656,13 +661,12 @@ impl DatabaseSyncEngine

{ } pub async fn pull_changes_from_remote(&mut self, coro: &Coro) -> Result<()> { - let now = self.io.now(); let changes = self.wait_changes_from_remote(coro).await?; if changes.file_slot.is_some() { - self.apply_changes_from_remote(coro, changes, now).await?; + self.apply_changes_from_remote(coro, changes).await?; } else { self.update_meta(coro, |m| { - m.last_pull_unix_time = now.secs; + m.last_pull_unix_time = changes.time.secs; }) .await?; } diff --git a/sync/engine/src/types.rs b/sync/engine/src/types.rs index 8837e35bf..1b78e8cb1 100644 --- a/sync/engine/src/types.rs +++ b/sync/engine/src/types.rs @@ -67,6 +67,7 @@ pub struct SyncEngineStats { pub revert_wal_size: u64, pub last_pull_unix_time: i64, pub last_push_unix_time: Option, + pub revision: Option, } #[derive(Debug, Clone, Copy, PartialEq)] diff --git a/testing/alter_table.test b/testing/alter_table.test index 6c18dff41..e9e83c403 100755 --- a/testing/alter_table.test +++ b/testing/alter_table.test @@ -19,6 +19,14 @@ do_execsql_test_on_specific_db {:memory:} alter-table-rename-column { "CREATE INDEX i ON t (b)" } +do_execsql_test_on_specific_db {:memory:} alter-table-rename-quoted-column { + CREATE TABLE t (a INTEGER); + ALTER TABLE t RENAME a TO "ab cd"; + SELECT sql FROM sqlite_schema; +} { + "CREATE TABLE t (\"ab cd\" INTEGER)" +} + do_execsql_test_on_specific_db {:memory:} alter-table-add-column { CREATE TABLE t (a); INSERT INTO t VALUES (1); @@ -74,6 +82,14 @@ do_execsql_test_on_specific_db {:memory:} alter-table-add-column-default { "0.1|hello" } +do_execsql_test_on_specific_db {:memory:} alter-table-add-quoted-column { + CREATE TABLE test (a); + ALTER TABLE test ADD COLUMN [b c]; + SELECT sql FROM sqlite_schema; +} { + "CREATE TABLE test (a, [b c])" +} + do_execsql_test_on_specific_db {:memory:} alter-table-drop-column { CREATE TABLE t (a, b); INSERT INTO t VALUES (1, 1), (2, 2), (3, 3); diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 6a30e7509..a0cf560c4 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -16,24 +16,24 @@ path = "integration/mod.rs" [dependencies] anyhow.workspace = true -env_logger = "0.10.1" -turso_core = { path = "../core", features = ["conn_raw_api"] } -turso = { path = "../bindings/rust", features = ["conn_raw_api"] } -tokio = { version = "1.47", features = ["full"] } +env_logger = { workspace = true } +turso_core = { workspace = true, features = ["conn_raw_api"] } +turso = { workspace = true, features = ["conn_raw_api"] } +tokio = { workspace = true, features = ["full"] } rusqlite.workspace = true -tempfile = "3.0.7" +tempfile = { workspace = true } log = "0.4.22" assert_cmd = "^2" -rand_chacha = "0.9.0" -rand = "0.9.0" +rand_chacha = { workspace = true } +rand = { workspace = true } zerocopy = "0.8.26" ctor = "0.5.0" twox-hash = "2.1.1" [dev-dependencies] test-log = { version = "0.2.17", features = ["trace"] } -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } -tracing = "0.1.41" +tracing-subscriber = { workspace = true, features = ["env-filter"] } +tracing = { workspace = true } [features] encryption = ["turso_core/encryption"] diff --git a/tests/integration/fuzz_transaction/mod.rs b/tests/integration/fuzz_transaction/mod.rs index 644ee847c..74dad6571 100644 --- a/tests/integration/fuzz_transaction/mod.rs +++ b/tests/integration/fuzz_transaction/mod.rs @@ -1,14 +1,14 @@ use rand::seq::IndexedRandom; use rand::Rng; use rand_chacha::{rand_core::SeedableRng, ChaCha8Rng}; -use std::collections::HashMap; +use std::collections::BTreeMap; use turso::{Builder, Value}; // In-memory representation of the database state #[derive(Debug, Clone, PartialEq)] struct DbRow { id: i64, - other_columns: HashMap, + other_columns: BTreeMap, } impl std::fmt::Display for DbRow { @@ -33,9 +33,9 @@ impl std::fmt::Display for DbRow { #[derive(Debug, Clone)] struct TransactionState { // The schema this transaction can see (snapshot) - schema: HashMap, + schema: BTreeMap, // The rows this transaction can see (snapshot) - visible_rows: HashMap, + visible_rows: BTreeMap, // Pending changes in this transaction pending_changes: Vec, } @@ -55,19 +55,24 @@ struct TableSchema { #[derive(Debug)] struct ShadowDb { // Schema - schema: HashMap, + schema: BTreeMap, // Committed state (what's actually in the database) - committed_rows: HashMap, + committed_rows: BTreeMap, // Transaction states - transactions: HashMap>, + transactions: BTreeMap>, + query_gen_options: QueryGenOptions, } impl ShadowDb { - fn new(initial_schema: HashMap) -> Self { + fn new( + initial_schema: BTreeMap, + query_gen_options: QueryGenOptions, + ) -> Self { Self { schema: initial_schema, - committed_rows: HashMap::new(), - transactions: HashMap::new(), + committed_rows: BTreeMap::new(), + transactions: BTreeMap::new(), + query_gen_options, } } @@ -185,7 +190,7 @@ impl ShadowDb { &mut self, tx_id: usize, id: i64, - other_columns: HashMap, + other_columns: BTreeMap, ) -> Result<(), String> { if let Some(tx_state) = self.transactions.get_mut(&tx_id) { // Check if row exists in visible state @@ -212,7 +217,7 @@ impl ShadowDb { &mut self, tx_id: usize, id: i64, - other_columns: HashMap, + other_columns: BTreeMap, ) -> Result<(), String> { if let Some(tx_state) = self.transactions.get_mut(&tx_id) { // Check if row exists in visible state @@ -388,16 +393,18 @@ impl std::fmt::Display for AlterTableOp { #[derive(Debug, Clone)] enum Operation { - Begin, + Begin { + concurrent: bool, + }, Commit, Rollback, Insert { id: i64, - other_columns: HashMap, + other_columns: BTreeMap, }, Update { id: i64, - other_columns: HashMap, + other_columns: BTreeMap, }, Delete { id: i64, @@ -423,7 +430,9 @@ fn value_to_sql(v: &Value) -> String { impl std::fmt::Display for Operation { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Operation::Begin => write!(f, "BEGIN"), + Operation::Begin { concurrent } => { + write!(f, "BEGIN{}", if *concurrent { " CONCURRENT" } else { "" }) + } Operation::Commit => write!(f, "COMMIT"), Operation::Rollback => write!(f, "ROLLBACK"), Operation::Insert { id, other_columns } => { @@ -477,37 +486,121 @@ fn rng_from_time_or_env() -> (ChaCha8Rng, u64) { /// Verify translation isolation semantics with multiple concurrent connections. /// This test is ignored because it still fails sometimes; unsure if it fails due to a bug in the test or a bug in the implementation. async fn test_multiple_connections_fuzz() { - multiple_connections_fuzz(false).await + multiple_connections_fuzz(FuzzOptions::default()).await } #[tokio::test] #[ignore = "MVCC is currently under development, it is expected to fail"] // Same as test_multiple_connections_fuzz, but with MVCC enabled. async fn test_multiple_connections_fuzz_mvcc() { - multiple_connections_fuzz(true).await + let mvcc_fuzz_options = FuzzOptions { + mvcc_enabled: true, + max_num_connections: 2, + query_gen_options: QueryGenOptions { + weight_begin_deferred: 8, + weight_begin_concurrent: 8, + weight_commit: 8, + weight_rollback: 8, + weight_checkpoint: 0, + weight_ddl: 0, + weight_dml: 76, + dml_gen_options: DmlGenOptions { + weight_insert: 25, + weight_delete: 25, + weight_select: 25, + weight_update: 25, + }, + }, + ..FuzzOptions::default() + }; + multiple_connections_fuzz(mvcc_fuzz_options).await } -async fn multiple_connections_fuzz(mvcc_enabled: bool) { +#[derive(Debug, Clone)] +struct FuzzOptions { + num_iterations: usize, + operations_per_connection: usize, + max_num_connections: usize, + query_gen_options: QueryGenOptions, + mvcc_enabled: bool, +} + +#[derive(Debug, Clone)] +struct QueryGenOptions { + weight_begin_deferred: usize, + weight_begin_concurrent: usize, + weight_commit: usize, + weight_rollback: usize, + weight_checkpoint: usize, + weight_ddl: usize, + weight_dml: usize, + dml_gen_options: DmlGenOptions, +} + +#[derive(Debug, Clone)] +struct DmlGenOptions { + weight_insert: usize, + weight_update: usize, + weight_delete: usize, + weight_select: usize, +} + +impl Default for FuzzOptions { + fn default() -> Self { + Self { + num_iterations: 50, + operations_per_connection: 30, + max_num_connections: 8, + query_gen_options: QueryGenOptions::default(), + mvcc_enabled: false, + } + } +} + +impl Default for QueryGenOptions { + fn default() -> Self { + Self { + weight_begin_deferred: 10, + weight_begin_concurrent: 0, + weight_commit: 10, + weight_rollback: 10, + weight_checkpoint: 5, + weight_ddl: 5, + weight_dml: 55, + dml_gen_options: DmlGenOptions::default(), + } + } +} + +impl Default for DmlGenOptions { + fn default() -> Self { + Self { + weight_insert: 25, + weight_update: 25, + weight_delete: 25, + weight_select: 25, + } + } +} + +async fn multiple_connections_fuzz(opts: FuzzOptions) { let (mut rng, seed) = rng_from_time_or_env(); println!("Multiple connections fuzz test seed: {seed}"); - const NUM_ITERATIONS: usize = 50; - const OPERATIONS_PER_CONNECTION: usize = 30; - const MAX_NUM_CONNECTIONS: usize = 8; - - for iteration in 0..NUM_ITERATIONS { - let num_connections = rng.random_range(2..=MAX_NUM_CONNECTIONS); + for iteration in 0..opts.num_iterations { + let num_connections = rng.random_range(2..=opts.max_num_connections); println!("--- Seed {seed} Iteration {iteration} ---"); + println!("Options: {opts:?}"); // Create a fresh database for each iteration let tempfile = tempfile::NamedTempFile::new().unwrap(); let db = Builder::new_local(tempfile.path().to_str().unwrap()) - .with_mvcc(mvcc_enabled) + .with_mvcc(opts.mvcc_enabled) .build() .await .unwrap(); // SHARED shadow database for all connections - let mut schema = HashMap::new(); + let mut schema = BTreeMap::new(); schema.insert( "test_table".to_string(), TableSchema { @@ -525,7 +618,7 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { ], }, ); - let mut shared_shadow_db = ShadowDb::new(schema); + let mut shared_shadow_db = ShadowDb::new(schema, opts.query_gen_options.clone()); let mut next_tx_id = 0; // Create connections @@ -544,26 +637,67 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { connections.push((conn, conn_id, None::)); // (connection, conn_id, current_tx_id) } + let is_acceptable_error = |e: &turso::Error| -> bool { + let e_string = e.to_string(); + e_string.contains("is locked") + || e_string.contains("busy") + || e_string.contains("Write-write conflict") + }; + let requires_rollback = |e: &turso::Error| -> bool { + let e_string = e.to_string(); + e_string.contains("Write-write conflict") + }; + + let handle_error = |e: &turso::Error, + tx_id: &mut Option, + conn_id: usize, + op_num: usize, + shadow_db: &mut ShadowDb| { + println!("Connection {conn_id}(op={op_num}) FAILED: {e}"); + if requires_rollback(e) { + if let Some(tx_id) = tx_id { + println!("Connection {conn_id}(op={op_num}) rolling back transaction {tx_id}"); + shadow_db.rollback_transaction(*tx_id); + } + *tx_id = None; + } + if is_acceptable_error(e) { + return; + } + panic!("Unexpected error: {e}"); + }; + // Interleave operations between all connections - for op_num in 0..OPERATIONS_PER_CONNECTION { + for op_num in 0..opts.operations_per_connection { for (conn, conn_id, current_tx_id) in &mut connections { // Generate operation based on current transaction state let (operation, visible_rows) = generate_operation(&mut rng, *current_tx_id, &mut shared_shadow_db); let is_in_tx = current_tx_id.is_some(); + let is_in_tx_str = if is_in_tx { + format!("true(tx_id={:?})", current_tx_id.unwrap()) + } else { + "false".to_string() + }; let has_snapshot = current_tx_id.is_some_and(|tx_id| { shared_shadow_db.transactions.get(&tx_id).unwrap().is_some() }); - println!("Connection {conn_id}(op={op_num}): {operation}, is_in_tx={is_in_tx}, has_snapshot={has_snapshot}"); + println!("Connection {conn_id}(op={op_num}): {operation}, is_in_tx={is_in_tx_str}, has_snapshot={has_snapshot}"); match operation { - Operation::Begin => { + Operation::Begin { concurrent } => { shared_shadow_db.begin_transaction(next_tx_id, false); + if concurrent { + // in tursodb, BEGIN CONCURRENT immediately starts a transaction. + shared_shadow_db.take_snapshot_if_not_exists(next_tx_id); + } *current_tx_id = Some(next_tx_id); next_tx_id += 1; - conn.execute("BEGIN", ()).await.unwrap(); + let query = operation.to_string(); + + conn.execute(query.as_str(), ()).await.unwrap(); } Operation::Commit => { let Some(tx_id) = *current_tx_id else { @@ -578,13 +712,13 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { shared_shadow_db.commit_transaction(tx_id); *current_tx_id = None; } - Err(e) => { - println!("Connection {conn_id}(op={op_num}) FAILED: {e}"); - // Check if it's an acceptable error - if !e.to_string().contains("database is locked") { - panic!("Unexpected error during commit: {e}"); - } - } + Err(e) => handle_error( + &e, + current_tx_id, + *conn_id, + op_num, + &mut shared_shadow_db, + ), } } Operation::Rollback => { @@ -598,15 +732,13 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { shared_shadow_db.rollback_transaction(tx_id); *current_tx_id = None; } - Err(e) => { - println!("Connection {conn_id}(op={op_num}) FAILED: {e}"); - // Check if it's an acceptable error - if !e.to_string().contains("Busy") - && !e.to_string().contains("database is locked") - { - panic!("Unexpected error during rollback: {e}"); - } - } + Err(e) => handle_error( + &e, + current_tx_id, + *conn_id, + op_num, + &mut shared_shadow_db, + ), } } } @@ -645,13 +777,13 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { next_tx_id += 1; } } - Err(e) => { - println!("Connection {conn_id}(op={op_num}) FAILED: {e}"); - // Check if it's an acceptable error - if !e.to_string().contains("database is locked") { - panic!("Unexpected error during insert: {e}"); - } - } + Err(e) => handle_error( + &e, + current_tx_id, + *conn_id, + op_num, + &mut shared_shadow_db, + ), } } Operation::Update { id, other_columns } => { @@ -683,13 +815,13 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { next_tx_id += 1; } } - Err(e) => { - println!("Connection {conn_id}(op={op_num}) FAILED: {e}"); - // Check if it's an acceptable error - if !e.to_string().contains("database is locked") { - panic!("Unexpected error during update: {e}"); - } - } + Err(e) => handle_error( + &e, + current_tx_id, + *conn_id, + op_num, + &mut shared_shadow_db, + ), } } Operation::Delete { id } => { @@ -716,13 +848,13 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { next_tx_id += 1; } } - Err(e) => { - println!("Connection {conn_id}(op={op_num}) FAILED: {e}"); - // Check if it's an acceptable error - if !e.to_string().contains("database is locked") { - panic!("Unexpected error during delete: {e}"); - } - } + Err(e) => handle_error( + &e, + current_tx_id, + *conn_id, + op_num, + &mut shared_shadow_db, + ), } } Operation::Select => { @@ -735,9 +867,13 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { let ok = loop { match rows.next().await { Err(e) => { - if !e.to_string().contains("database is locked") { - panic!("Unexpected error during select: {e}"); - } + handle_error( + &e, + current_tx_id, + *conn_id, + op_num, + &mut shared_shadow_db, + ); break false; } Ok(None) => { @@ -747,7 +883,7 @@ async fn multiple_connections_fuzz(mvcc_enabled: bool) { let Value::Integer(id) = row.get_value(0).unwrap() else { panic!("Unexpected value for id: {:?}", row.get_value(0)); }; - let mut other_columns = HashMap::new(); + let mut other_columns = BTreeMap::new(); for i in 1..columns.len() { let column = columns.get(i).unwrap(); let value = row.get_value(i).unwrap(); @@ -879,120 +1015,169 @@ fn generate_operation( shadow_db.get_visible_rows(None) // No transaction } }; - match rng.random_range(0..100) { - 0..=9 => { - if !in_transaction { - (Operation::Begin, get_visible_rows()) - } else { - let visible_rows = get_visible_rows(); - ( - generate_data_operation(rng, &visible_rows, &schema_clone), - visible_rows, - ) - } - } - 10..=14 => { - if in_transaction { - (Operation::Commit, get_visible_rows()) - } else { - let visible_rows = get_visible_rows(); - ( - generate_data_operation(rng, &visible_rows, &schema_clone), - visible_rows, - ) - } - } - 15..=19 => { - if in_transaction { - (Operation::Rollback, get_visible_rows()) - } else { - let visible_rows = get_visible_rows(); - ( - generate_data_operation(rng, &visible_rows, &schema_clone), - visible_rows, - ) - } - } - 20..=22 => { - let mode = match rng.random_range(0..=3) { - 0 => CheckpointMode::Passive, - 1 => CheckpointMode::Restart, - 2 => CheckpointMode::Truncate, - 3 => CheckpointMode::Full, - _ => unreachable!(), - }; - (Operation::Checkpoint { mode }, get_visible_rows()) - } - 23..=26 => { - let op = match rng.random_range(0..6) { - 0..=2 => AlterTableOp::AddColumn { - name: format!("col_{}", rng.random_range(1..i64::MAX)), - ty: "TEXT".to_string(), - }, - 3..=4 => { - let table_schema = schema_clone.get("test_table").unwrap(); - let columns_no_id = table_schema - .columns - .iter() - .filter(|c| c.name != "id") - .collect::>(); - if columns_no_id.is_empty() { - AlterTableOp::AddColumn { - name: format!("col_{}", rng.random_range(1..i64::MAX)), - ty: "TEXT".to_string(), - } - } else { - let column = columns_no_id.choose(rng).unwrap(); - AlterTableOp::DropColumn { - name: column.name.clone(), - } - } - } - 5 => { - let columns_no_id = schema_clone - .get("test_table") - .unwrap() - .columns - .iter() - .filter(|c| c.name != "id") - .collect::>(); - if columns_no_id.is_empty() { - AlterTableOp::AddColumn { - name: format!("col_{}", rng.random_range(1..i64::MAX)), - ty: "TEXT".to_string(), - } - } else { - let column = columns_no_id.choose(rng).unwrap(); - AlterTableOp::RenameColumn { - old_name: column.name.clone(), - new_name: format!("col_{}", rng.random_range(1..i64::MAX)), - } - } - } - _ => unreachable!(), - }; - (Operation::AlterTable { op }, get_visible_rows()) - } - _ => { + + let mut start = 0; + let range_begin_deferred = start..start + shadow_db.query_gen_options.weight_begin_deferred; + start += shadow_db.query_gen_options.weight_begin_deferred; + let range_begin_concurrent = start..start + shadow_db.query_gen_options.weight_begin_concurrent; + start += shadow_db.query_gen_options.weight_begin_concurrent; + let range_commit = start..start + shadow_db.query_gen_options.weight_commit; + start += shadow_db.query_gen_options.weight_commit; + let range_rollback = start..start + shadow_db.query_gen_options.weight_rollback; + start += shadow_db.query_gen_options.weight_rollback; + let range_checkpoint = start..start + shadow_db.query_gen_options.weight_checkpoint; + start += shadow_db.query_gen_options.weight_checkpoint; + let range_ddl = start..start + shadow_db.query_gen_options.weight_ddl; + start += shadow_db.query_gen_options.weight_ddl; + let range_dml = start..start + shadow_db.query_gen_options.weight_dml; + start += shadow_db.query_gen_options.weight_dml; + + let random_val = rng.random_range(0..start); + + if range_begin_deferred.contains(&random_val) { + if !in_transaction { + (Operation::Begin { concurrent: false }, get_visible_rows()) + } else { let visible_rows = get_visible_rows(); ( - generate_data_operation(rng, &visible_rows, &schema_clone), + generate_data_operation( + rng, + &visible_rows, + &schema_clone, + &shadow_db.query_gen_options.dml_gen_options, + ), visible_rows, ) } + } else if range_begin_concurrent.contains(&random_val) { + if !in_transaction { + (Operation::Begin { concurrent: true }, get_visible_rows()) + } else { + let visible_rows = get_visible_rows(); + ( + generate_data_operation( + rng, + &visible_rows, + &schema_clone, + &shadow_db.query_gen_options.dml_gen_options, + ), + visible_rows, + ) + } + } else if range_commit.contains(&random_val) { + if in_transaction { + (Operation::Commit, get_visible_rows()) + } else { + let visible_rows = get_visible_rows(); + ( + generate_data_operation( + rng, + &visible_rows, + &schema_clone, + &shadow_db.query_gen_options.dml_gen_options, + ), + visible_rows, + ) + } + } else if range_rollback.contains(&random_val) { + if in_transaction { + (Operation::Rollback, get_visible_rows()) + } else { + let visible_rows = get_visible_rows(); + ( + generate_data_operation( + rng, + &visible_rows, + &schema_clone, + &shadow_db.query_gen_options.dml_gen_options, + ), + visible_rows, + ) + } + } else if range_checkpoint.contains(&random_val) { + let mode = match rng.random_range(0..=3) { + 0 => CheckpointMode::Passive, + 1 => CheckpointMode::Restart, + 2 => CheckpointMode::Truncate, + 3 => CheckpointMode::Full, + _ => unreachable!(), + }; + (Operation::Checkpoint { mode }, get_visible_rows()) + } else if range_ddl.contains(&random_val) { + let op = match rng.random_range(0..6) { + 0..=2 => AlterTableOp::AddColumn { + name: format!("col_{}", rng.random_range(1..i64::MAX)), + ty: "TEXT".to_string(), + }, + 3..=4 => { + let table_schema = schema_clone.get("test_table").unwrap(); + let columns_no_id = table_schema + .columns + .iter() + .filter(|c| c.name != "id") + .collect::>(); + if columns_no_id.is_empty() { + AlterTableOp::AddColumn { + name: format!("col_{}", rng.random_range(1..i64::MAX)), + ty: "TEXT".to_string(), + } + } else { + let column = columns_no_id.choose(rng).unwrap(); + AlterTableOp::DropColumn { + name: column.name.clone(), + } + } + } + 5 => { + let columns_no_id = schema_clone + .get("test_table") + .unwrap() + .columns + .iter() + .filter(|c| c.name != "id") + .collect::>(); + if columns_no_id.is_empty() { + AlterTableOp::AddColumn { + name: format!("col_{}", rng.random_range(1..i64::MAX)), + ty: "TEXT".to_string(), + } + } else { + let column = columns_no_id.choose(rng).unwrap(); + AlterTableOp::RenameColumn { + old_name: column.name.clone(), + new_name: format!("col_{}", rng.random_range(1..i64::MAX)), + } + } + } + _ => unreachable!(), + }; + (Operation::AlterTable { op }, get_visible_rows()) + } else if range_dml.contains(&random_val) { + let visible_rows = get_visible_rows(); + ( + generate_data_operation( + rng, + &visible_rows, + &schema_clone, + &shadow_db.query_gen_options.dml_gen_options, + ), + visible_rows, + ) + } else { + unreachable!() } } fn generate_data_operation( rng: &mut ChaCha8Rng, visible_rows: &[DbRow], - schema: &HashMap, + schema: &BTreeMap, + dml_gen_options: &DmlGenOptions, ) -> Operation { let table_schema = schema.get("test_table").unwrap(); - let op_num = rng.random_range(0..4); - let mut generate_insert_operation = || { + let generate_insert_operation = |rng: &mut ChaCha8Rng| { let id = rng.random_range(1..i64::MAX); - let mut other_columns = HashMap::new(); + let mut other_columns = BTreeMap::new(); for column in table_schema.columns.iter() { if column.name == "id" { continue; @@ -1009,61 +1194,65 @@ fn generate_data_operation( } Operation::Insert { id, other_columns } }; - match op_num { - 0 => { - // Insert - generate_insert_operation() - } - 1 => { - // Update - if visible_rows.is_empty() { - // No rows to update, try insert instead - generate_insert_operation() - } else { - let columns_no_id = table_schema - .columns + let mut start = 0; + let range_insert = start..start + dml_gen_options.weight_insert; + start += dml_gen_options.weight_insert; + let range_update = start..start + dml_gen_options.weight_update; + start += dml_gen_options.weight_update; + let range_delete = start..start + dml_gen_options.weight_delete; + start += dml_gen_options.weight_delete; + let range_select = start..start + dml_gen_options.weight_select; + start += dml_gen_options.weight_select; + + let random_val = rng.random_range(0..start); + + if range_insert.contains(&random_val) { + generate_insert_operation(rng) + } else if range_update.contains(&random_val) { + if visible_rows.is_empty() { + // No rows to update, try insert instead + generate_insert_operation(rng) + } else { + let columns_no_id = table_schema + .columns + .iter() + .filter(|c| c.name != "id") + .collect::>(); + if columns_no_id.is_empty() { + // No columns to update, try insert instead + return generate_insert_operation(rng); + } + let id = visible_rows.choose(rng).unwrap().id; + let col_name_to_update = columns_no_id.choose(rng).unwrap().name.clone(); + let mut other_columns = BTreeMap::new(); + other_columns.insert( + col_name_to_update.clone(), + match columns_no_id .iter() - .filter(|c| c.name != "id") - .collect::>(); - if columns_no_id.is_empty() { - // No columns to update, try insert instead - return generate_insert_operation(); - } - let id = visible_rows.choose(rng).unwrap().id; - let col_name_to_update = columns_no_id.choose(rng).unwrap().name.clone(); - let mut other_columns = HashMap::new(); - other_columns.insert( - col_name_to_update.clone(), - match columns_no_id - .iter() - .find(|c| c.name == col_name_to_update) - .unwrap() - .ty - .as_str() - { - "TEXT" => Value::Text(format!("updated_{}", rng.random_range(1..i64::MAX))), - "INTEGER" => Value::Integer(rng.random_range(1..i64::MAX)), - "REAL" => Value::Real(rng.random_range(1..i64::MAX) as f64), - _ => Value::Null, - }, - ); - Operation::Update { id, other_columns } - } + .find(|c| c.name == col_name_to_update) + .unwrap() + .ty + .as_str() + { + "TEXT" => Value::Text(format!("updated_{}", rng.random_range(1..i64::MAX))), + "INTEGER" => Value::Integer(rng.random_range(1..i64::MAX)), + "REAL" => Value::Real(rng.random_range(1..i64::MAX) as f64), + _ => Value::Null, + }, + ); + Operation::Update { id, other_columns } } - 2 => { - // Delete - if visible_rows.is_empty() { - // No rows to delete, try insert instead - generate_insert_operation() - } else { - let id = visible_rows.choose(rng).unwrap().id; - Operation::Delete { id } - } + } else if range_delete.contains(&random_val) { + if visible_rows.is_empty() { + // No rows to delete, try insert instead + generate_insert_operation(rng) + } else { + let id = visible_rows.choose(rng).unwrap().id; + Operation::Delete { id } } - 3 => { - // Select - Operation::Select - } - _ => unreachable!(), + } else if range_select.contains(&random_val) { + Operation::Select + } else { + unreachable!() } } diff --git a/tests/integration/query_processing/test_transactions.rs b/tests/integration/query_processing/test_transactions.rs index f30044934..5de2bb566 100644 --- a/tests/integration/query_processing/test_transactions.rs +++ b/tests/integration/query_processing/test_transactions.rs @@ -355,6 +355,109 @@ fn test_mvcc_concurrent_insert_basic() { ); } +#[test] +fn test_mvcc_update_same_row_twice() { + let tmp_db = TempDatabase::new_with_opts( + "test_mvcc_update_same_row_twice.db", + turso_core::DatabaseOpts::new().with_mvcc(true), + ); + let conn1 = tmp_db.connect_limbo(); + + conn1 + .execute("CREATE TABLE test (id INTEGER PRIMARY KEY, value TEXT)") + .unwrap(); + + conn1 + .execute("INSERT INTO test (id, value) VALUES (1, 'first')") + .unwrap(); + + conn1 + .execute("UPDATE test SET value = 'second' WHERE id = 1") + .unwrap(); + + let stmt = conn1 + .query("SELECT value FROM test WHERE id = 1") + .unwrap() + .unwrap(); + let row = helper_read_single_row(stmt); + let Value::Text(value) = &row[0] else { + panic!("expected text value"); + }; + assert_eq!(value.as_str(), "second"); + + conn1 + .execute("UPDATE test SET value = 'third' WHERE id = 1") + .unwrap(); + + let stmt = conn1 + .query("SELECT value FROM test WHERE id = 1") + .unwrap() + .unwrap(); + let row = helper_read_single_row(stmt); + let Value::Text(value) = &row[0] else { + panic!("expected text value"); + }; + assert_eq!(value.as_str(), "third"); +} + +#[test] +fn test_mvcc_concurrent_conflicting_update() { + let tmp_db = TempDatabase::new_with_opts( + "test_mvcc_concurrent_conflicting_update.db", + turso_core::DatabaseOpts::new().with_mvcc(true), + ); + let conn1 = tmp_db.connect_limbo(); + let conn2 = tmp_db.connect_limbo(); + + conn1 + .execute("CREATE TABLE test (id INTEGER, value TEXT)") + .unwrap(); + + conn1 + .execute("INSERT INTO test (id, value) VALUES (1, 'first')") + .unwrap(); + + conn1.execute("BEGIN CONCURRENT").unwrap(); + conn2.execute("BEGIN CONCURRENT").unwrap(); + + conn1 + .execute("UPDATE test SET value = 'second' WHERE id = 1") + .unwrap(); + let err = conn2 + .execute("UPDATE test SET value = 'third' WHERE id = 1") + .expect_err("expected error"); + assert!(matches!(err, LimboError::WriteWriteConflict)); +} + +#[test] +fn test_mvcc_concurrent_conflicting_update_2() { + let tmp_db = TempDatabase::new_with_opts( + "test_mvcc_concurrent_conflicting_update.db", + turso_core::DatabaseOpts::new().with_mvcc(true), + ); + let conn1 = tmp_db.connect_limbo(); + let conn2 = tmp_db.connect_limbo(); + + conn1 + .execute("CREATE TABLE test (id INTEGER, value TEXT)") + .unwrap(); + + conn1 + .execute("INSERT INTO test (id, value) VALUES (1, 'first'), (2, 'first')") + .unwrap(); + + conn1.execute("BEGIN CONCURRENT").unwrap(); + conn2.execute("BEGIN CONCURRENT").unwrap(); + + conn1 + .execute("UPDATE test SET value = 'second' WHERE id = 1") + .unwrap(); + let err = conn2 + .execute("UPDATE test SET value = 'third' WHERE id BETWEEN 0 AND 10") + .expect_err("expected error"); + assert!(matches!(err, LimboError::WriteWriteConflict)); +} + fn helper_read_all_rows(mut stmt: turso_core::Statement) -> Vec> { let mut ret = Vec::new(); loop { diff --git a/vendored/sqlite3-parser/Cargo.toml b/vendored/sqlite3-parser/Cargo.toml index 8161def05..0b6b39f8e 100644 --- a/vendored/sqlite3-parser/Cargo.toml +++ b/vendored/sqlite3-parser/Cargo.toml @@ -27,17 +27,17 @@ serde = ["dep:serde", "indexmap/serde", "bitflags/serde"] [dependencies] log = "0.4.22" memchr = "2.0" -fallible-iterator = "0.3" -bitflags = "2.0" -indexmap = "2.0" -miette = "7.4.0" +fallible-iterator = { workspace = true } +bitflags = { workspace = true } +indexmap = { workspace = true } +miette = { workspace = true } strum = { workspace = true } strum_macros = {workspace = true } serde = { workspace = true , optional = true, features = ["derive"] } smallvec = { version = "1.15.1", features = ["const_generics"] } [dev-dependencies] -env_logger = { version = "0.11", default-features = false } +env_logger = { workspace = true, default-features = false } [build-dependencies] cc = "1.0" diff --git a/vendored/sqlite3-parser/sqlparser_bench/Cargo.toml b/vendored/sqlite3-parser/sqlparser_bench/Cargo.toml index 3f99ef75c..0366bd63d 100644 --- a/vendored/sqlite3-parser/sqlparser_bench/Cargo.toml +++ b/vendored/sqlite3-parser/sqlparser_bench/Cargo.toml @@ -9,10 +9,10 @@ turso_sqlite3_parser = { path = "..", default-features = false, features = [ "YYNOERRORRECOVERY", "NDEBUG", ] } -fallible-iterator = "0.3" +fallible-iterator = { workspace = true } [dev-dependencies] -criterion = "0.5" +criterion = { workspace = true } [[bench]] name = "sqlparser_bench" diff --git a/whopper/Cargo.toml b/whopper/Cargo.toml index 7ca99652d..0695ebcf4 100644 --- a/whopper/Cargo.toml +++ b/whopper/Cargo.toml @@ -16,14 +16,14 @@ path = "main.rs" [dependencies] anyhow.workspace = true -clap = { version = "4.5", features = ["derive"] } +clap = { workspace = true, features = ["derive"] } memmap2 = "0.9" rand = { workspace = true } -rand_chacha = "0.9.0" +rand_chacha = { workspace = true } sql_generation = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } -turso_core = { path = "../core", features = ["simulator"]} +tracing-subscriber = { workspace = true, features = ["env-filter"] } +turso_core = { workspace = true, features = ["simulator"]} turso_parser = { workspace = true } [features]