Merge 'Python script to compare vfs performance' from Preston Thorpe

This PR adds a python script that uses the `TestLimboShell` setup to run
some semi naive benchmarks/comparisons against `io_uring` and `syscall`
IO back-ends.
### Usage:
```sh
make bench-vfs SQL="insert into products (name, price) values ('testing', randomblob(1024*4));" N=50
```
The script will execute the given `SQL` `N` times with each back-end,
get the average/mean and display them.
![image](https://github.com/user-
attachments/assets/b2399196-dbdd-4b98-8210-536e68979edd)
😬

Reviewed-by: Jussi Saurio <jussi.saurio@gmail.com>

Closes #1377

Makefile

@@ -70,7 +70,6 @@ test: limbo uv-sync test-compat test-vector test-sqlite3 test-shell test-extensi
 .PHONY: test
 
 test-extensions: limbo uv-sync
-	cargo build --package limbo_regexp
 	uv run --project limbo_test test-extensions
 .PHONY: test-extensions
 
@@ -110,6 +109,10 @@ test-update: limbo uv-sync
 	SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-update
 .PHONY: test-update
 
+bench-vfs: uv-sync
+	cargo build --release
+	uv run --project limbo_test bench-vfs "$(SQL)" "$(N)"
+
 clickbench:
 	./perf/clickbench/benchmark.sh
 .PHONY: clickbench

PERF.md

@@ -33,6 +33,18 @@ make clickbench
 This will build Limbo in release mode, create a database, and run the benchmarks with a small subset of the Clickbench dataset.
 It will run the queries for both Limbo and SQLite, and print the results.
 
+
+
+## Comparing VFS's/IO Back-ends (io_uring | syscall)
+
+```shell
+make bench-vfs SQL="select * from users;" N=500
+```
+
+The naive script will build and run limbo in release mode and execute the given SQL (against a copy of the `testing/testing.db` file)
+`N` times with each `vfs`. This is not meant to be a definitive or thorough performance benchmark but serves to compare the two.
+
+
 ## TPC-H
 
 1. Clone the Taratool TPC-H benchmarking tool:
@@ -57,3 +69,4 @@ index 6b894f9..c808e9a 100755
         )
  }
 ``` 
+

testing/cli_tests/vfs_bench.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+# vfs benchmarking/comparison
+import os
+from pathlib import Path
+import subprocess
+import statistics
+import argparse
+from time import perf_counter, sleep
+from typing import Dict
+
+from cli_tests.test_limbo_cli import TestLimboShell
+from cli_tests.console import info, error, test
+
+LIMBO_BIN = Path("./target/release/limbo")
+DB_FILE = Path("testing/temp.db")
+vfs_list = ["syscall", "io_uring"]
+
+
+def append_time(times, start, perf_counter):
+    times.append(perf_counter() - start)
+    return True
+
+
+def bench_one(vfs: str, sql: str, iterations: int) -> list[float]:
+    """
+    Launch a single Limbo process with the requested VFS, run `sql`
+    `iterations` times, return a list of elapsed wall‑clock times.
+    """
+    shell = TestLimboShell(
+        exec_name=str(LIMBO_BIN),
+        flags=f"-q -m list --vfs {vfs} {DB_FILE}",
+        init_commands="",
+    )
+
+    times: list[float] = []
+
+    for i in range(1, iterations + 1):
+        start = perf_counter()
+        _ = shell.run_test_fn(
+            sql, lambda x: x is not None and append_time(times, start, perf_counter)
+        )
+        test(f"  {vfs} | run {i:>3}: {times[-1]:.6f}s")
+
+    shell.quit()
+    return times
+
+
+def setup_temp_db() -> None:
+    cmd = ["sqlite3", "testing/testing.db", ".clone testing/temp.db"]
+    proc = subprocess.run(cmd, check=True)
+    proc.check_returncode()
+    sleep(0.3)  # make sure it's finished
+
+
+def cleanup_temp_db() -> None:
+    if DB_FILE.exists():
+        DB_FILE.unlink()
+        os.remove("testing/temp.db-wal")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Benchmark a SQL statement against all Limbo VFS back‑ends."
+    )
+    parser.add_argument("sql", help="SQL statement to execute (quote it)")
+    parser.add_argument("iterations", type=int, help="number of repetitions")
+    args = parser.parse_args()
+    setup_temp_db()
+
+    sql, iterations = args.sql, args.iterations
+    if iterations <= 0:
+        error("iterations must be a positive integer")
+        parser.error("Invalid Arguments")
+
+    info(f"SQL        : {sql}")
+    info(f"Iterations : {iterations}")
+    info(f"Database   : {DB_FILE.resolve()}")
+    info("-" * 60)
+    averages: Dict[str, float] = {}
+
+    for vfs in vfs_list:
+        test(f"\n### VFS: {vfs} ###")
+        times = bench_one(vfs, sql, iterations)
+        info(f"All times ({vfs}):", " ".join(f"{t:.6f}" for t in times))
+        avg = statistics.mean(times)
+        averages[vfs] = avg
+
+    info("\n" + "-" * 60)
+    info("Average runtime per VFS")
+    info("-" * 60)
+
+    for vfs in vfs_list:
+        info(f"vfs: {vfs} : {averages[vfs]:.6f} s")
+    info("-" * 60)
+
+    baseline = "syscall"
+    baseline_avg = averages[baseline]
+
+    name_pad = max(len(v) for v in vfs_list)
+    for vfs in vfs_list:
+        avg = averages[vfs]
+        if vfs == baseline:
+            info(f"{vfs:<{name_pad}} : {avg:.6f}  (baseline)")
+        else:
+            pct = (avg - baseline_avg) / baseline_avg * 100.0
+            faster_slower = "slower" if pct > 0 else "faster"
+            info(
+                f"{vfs:<{name_pad}} : {avg:.6f}  ({abs(pct):.1f}% {faster_slower} than {baseline})"
+            )
+        info("-" * 60)
+    cleanup_temp_db()
+
+
+if __name__ == "__main__":
+    main()

testing/pyproject.toml

@@ -15,6 +15,7 @@ test-shell = "cli_tests.cli_test_cases:main"
 test-extensions = "cli_tests.extensions:main"
 test-update = "cli_tests.update:main"
 test-memory = "cli_tests.memory:main"
+bench-vfs = "cli_tests.vfs_bench:main"
 
 [tool.uv]
 package = true