From 7f170756ae347d9d61b40ef41333cea7e653b4eb Mon Sep 17 00:00:00 2001
From: PThorpe92 <preston@unlockedlabs.org>
Date: Mon, 21 Apr 2025 12:22:20 -0400
Subject: [PATCH 1/5] Add python script to benchmark vfs against eachother

---
 testing/cli_tests/vfs_bench.py | 116 +++++++++++++++++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 testing/cli_tests/vfs_bench.py

diff --git a/testing/cli_tests/vfs_bench.py b/testing/cli_tests/vfs_bench.py
new file mode 100644
index 000000000..ae5a969d0
--- /dev/null
+++ b/testing/cli_tests/vfs_bench.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+# vfs benchmarking/comparison
+import os
+from pathlib import Path
+import subprocess
+import statistics
+import argparse
+from time import perf_counter, sleep
+from typing import Dict
+
+from cli_tests.test_limbo_cli import TestLimboShell
+from cli_tests.console import info, error, test
+
+LIMBO_BIN = Path("./target/release/limbo")
+DB_FILE = Path("testing/temp.db")
+vfs_list = ["syscall", "io_uring"]
+
+
+def append_time(times, start, perf_counter):
+    times.append(perf_counter() - start)
+    return True
+
+
+def bench_one(vfs: str, sql: str, iterations: int) -> list[float]:
+    """
+    Launch a single Limbo process with the requested VFS, run `sql`
+    `iterations` times, return a list of elapsed wall‑clock times.
+    """
+    shell = TestLimboShell(
+        exec_name=str(LIMBO_BIN),
+        flags=f"-q -m list --vfs {vfs} {DB_FILE}",
+        init_commands="",
+    )
+
+    times: list[float] = []
+
+    for i in range(1, iterations + 1):
+        start = perf_counter()
+        _ = shell.run_test_fn(
+            sql, lambda x: x is not None and append_time(times, start, perf_counter)
+        )
+        test(f"  {vfs} | run {i:>3}: {times[-1]:.6f}s")
+
+    shell.quit()
+    return times
+
+
+def setup_temp_db() -> None:
+    cmd = ["sqlite3", "testing/testing.db", ".clone testing/temp.db"]
+    proc = subprocess.run(cmd, check=True)
+    proc.check_returncode()
+    sleep(0.3)  # make sure it's finished
+
+
+def cleanup_temp_db() -> None:
+    if DB_FILE.exists():
+        DB_FILE.unlink()
+        os.remove("testing/temp.db-wal")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Benchmark a SQL statement against all Limbo VFS back‑ends."
+    )
+    parser.add_argument("sql", help="SQL statement to execute (quote it)")
+    parser.add_argument("iterations", type=int, help="number of repetitions")
+    args = parser.parse_args()
+    setup_temp_db()
+
+    sql, iterations = args.sql, args.iterations
+    if iterations <= 0:
+        error("iterations must be a positive integer")
+        parser.error("Invalid Arguments")
+
+    info(f"SQL        : {sql}")
+    info(f"Iterations : {iterations}")
+    info(f"Database   : {DB_FILE.resolve()}")
+    info("-" * 60)
+    averages: Dict[str, float] = {}
+
+    for vfs in vfs_list:
+        test(f"\n### VFS: {vfs} ###")
+        times = bench_one(vfs, sql, iterations)
+        info(f"All times ({vfs}):", " ".join(f"{t:.6f}" for t in times))
+        avg = statistics.mean(times)
+        averages[vfs] = avg
+
+    info("\n" + "-" * 60)
+    info("Average runtime per VFS")
+    info("-" * 60)
+
+    for vfs in vfs_list:
+        info(f"vfs: {vfs} : {averages[vfs]:.6f} s")
+    info("-" * 60)
+
+    baseline = "syscall"
+    baseline_avg = averages[baseline]
+
+    name_pad = max(len(v) for v in vfs_list)
+    for vfs in vfs_list:
+        avg = averages[vfs]
+        if vfs == baseline:
+            info(f"{vfs:<{name_pad}} : {avg:.6f}  (baseline)")
+        else:
+            pct = (avg - baseline_avg) / baseline_avg * 100.0
+            faster_slower = "slower" if pct > 0 else "faster"
+            info(
+                f"{vfs:<{name_pad}} : {avg:.6f}  ({abs(pct):.1f}% {faster_slower} than {baseline})"
+            )
+        info("-" * 60)
+    cleanup_temp_db()
+
+
+if __name__ == "__main__":
+    main()

From 2037fbeba540a4330a438c33c31c2b60c233ebc5 Mon Sep 17 00:00:00 2001
From: PThorpe92 <preston@unlockedlabs.org>
Date: Mon, 21 Apr 2025 12:22:40 -0400
Subject: [PATCH 2/5] Add bench-vfs command to makefile

---
 Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 623fbb6ce..9c5364aef 100644
--- a/Makefile
+++ b/Makefile
@@ -70,7 +70,6 @@ test: limbo uv-sync test-compat test-vector test-sqlite3 test-shell test-extensi
 .PHONY: test
 
 test-extensions: limbo uv-sync
-	cargo build --package limbo_regexp
 	uv run --project limbo_test test-extensions
 .PHONY: test-extensions
 
@@ -110,6 +109,9 @@ test-update: limbo uv-sync
 	SQLITE_EXEC=$(SQLITE_EXEC) uv run --project limbo_test test-update
 .PHONY: test-update
 
+bench-vfs: uv-sync
+	uv run --project limbo_test bench-vfs "$(SQL)" "$(N)"
+
 clickbench:
 	./perf/clickbench/benchmark.sh
 .PHONY: clickbench

From 9bbd6a3a7f3380b08e8a8b8000dcfb19f80955aa Mon Sep 17 00:00:00 2001
From: PThorpe92 <preston@unlockedlabs.org>
Date: Mon, 21 Apr 2025 12:23:06 -0400
Subject: [PATCH 3/5] Add vfs bench to testing pyproject.toml

---
 testing/pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/testing/pyproject.toml b/testing/pyproject.toml
index 58292dd91..cdd30ec54 100644
--- a/testing/pyproject.toml
+++ b/testing/pyproject.toml
@@ -15,6 +15,7 @@ test-shell = "cli_tests.cli_test_cases:main"
 test-extensions = "cli_tests.extensions:main"
 test-update = "cli_tests.update:main"
 test-memory = "cli_tests.memory:main"
+bench-vfs = "cli_tests.vfs_bench:main"
 
 [tool.uv]
 package = true

From f180de4d950aa072f9043a3a30feedbb7434106d Mon Sep 17 00:00:00 2001
From: PThorpe92 <preston@unlockedlabs.org>
Date: Mon, 21 Apr 2025 12:24:18 -0400
Subject: [PATCH 4/5] Write quick note about vfs benchmark script in PERF.md

---
 PERF.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/PERF.md b/PERF.md
index 40edcf7ea..fb25045fb 100644
--- a/PERF.md
+++ b/PERF.md
@@ -32,3 +32,13 @@ make clickbench
 
 This will build Limbo in release mode, create a database, and run the benchmarks with a small subset of the Clickbench dataset.
 It will run the queries for both Limbo and SQLite, and print the results.
+
+
+## Comparing VFS's/IO Back-ends (io_uring | syscall)
+
+```shell
+make bench-vfs SQL="select * from users;" N=500
+```
+
+The naive script will build and run limbo in release mode and execute the given SQL (against a copy of the `testing/testing.db` file)
+`N` times with each `vfs`. This is not meant to be a definitive or thorough performance benchmark but serves to compare the two.

From 2e33ce6896cb6e8796083ca9611b0ae995971467 Mon Sep 17 00:00:00 2001
From: PThorpe92 <preston@unlockedlabs.org>
Date: Mon, 21 Apr 2025 12:31:38 -0400
Subject: [PATCH 5/5] Add release build to bench vfs in makefile to ensure
 there is an exec target

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 9c5364aef..06afa0e5d 100644
--- a/Makefile
+++ b/Makefile
@@ -110,6 +110,7 @@ test-update: limbo uv-sync
 .PHONY: test-update
 
 bench-vfs: uv-sync
+	cargo build --release
 	uv run --project limbo_test bench-vfs "$(SQL)" "$(N)"
 
 clickbench: