core_tester: sequential btree write test

Signed-off-by: Pere Diaz Bou <pere-altea@hotmail.com>

Cargo.lock

@@ -388,6 +388,19 @@ version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
 
+[[package]]
+name = "core_tester"
+version = "0.0.4"
+dependencies = [
+ "anyhow",
+ "clap",
+ "dirs",
+ "env_logger 0.10.2",
+ "limbo_core",
+ "rstest",
+ "rustyline",
+]
+
 [[package]]
 name = "cpp_demangle"
 version = "0.4.3"

Cargo.toml

@@ -3,12 +3,13 @@
 [workspace]
 resolver = "2"
 members = [
-  "bindings/python",
-  "bindings/wasm",
-  "cli",
-  "sqlite3",
-  "core",
-  "simulator",
+    "bindings/python",
+    "bindings/wasm",
+    "cli",
+    "sqlite3",
+    "core",
+    "simulator",
+    "core_tester",
 ]
 exclude = ["perf/latency/limbo"]
 
@@ -28,7 +29,12 @@ ci = "github"
 # The installers to generate for each app
 installers = ["shell", "powershell"]
 # Target platforms to build apps for (Rust target-triple syntax)
-targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"]
+targets = [
+    "aarch64-apple-darwin",
+    "x86_64-apple-darwin",
+    "x86_64-unknown-linux-gnu",
+    "x86_64-pc-windows-msvc",
+]
 # Which actions to run on pull requests
 pr-run-mode = "plan"
 # Path that installers should place binaries in

core_tester/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "core_tester"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+description = "Internal tester of write path"
+
+[[bin]]
+name = "core_tester"
+path = "src/main.rs"
+
+
+[dependencies]
+anyhow = "1.0.75"
+clap = { version = "4.4.0", features = ["derive"] }
+dirs = "5.0.1"
+env_logger = "0.10.1"
+limbo_core = { path = "../core" }
+rustyline = "12.0.0"
+
+[dev-dependencies]
+
+rstest = "0.18.2"

core_tester/README.md

@@ -0,0 +1,5 @@
+Currently the best way to run these tests are like this due to long running tests:
+
+```bash
+cargo test test_sequential_write -- --nocapture
+```

core_tester/reset.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+export RUST_BACKTRACE=1
+rm test.db -f
+rm query-log.log -f 
+# for now only integer primary key supported
+echo "create table test (x INTEGER PRIMARY KEY);" | tee -a query-log.log | sqlite3 test.db

core_tester/src/main.rs

@@ -0,0 +1,108 @@
+use clap::{Parser, ValueEnum};
+use limbo_core::{Database, RowResult, Value};
+use rustyline::{error::ReadlineError, DefaultEditor};
+use std::borrow::Borrow;
+use std::fmt::format;
+use std::path::PathBuf;
+use std::process::Command;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
+enum OutputMode {
+    Raw,
+    Pretty,
+}
+
+impl std::fmt::Display for OutputMode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.to_possible_value()
+            .expect("no values are skipped")
+            .get_name()
+            .fmt(f)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_sequential_write() -> anyhow::Result<()> {
+        env_logger::init();
+        let path = "test.db";
+
+        let io: Arc<dyn limbo_core::IO> = Arc::new(limbo_core::PlatformIO::new()?);
+        dbg!(path);
+
+        // run reset command
+        let result = Command::new("./reset.sh")
+            .output()
+            .expect("failed to execute process");
+        println!("finished creating db {:?}", result.stdout);
+        println!("finished creating db {:?}", result.stderr);
+        let db = Database::open_file(io.clone(), path)?;
+        let conn = db.connect();
+
+        let list_query = "SELECT * FROM test";
+        let max_iterations = 10000;
+        for i in 0..max_iterations {
+            if (i % 100) == 0 {
+                let progress = (i as f64 / max_iterations as f64) * 100.0;
+                println!("progress {:.1}%", progress);
+            }
+            let insert_query = format!("INSERT INTO test VALUES ({})", i);
+            match conn.query(insert_query) {
+                Ok(Some(ref mut rows)) => loop {
+                    match rows.next_row()? {
+                        RowResult::IO => {
+                            io.run_once()?;
+                        }
+                        RowResult::Done => break,
+                        _ => unreachable!(),
+                    }
+                },
+                Ok(None) => {}
+                Err(err) => {
+                    eprintln!("{}", err);
+                }
+            };
+
+            let mut current_read_index = 0;
+            match conn.query(list_query) {
+                Ok(Some(ref mut rows)) => loop {
+                    match rows.next_row()? {
+                        RowResult::Row(row) => {
+                            let first_value = row.values.first().expect("missing id");
+                            let id = match first_value {
+                                Value::Integer(i) => *i as i32,
+                                Value::Float(f) => *f as i32,
+                                _ => unreachable!(),
+                            };
+                            assert_eq!(current_read_index, id);
+                            current_read_index += 1;
+                        }
+                        RowResult::IO => {
+                            io.run_once()?;
+                        }
+                        RowResult::Done => break,
+                    }
+                },
+                Ok(None) => {}
+                Err(err) => {
+                    eprintln!("{}", err);
+                }
+            }
+            conn.cacheflush()?;
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn simple_test() {
+        assert_eq!(2 + 2, 4);
+    }
+}
+
+fn main() -> anyhow::Result<()> {
+    Ok(())
+}