diff --git a/Cargo.lock b/Cargo.lock index 8b3813db8..174e37be4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1660,6 +1660,7 @@ dependencies = [ "limbo_completion", "limbo_crypto", "limbo_ext", + "limbo_ext_tests", "limbo_ipaddr", "limbo_macros", "limbo_percentile", @@ -1717,19 +1718,19 @@ dependencies = [ ] [[package]] -name = "limbo_ipaddr" +name = "limbo_ext_tests" version = "0.0.16" dependencies = [ - "ipnetwork", + "lazy_static", "limbo_ext", "mimalloc", ] [[package]] -name = "limbo_kv" +name = "limbo_ipaddr" version = "0.0.16" dependencies = [ - "lazy_static", + "ipnetwork", "limbo_ext", "mimalloc", ] diff --git a/Cargo.toml b/Cargo.toml index e66fbb044..cb98958f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ "extensions/completion", "extensions/core", "extensions/crypto", - "extensions/kvstore", + "extensions/tests", "extensions/percentile", "extensions/regexp", "extensions/series", @@ -47,6 +47,7 @@ limbo_uuid = { path = "extensions/uuid", version = "0.0.16" } limbo_sqlite3_parser = { path = "vendored/sqlite3-parser", version = "0.0.16" } limbo_ipaddr = { path = "extensions/ipaddr", version = "0.0.16" } limbo_completion = { path = "extensions/completion", version = "0.0.16" } +limbo_ext_tests = { path = "extensions/tests", version = "0.0.16" } # Config for 'cargo dist' [workspace.metadata.dist] diff --git a/core/Cargo.toml b/core/Cargo.toml index 7be562867..a33f3bba2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -26,6 +26,7 @@ crypto = ["limbo_crypto/static"] series = ["limbo_series/static"] ipaddr = ["limbo_ipaddr/static"] completion = ["limbo_completion/static"] +testvfs = ["limbo_ext_tests/static"] [target.'cfg(target_os = "linux")'.dependencies] io-uring = { version = "0.6.1", optional = true } @@ -68,6 +69,7 @@ limbo_crypto = { workspace = true, optional = true, features = ["static"] } limbo_series = { workspace = true, optional = true, features = ["static"] } limbo_ipaddr = { workspace = true, optional = true, features = ["static"] } limbo_completion = { workspace = true, optional = true, features = ["static"] } +limbo_ext_tests = { workspace = true, optional = true, features = ["static"] } miette = "7.4.0" strum = "0.26" parking_lot = "0.12.3" diff --git a/extensions/kvstore/Cargo.toml b/extensions/tests/Cargo.toml similarity index 81% rename from extensions/kvstore/Cargo.toml rename to extensions/tests/Cargo.toml index cac010bb6..84c9cbae2 100644 --- a/extensions/kvstore/Cargo.toml +++ b/extensions/tests/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "limbo_kv" +name = "limbo_ext_tests" version.workspace = true authors.workspace = true edition.workspace = true @@ -17,4 +17,4 @@ lazy_static = "1.5.0" limbo_ext = { workspace = true, features = ["static"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] -mimalloc = { version = "*", default-features = false } +mimalloc = { version = "0.1", default-features = false } diff --git a/extensions/kvstore/src/lib.rs b/extensions/tests/src/lib.rs similarity index 96% rename from extensions/kvstore/src/lib.rs rename to extensions/tests/src/lib.rs index dbd17d5f1..baae4ba36 100644 --- a/extensions/kvstore/src/lib.rs +++ b/extensions/tests/src/lib.rs @@ -1,19 +1,21 @@ use lazy_static::lazy_static; +use limbo_ext::register_extension; use limbo_ext::{ - register_extension, scalar, ExtResult, ResultCode, VTabCursor, VTabKind, VTabModule, - VTabModuleDerive, Value, VfsDerive, VfsExtension, VfsFile, + scalar, ExtResult, ResultCode, VTabCursor, VTabKind, VTabModule, VTabModuleDerive, Value, + VfsDerive, VfsExtension, VfsFile, }; use std::collections::BTreeMap; use std::fs::{File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use std::sync::Mutex; -lazy_static! { - static ref GLOBAL_STORE: Mutex> = Mutex::new(BTreeMap::new()); -} - register_extension! { vtabs: { KVStoreVTab }, + vfs: { TestFS }, +} + +lazy_static! { + static ref GLOBAL_STORE: Mutex> = Mutex::new(BTreeMap::new()); } #[derive(VTabModuleDerive, Default)] @@ -149,12 +151,12 @@ impl VTabCursor for KVStoreCursor { } } -struct TestFile { +pub struct TestFile { file: File, } #[derive(VfsDerive, Default)] -struct TestFS; +pub struct TestFS; // Test that we can have additional extension types in the same file // and still register the vfs at comptime if linking staticly diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 0fd69a4db..e5b6702f7 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -1,7 +1,7 @@ mod args; use args::{RegisterExtensionInput, ScalarInfo}; use quote::{format_ident, quote}; -use syn::{parse_macro_input, DeriveInput, ItemFn}; +use syn::{parse_macro_input, DeriveInput, Item, ItemFn, ItemMod}; extern crate proc_macro; use proc_macro::{token_stream::IntoIter, Group, TokenStream, TokenTree}; use std::collections::HashMap; @@ -980,3 +980,42 @@ pub fn register_extension(input: TokenStream) -> TokenStream { TokenStream::from(expanded) } + +/// Recursively search for a function in the module tree +fn find_function_path( + function_name: &syn::Ident, + module_path: String, + items: &[Item], +) -> Option { + for item in items { + match item { + // if it's a function, check if its name matches + Item::Fn(func) if func.sig.ident == *function_name => { + return Some(module_path.clone()); + } + // recursively search inside modules + Item::Mod(ItemMod { + ident, + content: Some((_, sub_items)), + .. + }) => { + let new_path = format!("{}::{}", module_path, ident); + if let Some(path) = find_function_path(function_name, new_path, sub_items) { + return Some(path); + } + } + _ => {} + } + } + None +} + +fn locate_function(ident: syn::Ident) -> syn::Ident { + let syntax_tree: syn::File = syn::parse_file(include_str!("lib.rs")).unwrap(); + + if let Some(full_path) = find_function_path(&ident, "crate".to_string(), &syntax_tree.items) { + return format_ident!("{full_path}::{ident}"); + } + + panic!("Function `{}` not found in crate!", ident); +} diff --git a/testing/cli_tests/extensions.py b/testing/cli_tests/extensions.py index a1d278c90..6171bdca7 100755 --- a/testing/cli_tests/extensions.py +++ b/testing/cli_tests/extensions.py @@ -337,7 +337,7 @@ def test_series(): def test_kv(): - ext_path = "./target/debug/liblimbo_kv" + ext_path = "target/debug/liblimbo_testextension" limbo = TestLimboShell() limbo.run_test_fn( "create virtual table t using kv_store;", @@ -401,17 +401,18 @@ def test_kv(): ) limbo.quit() + def test_ipaddr(): limbo = TestLimboShell() ext_path = "./target/debug/liblimbo_ipaddr" - + limbo.run_test_fn( "SELECT ipfamily('192.168.1.1');", lambda res: "error: no such function: " in res, "ipfamily function returns null when ext not loaded", ) limbo.execute_dot(f".load {ext_path}") - + limbo.run_test_fn( "SELECT ipfamily('192.168.1.1');", lambda res: "4" == res, @@ -455,7 +456,7 @@ def test_ipaddr(): lambda res: "128" == res, "ipmasklen function returns the mask length for IPv6", ) - + limbo.run_test_fn( "SELECT ipnetwork('192.168.16.12/24');", lambda res: "192.168.16.0/24" == res, @@ -466,7 +467,76 @@ def test_ipaddr(): lambda res: "2001:db8::1/128" == res, "ipnetwork function returns the network for IPv6", ) - + limbo.quit() + + +def test_vfs(): + limbo = TestLimboShell() + ext_path = "target/debug/liblimbo_testextension" + limbo.run_test_fn(".vfslist", lambda x: "testvfs" not in x, "testvfs not loaded") + limbo.execute_dot(f".load {ext_path}") + limbo.run_test_fn( + ".vfslist", lambda res: "testvfs" in res, "testvfs extension loaded" + ) + limbo.execute_dot(".open testing/vfs.db testvfs") + limbo.execute_dot("create table test (id integer primary key, value float);") + limbo.execute_dot("create table vfs (id integer primary key, value blob);") + for i in range(50): + limbo.execute_dot("insert into test (value) values (randomblob(32*1024));") + limbo.execute_dot(f"insert into vfs (value) values ({i});") + limbo.run_test_fn( + "SELECT count(*) FROM test;", + lambda res: res == "50", + "Tested large write to testfs", + ) + limbo.run_test_fn( + "SELECT count(*) FROM vfs;", + lambda res: res == "50", + "Tested large write to testfs", + ) + print("Tested large write to testfs") + # open regular db file to ensure we don't segfault when vfs file is dropped + limbo.execute_dot(".open testing/vfs2.db") + limbo.execute_dot("create table test (id integer primary key, value float);") + limbo.execute_dot("insert into test (value) values (1.0);") + limbo.quit() + + +def test_sqlite_vfs_compat(): + sqlite = TestLimboShell( + init_commands="", + exec_name="sqlite3", + flags="testing/vfs.db", + ) + sqlite.run_test_fn( + ".show", + lambda res: "filename: testing/vfs.db" in res, + "Opened db file created with vfs extension in sqlite3", + ) + sqlite.run_test_fn( + ".schema", + lambda res: "CREATE TABLE test (id integer PRIMARY KEY, value float);" in res, + "Tables created by vfs extension exist in db file", + ) + sqlite.run_test_fn( + "SELECT count(*) FROM test;", + lambda res: res == "50", + "Tested large write to testfs", + ) + sqlite.run_test_fn( + "SELECT count(*) FROM vfs;", + lambda res: res == "50", + "Tested large write to testfs", + ) + sqlite.quit() + + +def cleanup(): + if os.path.exists("testing/vfs.db"): + os.remove("testing/vfs.db") + if os.path.exists("testing/vfs.db-wal"): + os.remove("testing/vfs.db-wal") + if __name__ == "__main__": try: @@ -477,7 +547,11 @@ if __name__ == "__main__": test_series() test_kv() test_ipaddr() + test_vfs() + test_sqlite_vfs_compat() except Exception as e: print(f"Test FAILED: {e}") + cleanup() exit(1) + cleanup() print("All tests passed successfully.") diff --git a/testing/cli_tests/test_limbo_cli.py b/testing/cli_tests/test_limbo_cli.py index ad82952a6..38186bf48 100755 --- a/testing/cli_tests/test_limbo_cli.py +++ b/testing/cli_tests/test_limbo_cli.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import os import select +from time import sleep import subprocess -from dataclasses import dataclass, field from pathlib import Path from typing import Callable, List, Optional @@ -10,16 +10,14 @@ from typing import Callable, List, Optional PIPE_BUF = 4096 -@dataclass class ShellConfig: - sqlite_exec: str = os.getenv("LIMBO_TARGET", "./target/debug/limbo") - sqlite_flags: List[str] = field( - default_factory=lambda: os.getenv("SQLITE_FLAGS", "-q").split() - ) - cwd = os.getcwd() - test_dir: Path = field(default_factory=lambda: Path("testing")) - py_folder: Path = field(default_factory=lambda: Path("cli_tests")) - test_files: Path = field(default_factory=lambda: Path("test_files")) + def __init__(self, exe_name, flags: str = "-q"): + self.sqlite_exec: str = exe_name + self.sqlite_flags: List[str] = flags.split() + self.cwd = os.getcwd() + self.test_dir: Path = Path("testing") + self.py_folder: Path = Path("cli_tests") + self.test_files: Path = Path("test_files") class LimboShell: @@ -92,14 +90,24 @@ class LimboShell: def quit(self) -> None: self._write_to_pipe(".quit") + sleep(0.3) self.pipe.terminate() + self.pipe.kill() class TestLimboShell: def __init__( - self, init_commands: Optional[str] = None, init_blobs_table: bool = False + self, + init_commands: Optional[str] = None, + init_blobs_table: bool = False, + exec_name: Optional[str] = None, + flags="", ): - self.config = ShellConfig() + if exec_name is None: + exec_name = "./target/debug/limbo" + if flags == "": + flags = "-q" + self.config = ShellConfig(exe_name=exec_name, flags=flags) if init_commands is None: # Default initialization init_commands = """ @@ -132,6 +140,11 @@ INSERT INTO t VALUES (zeroblob(1024 - 1), zeroblob(1024 - 2), zeroblob(1024 - 3) f"Actual:\n{repr(actual)}" ) + def debug_print(self, sql: str): + print(f"debugging: {sql}") + actual = self.shell.execute(sql) + print(f"OUTPUT:\n{repr(actual)}") + def run_test_fn( self, sql: str, validate: Callable[[str], bool], desc: str = "" ) -> None: