From 3d188eba0fc9d386fe41fb063f9e7637e5d55617 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sat, 18 Jan 2025 22:35:29 -0500 Subject: [PATCH 1/5] Enable staticly linking with builtin extensions --- Cargo.lock | 2 ++ core/Cargo.toml | 10 ++++++--- core/ext/mod.rs | 27 +++++++++++++++++++++++ core/lib.rs | 4 ++++ extensions/percentile/Cargo.toml | 5 ++++- extensions/uuid/Cargo.toml | 4 +++- extensions/uuid/src/lib.rs | 10 ++++++++- macros/src/args.rs | 37 +++++++++++++++++++++----------- macros/src/lib.rs | 21 ++++++++++++++---- 9 files changed, 98 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 540fb77d2..f2f9a47c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1270,6 +1270,8 @@ dependencies = [ "libloading", "limbo_ext", "limbo_macros", + "limbo_percentile", + "limbo_uuid", "log", "miette", "mimalloc", diff --git a/core/Cargo.toml b/core/Cargo.toml index c378d8bcc..0d8d6d0e2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,15 +14,17 @@ name = "limbo_core" path = "lib.rs" [features] -default = ["fs", "json", "uuid", "io_uring"] +default = ["fs", "json", "uuid", "io_uring", "percentile", "static"] fs = [] json = [ "dep:jsonb", "dep:pest", "dep:pest_derive", ] -uuid = ["dep:uuid"] +uuid = ["dep:uuid", "limbo_uuid"] io_uring = ["dep:io-uring", "rustix/io_uring"] +percentile = ["limbo_percentile"] +static = ["limbo_uuid/static", "limbo_percentile/static"] [target.'cfg(target_os = "linux")'.dependencies] io-uring = { version = "0.6.1", optional = true } @@ -33,6 +35,7 @@ rustix = "0.38.34" [target.'cfg(not(target_family = "wasm"))'.dependencies] mimalloc = { version = "*", default-features = false } +libloading = "0.8.6" [dependencies] limbo_ext = { path = "../extensions/core" } @@ -58,8 +61,9 @@ rand = "0.8.5" bumpalo = { version = "3.16.0", features = ["collections", "boxed"] } limbo_macros = { path = "../macros" } uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true } +limbo_uuid = { path = "../extensions/uuid", optional = true, features = ["static"] } +limbo_percentile = { path = "../extensions/percentile", optional = true, features = ["static"] } miette = "7.4.0" -libloading = "0.8.6" [target.'cfg(not(target_family = "windows"))'.dev-dependencies] pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] } diff --git a/core/ext/mod.rs b/core/ext/mod.rs index cf3fa6109..6dca7eb58 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -73,4 +73,31 @@ impl Database { register_aggregate_function, } } + + pub fn register_builtins(&self) -> Result<(), String> { + #[cfg(feature = "uuid")] + self.register_uuid()?; + #[cfg(feature = "percentile")] + self.register_percentile()?; + Ok(()) + } + + #[cfg(feature = "uuid")] + pub fn register_uuid(&self) -> Result<(), String> { + let ext_api = Box::new(self.build_limbo_ext()); + if unsafe { !::limbo_uuid::register_extension_static(&ext_api).is_ok() } { + return Err("Failed to register uuid extension".to_string()); + } + Ok(()) + } + + #[cfg(feature = "percentile")] + pub fn register_percentile(&self) -> Result<(), String> { + let ext_api = self.build_limbo_ext(); + let res = unsafe { ::limbo_percentile::register_extension(&ext_api) }; + if !res.is_ok() { + return Err("Failed to register percentile extension".to_string()); + } + Ok(()) + } } diff --git a/core/lib.rs b/core/lib.rs index f093762fb..977064b0b 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -1,4 +1,5 @@ mod error; +#[cfg(not(target_family = "wasm"))] mod ext; mod function; mod io; @@ -138,6 +139,9 @@ impl Database { _shared_wal: shared_wal.clone(), syms, }; + if let Err(e) = db.register_builtins() { + return Err(LimboError::ExtensionError(e)); + } let db = Arc::new(db); let conn = Rc::new(Connection { db: db.clone(), diff --git a/extensions/percentile/Cargo.toml b/extensions/percentile/Cargo.toml index 91340b813..55bd89b48 100644 --- a/extensions/percentile/Cargo.toml +++ b/extensions/percentile/Cargo.toml @@ -7,7 +7,10 @@ license.workspace = true repository.workspace = true [lib] -crate-type = ["cdylib"] +crate-type = ["cdylib", "lib"] + +[features] +static = [] [dependencies] limbo_ext = { path = "../core" } diff --git a/extensions/uuid/Cargo.toml b/extensions/uuid/Cargo.toml index 2b5a80a1c..764a75845 100644 --- a/extensions/uuid/Cargo.toml +++ b/extensions/uuid/Cargo.toml @@ -9,8 +9,10 @@ repository.workspace = true [lib] crate-type = ["cdylib", "lib"] +[features] +static= [] [dependencies] -limbo_ext = { path = "../core"} +limbo_ext = { path = "../core" } uuid = { version = "1.11.0", features = ["v4", "v7"] } log = "0.4.20" diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index 670ba8b18..cfcb879e5 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -1,7 +1,9 @@ use limbo_ext::{register_extension, scalar, Value, ValueType}; +#[cfg(feature = "static")] register_extension! { - scalars: { uuid4_str, uuid4_blob, uuid7_str, uuid7, uuid7_ts, uuid_str, uuid_blob } + static_build: true, + scalars: {uuid4_str, uuid4_blob, uuid7_str, uuid7, uuid7_ts, uuid_str, uuid_blob }, } #[scalar(name = "uuid4_str", alias = "gen_random_uuid")] @@ -133,3 +135,9 @@ fn uuid_to_unix(uuid: &[u8; 16]) -> u64 { | ((uuid[4] as u64) << 8) | (uuid[5] as u64) } + +#[cfg(not(feature = "static"))] +register_extension! { + static_build: false, + scalars: { uuid4_str, uuid4_blob, uuid7_str, uuid7, uuid7_ts, uuid_str, uuid_blob } +} diff --git a/macros/src/args.rs b/macros/src/args.rs index ec65be8b4..e81453c75 100644 --- a/macros/src/args.rs +++ b/macros/src/args.rs @@ -6,43 +6,56 @@ use syn::{Ident, LitStr, Token}; pub(crate) struct RegisterExtensionInput { pub aggregates: Vec, pub scalars: Vec, + pub is_static: bool, } impl syn::parse::Parse for RegisterExtensionInput { fn parse(input: syn::parse::ParseStream) -> syn::Result { let mut aggregates = Vec::new(); let mut scalars = Vec::new(); + let mut is_static = false; while !input.is_empty() { if input.peek(syn::Ident) && input.peek2(Token![:]) { let section_name: Ident = input.parse()?; input.parse::()?; - let content; - syn::braced!(content in input); - if section_name == "aggregates" { - aggregates = Punctuated::::parse_terminated(&content)? - .into_iter() - .collect(); - } else if section_name == "scalars" { - scalars = Punctuated::::parse_terminated(&content)? + if section_name == "static_build" { + let val: syn::LitBool = input.parse()?; + is_static = val.value; + + if input.peek(Token![,]) { + input.parse::()?; + } + } else if section_name == "aggregates" || section_name == "scalars" { + let content; + syn::braced!(content in input); + + let parsed_items = Punctuated::::parse_terminated(&content)? .into_iter() .collect(); + + if section_name == "aggregates" { + aggregates = parsed_items; + } else { + scalars = parsed_items; + } + + if input.peek(Token![,]) { + input.parse::()?; + } } else { return Err(syn::Error::new(section_name.span(), "Unknown section")); } } else { return Err(input.error("Expected aggregates: or scalars: section")); } - - if input.peek(Token![,]) { - input.parse::()?; - } } Ok(Self { aggregates, scalars, + is_static, }) } } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index ffb1e1524..c8e60ec7d 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -359,10 +359,10 @@ pub fn derive_agg_func(input: TokenStream) -> TokenStream { #[proc_macro] pub fn register_extension(input: TokenStream) -> TokenStream { let input_ast = parse_macro_input!(input as RegisterExtensionInput); - let RegisterExtensionInput { aggregates, scalars, + is_static, } = input_ast; let scalar_calls = scalars.iter().map(|scalar_ident| { @@ -390,15 +390,28 @@ pub fn register_extension(input: TokenStream) -> TokenStream { } }); - let expanded = quote! { - #[no_mangle] - pub extern "C" fn register_extension(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode { + let expanded = if is_static { + quote! { + pub unsafe extern "C" fn register_extension_static(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode { let api = unsafe { &*api }; #(#scalar_calls)* #(#aggregate_calls)* ::limbo_ext::ResultCode::OK + } + } + } else { + quote! { + #[no_mangle] + pub unsafe extern "C" fn register_extension(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode { + let api = unsafe { &*api }; + #(#scalar_calls)* + + #(#aggregate_calls)* + + ::limbo_ext::ResultCode::OK + } } }; From c1152670a305e572727d8b9c09b1f01b52fc47c4 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 19 Jan 2025 19:15:22 -0500 Subject: [PATCH 2/5] Remove manual extension registration --- core/ext/mod.rs | 2 +- extensions/percentile/src/lib.rs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/core/ext/mod.rs b/core/ext/mod.rs index 6dca7eb58..48679cb99 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -94,7 +94,7 @@ impl Database { #[cfg(feature = "percentile")] pub fn register_percentile(&self) -> Result<(), String> { let ext_api = self.build_limbo_ext(); - let res = unsafe { ::limbo_percentile::register_extension(&ext_api) }; + let res = unsafe { ::limbo_percentile::register_extension_static(&ext_api) }; if !res.is_ok() { return Err("Failed to register percentile extension".to_string()); } diff --git a/extensions/percentile/src/lib.rs b/extensions/percentile/src/lib.rs index ceecc0e3b..7d4e2b44b 100644 --- a/extensions/percentile/src/lib.rs +++ b/extensions/percentile/src/lib.rs @@ -1,6 +1,14 @@ use limbo_ext::{register_extension, AggFunc, AggregateDerive, ResultCode, Value}; +#[cfg(feature = "static")] register_extension! { + static_build: true, + aggregates: { Median, Percentile, PercentileCont, PercentileDisc } +} + +#[cfg(not(feature = "static"))] +register_extension! { + static_build: true, aggregates: { Median, Percentile, PercentileCont, PercentileDisc } } From cc63aac305d8b6d3426aa1563b38acde353bdc25 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 19 Jan 2025 22:11:49 -0500 Subject: [PATCH 3/5] Fix tests to account for built-in extensions --- Makefile | 2 +- core/lib.rs | 1 + core/types.rs | 9 ++++++++- core/vdbe/mod.rs | 12 +++++++++--- testing/extensions.py | 34 ++++------------------------------ 5 files changed, 23 insertions(+), 35 deletions(-) diff --git a/Makefile b/Makefile index 109a3f147..69fcccf83 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ test: limbo test-compat test-sqlite3 test-shell test-extensions .PHONY: test test-extensions: limbo - cargo build --package limbo_uuid + cargo build --package limbo_regexp ./testing/extensions.py .PHONY: test-extensions diff --git a/core/lib.rs b/core/lib.rs index 977064b0b..9cc739f75 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -139,6 +139,7 @@ impl Database { _shared_wal: shared_wal.clone(), syms, }; + #[cfg(not(target_family = "wasm"))] if let Err(e) = db.register_builtins() { return Err(LimboError::ExtensionError(e)); } diff --git a/core/types.rs b/core/types.rs index a3c31e1d9..d39e4045c 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1,6 +1,8 @@ +#[cfg(not(target_family = "wasm"))] use limbo_ext::{AggCtx, FinalizeFunction, StepFunction}; use crate::error::LimboError; +#[cfg(not(target_family = "wasm"))] use crate::ext::{ExtValue, ExtValueType}; use crate::storage::sqlite3_ondisk::write_varint; use crate::Result; @@ -73,7 +75,7 @@ impl OwnedValue { Self::Text(LimboText::new(text)) } } - +#[cfg(not(target_family = "wasm"))] #[derive(Debug, Clone, PartialEq)] pub struct ExternalAggState { pub state: *mut AggCtx, @@ -83,6 +85,7 @@ pub struct ExternalAggState { pub finalized_value: Option, } +#[cfg(not(target_family = "wasm"))] impl ExternalAggState { pub fn cache_final_value(&mut self, value: OwnedValue) -> &OwnedValue { self.finalized_value = Some(value); @@ -115,6 +118,7 @@ impl Display for OwnedValue { } impl OwnedValue { + #[cfg(not(target_family = "wasm"))] pub fn to_ffi(&self) -> ExtValue { match self { Self::Null => ExtValue::null(), @@ -127,6 +131,7 @@ impl OwnedValue { } } + #[cfg(not(target_family = "wasm"))] pub fn from_ffi(v: &ExtValue) -> Self { match v.value_type() { ExtValueType::Null => OwnedValue::Null, @@ -172,12 +177,14 @@ pub enum AggContext { Max(Option), Min(Option), GroupConcat(OwnedValue), + #[cfg(not(target_family = "wasm"))] External(ExternalAggState), } const NULL: OwnedValue = OwnedValue::Null; impl AggContext { + #[cfg(not(target_family = "wasm"))] pub fn compute_external(&mut self) { if let Self::External(ext_state) = self { if ext_state.finalized_value.is_none() { diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 98b328b71..5fee22ded 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -31,9 +31,9 @@ use crate::pseudo::PseudoCursor; use crate::result::LimboResult; use crate::storage::sqlite3_ondisk::DatabaseHeader; use crate::storage::{btree::BTreeCursor, pager::Pager}; -use crate::types::{ - AggContext, CursorResult, ExternalAggState, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp, -}; +#[cfg(not(target_family = "wasm"))] +use crate::types::ExternalAggState; +use crate::types::{AggContext, CursorResult, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp}; use crate::util::parse_schema_rows; use crate::vdbe::builder::CursorType; use crate::vdbe::insn::Insn; @@ -1312,6 +1312,8 @@ impl Program { OwnedValue::build_text(Rc::new("".to_string())), ))) } + + #[cfg(not(target_family = "wasm"))] AggFunc::External(func) => match func.as_ref() { ExtFunc::Aggregate { init, @@ -1473,6 +1475,8 @@ impl Program { *acc += col; } } + + #[cfg(not(target_family = "wasm"))] AggFunc::External(_) => { let (step_fn, state_ptr, argc) = { let OwnedValue::Agg(agg) = &state.registers[*acc_reg] else { @@ -1512,6 +1516,7 @@ impl Program { AggFunc::Max => {} AggFunc::Min => {} AggFunc::GroupConcat | AggFunc::StringAgg => {} + #[cfg(not(target_family = "wasm"))] AggFunc::External(_) => { agg.compute_external(); } @@ -2006,6 +2011,7 @@ impl Program { } } }, + #[cfg(not(target_family = "wasm"))] crate::function::Func::External(f) => match f.func { ExtFunc::Scalar(f) => { call_external_function! {f, *dest, state, arg_count, *start_reg }; diff --git a/testing/extensions.py b/testing/extensions.py index 9dcd27846..915465e9b 100755 --- a/testing/extensions.py +++ b/testing/extensions.py @@ -133,22 +133,8 @@ def assert_specific_time(result): def test_uuid(pipe): specific_time = "01945ca0-3189-76c0-9a8f-caf310fc8b8e" - extension_path = "./target/debug/liblimbo_uuid.so" - # before extension loads, assert no function - run_test( - pipe, - "SELECT uuid4();", - returns_error, - "uuid functions return null when ext not loaded", - ) - run_test(pipe, "SELECT uuid4_str();", returns_error) - run_test( - pipe, - f".load {extension_path}", - returns_null, - "load extension command works properly", - ) + # these are built into the binary, so we just test they work run_test( pipe, "SELECT hex(uuid4());", @@ -224,20 +210,7 @@ def validate_percentile_disc(res): def test_aggregates(pipe): - extension_path = "./target/debug/liblimbo_percentile.so" - # assert no function before extension loads - run_test( - pipe, - "SELECT median(1);", - returns_error, - "median agg function returns null when ext not loaded", - ) - run_test( - pipe, - f".load {extension_path}", - returns_null, - "load extension command works properly", - ) + # also built-in run_test( pipe, "select median(value) from numbers;", @@ -286,4 +259,5 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() + From f13d0359650fe2d0112d71587143ff6f492391d1 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Sun, 19 Jan 2025 22:44:31 -0500 Subject: [PATCH 4/5] Enable wasm to static link extensions --- Cargo.lock | 1 - core/Cargo.toml | 8 +++----- core/ext/mod.rs | 5 ++--- core/lib.rs | 4 ---- core/types.rs | 9 +-------- core/vdbe/mod.rs | 7 ------- extensions/core/Cargo.toml | 5 +++++ extensions/percentile/Cargo.toml | 4 ++-- extensions/percentile/src/lib.rs | 8 -------- extensions/uuid/Cargo.toml | 4 ++-- extensions/uuid/src/lib.rs | 8 -------- macros/Cargo.toml | 2 ++ macros/src/args.rs | 12 +----------- macros/src/lib.rs | 17 ++++++++--------- 14 files changed, 26 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2f9a47c2..8b10ef885 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1291,7 +1291,6 @@ dependencies = [ "sqlite3-parser", "tempfile", "thiserror 1.0.69", - "uuid", ] [[package]] diff --git a/core/Cargo.toml b/core/Cargo.toml index 0d8d6d0e2..81f3b454c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,17 +14,16 @@ name = "limbo_core" path = "lib.rs" [features] -default = ["fs", "json", "uuid", "io_uring", "percentile", "static"] +default = ["fs", "json", "uuid", "io_uring", "percentile"] fs = [] json = [ "dep:jsonb", "dep:pest", "dep:pest_derive", ] -uuid = ["dep:uuid", "limbo_uuid"] +uuid = ["limbo_uuid/static"] io_uring = ["dep:io-uring", "rustix/io_uring"] -percentile = ["limbo_percentile"] -static = ["limbo_uuid/static", "limbo_percentile/static"] +percentile = ["limbo_percentile/static"] [target.'cfg(target_os = "linux")'.dependencies] io-uring = { version = "0.6.1", optional = true } @@ -60,7 +59,6 @@ pest_derive = { version = "2.0", optional = true } rand = "0.8.5" bumpalo = { version = "3.16.0", features = ["collections", "boxed"] } limbo_macros = { path = "../macros" } -uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true } limbo_uuid = { path = "../extensions/uuid", optional = true, features = ["static"] } limbo_percentile = { path = "../extensions/percentile", optional = true, features = ["static"] } miette = "7.4.0" diff --git a/core/ext/mod.rs b/core/ext/mod.rs index 48679cb99..377853769 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -85,7 +85,7 @@ impl Database { #[cfg(feature = "uuid")] pub fn register_uuid(&self) -> Result<(), String> { let ext_api = Box::new(self.build_limbo_ext()); - if unsafe { !::limbo_uuid::register_extension_static(&ext_api).is_ok() } { + if unsafe { !limbo_uuid::register_extension_static(&ext_api).is_ok() } { return Err("Failed to register uuid extension".to_string()); } Ok(()) @@ -94,8 +94,7 @@ impl Database { #[cfg(feature = "percentile")] pub fn register_percentile(&self) -> Result<(), String> { let ext_api = self.build_limbo_ext(); - let res = unsafe { ::limbo_percentile::register_extension_static(&ext_api) }; - if !res.is_ok() { + if unsafe { !limbo_percentile::register_extension_static(&ext_api).is_ok() } { return Err("Failed to register percentile extension".to_string()); } Ok(()) diff --git a/core/lib.rs b/core/lib.rs index 9cc739f75..83cda335f 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -1,5 +1,4 @@ mod error; -#[cfg(not(target_family = "wasm"))] mod ext; mod function; mod io; @@ -22,7 +21,6 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; use fallible_iterator::FallibleIterator; #[cfg(not(target_family = "wasm"))] use libloading::{Library, Symbol}; -#[cfg(not(target_family = "wasm"))] use limbo_ext::{ExtensionApi, ExtensionEntryPoint}; use log::trace; use schema::Schema; @@ -139,7 +137,6 @@ impl Database { _shared_wal: shared_wal.clone(), syms, }; - #[cfg(not(target_family = "wasm"))] if let Err(e) = db.register_builtins() { return Err(LimboError::ExtensionError(e)); } @@ -565,7 +562,6 @@ impl SymbolTable { pub fn new() -> Self { Self { functions: HashMap::new(), - // TODO: wasm libs will be very different #[cfg(not(target_family = "wasm"))] extensions: Vec::new(), } diff --git a/core/types.rs b/core/types.rs index d39e4045c..a3c31e1d9 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1,8 +1,6 @@ -#[cfg(not(target_family = "wasm"))] use limbo_ext::{AggCtx, FinalizeFunction, StepFunction}; use crate::error::LimboError; -#[cfg(not(target_family = "wasm"))] use crate::ext::{ExtValue, ExtValueType}; use crate::storage::sqlite3_ondisk::write_varint; use crate::Result; @@ -75,7 +73,7 @@ impl OwnedValue { Self::Text(LimboText::new(text)) } } -#[cfg(not(target_family = "wasm"))] + #[derive(Debug, Clone, PartialEq)] pub struct ExternalAggState { pub state: *mut AggCtx, @@ -85,7 +83,6 @@ pub struct ExternalAggState { pub finalized_value: Option, } -#[cfg(not(target_family = "wasm"))] impl ExternalAggState { pub fn cache_final_value(&mut self, value: OwnedValue) -> &OwnedValue { self.finalized_value = Some(value); @@ -118,7 +115,6 @@ impl Display for OwnedValue { } impl OwnedValue { - #[cfg(not(target_family = "wasm"))] pub fn to_ffi(&self) -> ExtValue { match self { Self::Null => ExtValue::null(), @@ -131,7 +127,6 @@ impl OwnedValue { } } - #[cfg(not(target_family = "wasm"))] pub fn from_ffi(v: &ExtValue) -> Self { match v.value_type() { ExtValueType::Null => OwnedValue::Null, @@ -177,14 +172,12 @@ pub enum AggContext { Max(Option), Min(Option), GroupConcat(OwnedValue), - #[cfg(not(target_family = "wasm"))] External(ExternalAggState), } const NULL: OwnedValue = OwnedValue::Null; impl AggContext { - #[cfg(not(target_family = "wasm"))] pub fn compute_external(&mut self) { if let Self::External(ext_state) = self { if ext_state.finalized_value.is_none() { diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 5fee22ded..ebb7bd240 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -31,7 +31,6 @@ use crate::pseudo::PseudoCursor; use crate::result::LimboResult; use crate::storage::sqlite3_ondisk::DatabaseHeader; use crate::storage::{btree::BTreeCursor, pager::Pager}; -#[cfg(not(target_family = "wasm"))] use crate::types::ExternalAggState; use crate::types::{AggContext, CursorResult, OwnedRecord, OwnedValue, Record, SeekKey, SeekOp}; use crate::util::parse_schema_rows; @@ -1312,8 +1311,6 @@ impl Program { OwnedValue::build_text(Rc::new("".to_string())), ))) } - - #[cfg(not(target_family = "wasm"))] AggFunc::External(func) => match func.as_ref() { ExtFunc::Aggregate { init, @@ -1475,8 +1472,6 @@ impl Program { *acc += col; } } - - #[cfg(not(target_family = "wasm"))] AggFunc::External(_) => { let (step_fn, state_ptr, argc) = { let OwnedValue::Agg(agg) = &state.registers[*acc_reg] else { @@ -1516,7 +1511,6 @@ impl Program { AggFunc::Max => {} AggFunc::Min => {} AggFunc::GroupConcat | AggFunc::StringAgg => {} - #[cfg(not(target_family = "wasm"))] AggFunc::External(_) => { agg.compute_external(); } @@ -2011,7 +2005,6 @@ impl Program { } } }, - #[cfg(not(target_family = "wasm"))] crate::function::Func::External(f) => match f.func { ExtFunc::Scalar(f) => { call_external_function! {f, *dest, state, arg_count, *start_reg }; diff --git a/extensions/core/Cargo.toml b/extensions/core/Cargo.toml index f56436f5c..3194bcadb 100644 --- a/extensions/core/Cargo.toml +++ b/extensions/core/Cargo.toml @@ -6,6 +6,11 @@ edition.workspace = true license.workspace = true repository.workspace = true + +[features] +default = [] +static = [] + [dependencies] log = "0.4.20" limbo_macros = { path = "../../macros" } diff --git a/extensions/percentile/Cargo.toml b/extensions/percentile/Cargo.toml index 55bd89b48..499dec542 100644 --- a/extensions/percentile/Cargo.toml +++ b/extensions/percentile/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true crate-type = ["cdylib", "lib"] [features] -static = [] +static = ["limbo_ext/static"] [dependencies] -limbo_ext = { path = "../core" } +limbo_ext = { path = "../core", features = ["static"] } diff --git a/extensions/percentile/src/lib.rs b/extensions/percentile/src/lib.rs index 7d4e2b44b..ceecc0e3b 100644 --- a/extensions/percentile/src/lib.rs +++ b/extensions/percentile/src/lib.rs @@ -1,14 +1,6 @@ use limbo_ext::{register_extension, AggFunc, AggregateDerive, ResultCode, Value}; -#[cfg(feature = "static")] register_extension! { - static_build: true, - aggregates: { Median, Percentile, PercentileCont, PercentileDisc } -} - -#[cfg(not(feature = "static"))] -register_extension! { - static_build: true, aggregates: { Median, Percentile, PercentileCont, PercentileDisc } } diff --git a/extensions/uuid/Cargo.toml b/extensions/uuid/Cargo.toml index 764a75845..baf9a3a33 100644 --- a/extensions/uuid/Cargo.toml +++ b/extensions/uuid/Cargo.toml @@ -10,9 +10,9 @@ repository.workspace = true crate-type = ["cdylib", "lib"] [features] -static= [] +static= [ "limbo_ext/static" ] [dependencies] -limbo_ext = { path = "../core" } +limbo_ext = { path = "../core", features = ["static"] } uuid = { version = "1.11.0", features = ["v4", "v7"] } log = "0.4.20" diff --git a/extensions/uuid/src/lib.rs b/extensions/uuid/src/lib.rs index cfcb879e5..5a69dec76 100644 --- a/extensions/uuid/src/lib.rs +++ b/extensions/uuid/src/lib.rs @@ -1,8 +1,6 @@ use limbo_ext::{register_extension, scalar, Value, ValueType}; -#[cfg(feature = "static")] register_extension! { - static_build: true, scalars: {uuid4_str, uuid4_blob, uuid7_str, uuid7, uuid7_ts, uuid_str, uuid_blob }, } @@ -135,9 +133,3 @@ fn uuid_to_unix(uuid: &[u8; 16]) -> u64 { | ((uuid[4] as u64) << 8) | (uuid[5] as u64) } - -#[cfg(not(feature = "static"))] -register_extension! { - static_build: false, - scalars: { uuid4_str, uuid4_blob, uuid7_str, uuid7, uuid7_ts, uuid_str, uuid_blob } -} diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 71a300337..3aa895d9b 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -12,6 +12,8 @@ description = "The Limbo database library" [lib] proc-macro = true +[features] + [dependencies] quote = "1.0.38" proc-macro2 = "1.0.93" diff --git a/macros/src/args.rs b/macros/src/args.rs index e81453c75..d9e59cbd3 100644 --- a/macros/src/args.rs +++ b/macros/src/args.rs @@ -6,28 +6,19 @@ use syn::{Ident, LitStr, Token}; pub(crate) struct RegisterExtensionInput { pub aggregates: Vec, pub scalars: Vec, - pub is_static: bool, } impl syn::parse::Parse for RegisterExtensionInput { fn parse(input: syn::parse::ParseStream) -> syn::Result { let mut aggregates = Vec::new(); let mut scalars = Vec::new(); - let mut is_static = false; while !input.is_empty() { if input.peek(syn::Ident) && input.peek2(Token![:]) { let section_name: Ident = input.parse()?; input.parse::()?; - if section_name == "static_build" { - let val: syn::LitBool = input.parse()?; - is_static = val.value; - - if input.peek(Token![,]) { - input.parse::()?; - } - } else if section_name == "aggregates" || section_name == "scalars" { + if section_name == "aggregates" || section_name == "scalars" { let content; syn::braced!(content in input); @@ -55,7 +46,6 @@ impl syn::parse::Parse for RegisterExtensionInput { Ok(Self { aggregates, scalars, - is_static, }) } } diff --git a/macros/src/lib.rs b/macros/src/lib.rs index c8e60ec7d..53a5724ee 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -362,7 +362,6 @@ pub fn register_extension(input: TokenStream) -> TokenStream { let RegisterExtensionInput { aggregates, scalars, - is_static, } = input_ast; let scalar_calls = scalars.iter().map(|scalar_ident| { @@ -389,20 +388,21 @@ pub fn register_extension(input: TokenStream) -> TokenStream { } } }); + let static_aggregates = aggregate_calls.clone(); + let static_scalars = scalar_calls.clone(); - let expanded = if is_static { - quote! { + let expanded = quote! { + #[cfg(feature = "static")] pub unsafe extern "C" fn register_extension_static(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode { let api = unsafe { &*api }; - #(#scalar_calls)* + #(#static_scalars)* - #(#aggregate_calls)* + #(#static_aggregates)* ::limbo_ext::ResultCode::OK } - } - } else { - quote! { + + #[cfg(not(feature = "static"))] #[no_mangle] pub unsafe extern "C" fn register_extension(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode { let api = unsafe { &*api }; @@ -412,7 +412,6 @@ pub fn register_extension(input: TokenStream) -> TokenStream { ::limbo_ext::ResultCode::OK } - } }; TokenStream::from(expanded) From c5e60d8e08457cc4feb5335e443458c75dd83137 Mon Sep 17 00:00:00 2001 From: PThorpe92 Date: Tue, 21 Jan 2025 10:06:05 -0500 Subject: [PATCH 5/5] Enable only uuid by default, change tests back to account for this --- Cargo.lock | 1 + core/Cargo.toml | 4 +++- core/ext/mod.rs | 21 ++++++--------------- extensions/regexp/Cargo.toml | 5 ++++- testing/extensions.py | 17 ++++++++++++++--- 5 files changed, 28 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b10ef885..500a8ce3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1271,6 +1271,7 @@ dependencies = [ "limbo_ext", "limbo_macros", "limbo_percentile", + "limbo_regexp", "limbo_uuid", "log", "miette", diff --git a/core/Cargo.toml b/core/Cargo.toml index 81f3b454c..694617ec3 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,7 +14,7 @@ name = "limbo_core" path = "lib.rs" [features] -default = ["fs", "json", "uuid", "io_uring", "percentile"] +default = ["fs", "json", "uuid", "io_uring"] fs = [] json = [ "dep:jsonb", @@ -24,6 +24,7 @@ json = [ uuid = ["limbo_uuid/static"] io_uring = ["dep:io-uring", "rustix/io_uring"] percentile = ["limbo_percentile/static"] +regexp = ["limbo_regexp/static"] [target.'cfg(target_os = "linux")'.dependencies] io-uring = { version = "0.6.1", optional = true } @@ -60,6 +61,7 @@ rand = "0.8.5" bumpalo = { version = "3.16.0", features = ["collections", "boxed"] } limbo_macros = { path = "../macros" } limbo_uuid = { path = "../extensions/uuid", optional = true, features = ["static"] } +limbo_regexp = { path = "../extensions/regexp", optional = true, features = ["static"] } limbo_percentile = { path = "../extensions/percentile", optional = true, features = ["static"] } miette = "7.4.0" diff --git a/core/ext/mod.rs b/core/ext/mod.rs index 377853769..cbd2fa258 100644 --- a/core/ext/mod.rs +++ b/core/ext/mod.rs @@ -75,28 +75,19 @@ impl Database { } pub fn register_builtins(&self) -> Result<(), String> { + let ext_api = self.build_limbo_ext(); #[cfg(feature = "uuid")] - self.register_uuid()?; - #[cfg(feature = "percentile")] - self.register_percentile()?; - Ok(()) - } - - #[cfg(feature = "uuid")] - pub fn register_uuid(&self) -> Result<(), String> { - let ext_api = Box::new(self.build_limbo_ext()); if unsafe { !limbo_uuid::register_extension_static(&ext_api).is_ok() } { return Err("Failed to register uuid extension".to_string()); } - Ok(()) - } - - #[cfg(feature = "percentile")] - pub fn register_percentile(&self) -> Result<(), String> { - let ext_api = self.build_limbo_ext(); + #[cfg(feature = "percentile")] if unsafe { !limbo_percentile::register_extension_static(&ext_api).is_ok() } { return Err("Failed to register percentile extension".to_string()); } + #[cfg(feature = "regexp")] + if unsafe { !limbo_regexp::register_extension_static(&ext_api).is_ok() } { + return Err("Failed to register regexp extension".to_string()); + } Ok(()) } } diff --git a/extensions/regexp/Cargo.toml b/extensions/regexp/Cargo.toml index dc2f87c3b..9ca5c222f 100644 --- a/extensions/regexp/Cargo.toml +++ b/extensions/regexp/Cargo.toml @@ -6,11 +6,14 @@ edition.workspace = true license.workspace = true repository.workspace = true +[features] +static = ["limbo_ext/static"] + [lib] crate-type = ["cdylib", "lib"] [dependencies] -limbo_ext = { path = "../core"} +limbo_ext = { path = "../core", features = ["static"] } regex = "1.11.1" log = "0.4.20" diff --git a/testing/extensions.py b/testing/extensions.py index 915465e9b..74755a012 100755 --- a/testing/extensions.py +++ b/testing/extensions.py @@ -133,7 +133,6 @@ def assert_specific_time(result): def test_uuid(pipe): specific_time = "01945ca0-3189-76c0-9a8f-caf310fc8b8e" - # these are built into the binary, so we just test they work run_test( pipe, @@ -210,7 +209,20 @@ def validate_percentile_disc(res): def test_aggregates(pipe): - # also built-in + extension_path = "./target/debug/liblimbo_percentile.so" + # assert no function before extension loads + run_test( + pipe, + "SELECT median(1);", + returns_error, + "median agg function returns null when ext not loaded", + ) + run_test( + pipe, + f".load {extension_path}", + returns_null, + "load extension command works properly", + ) run_test( pipe, "select median(value) from numbers;", @@ -260,4 +272,3 @@ def main(): if __name__ == "__main__": main() -