Merge 'Enable static linking for 'built-in' extensions' from Preston Thorpe

This PR introduces the ability to build and link with an extension
library, enabling features like `uuid` to not have to be shipped as
independent libraries and loaded at runtime.
To build and link with an extension, you simply add it as a dependency
with the `static` feature, and call register_extension_static. in this
case, we feature flag that with `uuid`
```rust
    #[cfg(feature = "uuid")]
    pub fn register_uuid(&self) -> Result<(), String> {
        let ext_api = Box::new(self.build_limbo_ext());
        if unsafe { !limbo_uuid::register_extension_static(&ext_api).is_ok() } {
            return Err("Failed to register uuid extension".to_string());
        }
        Ok(())
    }
```
So fortunately wasm targets are no longer excluded from extensions, only
loading them at runtime for now

Closes #737

Cargo.lock

@@ -1339,6 +1339,9 @@ dependencies = [
  "libloading",
  "limbo_ext",
  "limbo_macros",
+ "limbo_percentile",
+ "limbo_regexp",
+ "limbo_uuid",
  "log",
  "miette",
  "mimalloc",
@@ -1358,7 +1361,6 @@ dependencies = [
  "sqlite3-parser",
  "tempfile",
  "thiserror 1.0.69",
- "uuid",
 ]
 
 [[package]]

Makefile

@@ -66,7 +66,7 @@ test: limbo test-compat test-sqlite3 test-shell test-extensions
 .PHONY: test
 
 test-extensions: limbo
-	cargo build --package limbo_uuid
+	cargo build --package limbo_regexp
 	./testing/extensions.py
 .PHONY: test-extensions
 

core/Cargo.toml

@@ -21,8 +21,10 @@ json = [
     "dep:pest",
     "dep:pest_derive",
 ]
-uuid = ["dep:uuid"]
+uuid = ["limbo_uuid/static"]
 io_uring = ["dep:io-uring", "rustix/io_uring"]
+percentile = ["limbo_percentile/static"]
+regexp = ["limbo_regexp/static"]
 
 [target.'cfg(target_os = "linux")'.dependencies]
 io-uring = { version = "0.6.1", optional = true }
@@ -33,6 +35,7 @@ rustix = "0.38.34"
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 mimalloc = { version = "*", default-features = false }
+libloading = "0.8.6"
 
 [dependencies]
 limbo_ext = { path = "../extensions/core" }
@@ -57,9 +60,10 @@ pest_derive = { version = "2.0", optional = true }
 rand = "0.8.5"
 bumpalo = { version = "3.16.0", features = ["collections", "boxed"] }
 limbo_macros = { path = "../macros" }
-uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true }
+limbo_uuid = { path = "../extensions/uuid", optional = true, features = ["static"] }
+limbo_regexp = { path = "../extensions/regexp", optional = true, features = ["static"] }
+limbo_percentile = { path = "../extensions/percentile", optional = true, features = ["static"] }
 miette = "7.4.0"
-libloading = "0.8.6"
 
 [target.'cfg(not(target_family = "windows"))'.dev-dependencies]
 pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] }

core/ext/mod.rs

@@ -73,4 +73,21 @@ impl Database {
             register_aggregate_function,
         }
     }
+
+    pub fn register_builtins(&self) -> Result<(), String> {
+        let ext_api = self.build_limbo_ext();
+        #[cfg(feature = "uuid")]
+        if unsafe { !limbo_uuid::register_extension_static(&ext_api).is_ok() } {
+            return Err("Failed to register uuid extension".to_string());
+        }
+        #[cfg(feature = "percentile")]
+        if unsafe { !limbo_percentile::register_extension_static(&ext_api).is_ok() } {
+            return Err("Failed to register percentile extension".to_string());
+        }
+        #[cfg(feature = "regexp")]
+        if unsafe { !limbo_regexp::register_extension_static(&ext_api).is_ok() } {
+            return Err("Failed to register regexp extension".to_string());
+        }
+        Ok(())
+    }
 }

core/lib.rs

@@ -21,7 +21,6 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 use fallible_iterator::FallibleIterator;
 #[cfg(not(target_family = "wasm"))]
 use libloading::{Library, Symbol};
-#[cfg(not(target_family = "wasm"))]
 use limbo_ext::{ExtensionApi, ExtensionEntryPoint};
 use log::trace;
 use schema::Schema;
@@ -138,6 +137,9 @@ impl Database {
             _shared_wal: shared_wal.clone(),
             syms,
         };
+        if let Err(e) = db.register_builtins() {
+            return Err(LimboError::ExtensionError(e));
+        }
         let db = Arc::new(db);
         let conn = Rc::new(Connection {
             db: db.clone(),
@@ -557,7 +559,6 @@ impl SymbolTable {
     pub fn new() -> Self {
         Self {
             functions: HashMap::new(),
-            // TODO: wasm libs will be very different
             #[cfg(not(target_family = "wasm"))]
             extensions: Vec::new(),
         }

extensions/core/Cargo.toml

@@ -6,6 +6,11 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+
+[features]
+default = []
+static = []
+
 [dependencies]
 log = "0.4.20"
 limbo_macros = { path = "../../macros" }

extensions/percentile/Cargo.toml

@@ -7,7 +7,10 @@ license.workspace = true
 repository.workspace = true
 
 [lib]
-crate-type = ["cdylib"]
+crate-type = ["cdylib", "lib"]
+
+[features]
+static = ["limbo_ext/static"]
 
 [dependencies]
-limbo_ext = { path = "../core" }
+limbo_ext = { path = "../core", features = ["static"] }

extensions/regexp/Cargo.toml

@@ -6,11 +6,14 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+[features]
+static = ["limbo_ext/static"]
+
 [lib]
 crate-type = ["cdylib", "lib"]
 
 
 [dependencies]
-limbo_ext = { path = "../core"}
+limbo_ext = { path = "../core", features = ["static"] }
 regex = "1.11.1"
 log = "0.4.20"

extensions/uuid/Cargo.toml

@@ -9,8 +9,10 @@ repository.workspace = true
 [lib]
 crate-type = ["cdylib", "lib"]
 
+[features]
+static= [ "limbo_ext/static" ]
 
 [dependencies]
-limbo_ext = { path = "../core"}
+limbo_ext = { path = "../core", features = ["static"] }
 uuid = { version = "1.11.0", features = ["v4", "v7"] }
 log = "0.4.20"

extensions/uuid/src/lib.rs

@@ -1,7 +1,7 @@
 use limbo_ext::{register_extension, scalar, Value, ValueType};
 
 register_extension! {
-    scalars: { uuid4_str, uuid4_blob, uuid7_str, uuid7, uuid7_ts, uuid_str, uuid_blob }
+    scalars: {uuid4_str, uuid4_blob, uuid7_str, uuid7, uuid7_ts, uuid_str, uuid_blob },
 }
 
 #[scalar(name = "uuid4_str", alias = "gen_random_uuid")]

macros/Cargo.toml

@@ -12,6 +12,8 @@ description = "The Limbo database library"
 [lib]
 proc-macro = true
 
+[features]
+
 [dependencies]
 quote = "1.0.38"
 proc-macro2 = "1.0.93"

macros/src/args.rs

@@ -17,27 +17,30 @@ impl syn::parse::Parse for RegisterExtensionInput {
             if input.peek(syn::Ident) && input.peek2(Token![:]) {
                 let section_name: Ident = input.parse()?;
                 input.parse::<Token![:]>()?;
-                let content;
-                syn::braced!(content in input);
 
-                if section_name == "aggregates" {
-                    aggregates = Punctuated::<Ident, Token![,]>::parse_terminated(&content)?
-                        .into_iter()
-                        .collect();
-                } else if section_name == "scalars" {
-                    scalars = Punctuated::<Ident, Token![,]>::parse_terminated(&content)?
+                if section_name == "aggregates" || section_name == "scalars" {
+                    let content;
+                    syn::braced!(content in input);
+
+                    let parsed_items = Punctuated::<Ident, Token![,]>::parse_terminated(&content)?
                         .into_iter()
                         .collect();
+
+                    if section_name == "aggregates" {
+                        aggregates = parsed_items;
+                    } else {
+                        scalars = parsed_items;
+                    }
+
+                    if input.peek(Token![,]) {
+                        input.parse::<Token![,]>()?;
+                    }
                 } else {
                     return Err(syn::Error::new(section_name.span(), "Unknown section"));
                 }
             } else {
                 return Err(input.error("Expected aggregates: or scalars: section"));
             }
-
-            if input.peek(Token![,]) {
-                input.parse::<Token![,]>()?;
-            }
         }
 
         Ok(Self {

macros/src/lib.rs

@@ -359,7 +359,6 @@ pub fn derive_agg_func(input: TokenStream) -> TokenStream {
 #[proc_macro]
 pub fn register_extension(input: TokenStream) -> TokenStream {
     let input_ast = parse_macro_input!(input as RegisterExtensionInput);
-
     let RegisterExtensionInput {
         aggregates,
         scalars,
@@ -389,17 +388,30 @@ pub fn register_extension(input: TokenStream) -> TokenStream {
             }
         }
     });
+    let static_aggregates = aggregate_calls.clone();
+    let static_scalars = scalar_calls.clone();
 
     let expanded = quote! {
-        #[no_mangle]
-        pub extern "C" fn register_extension(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode {
+        #[cfg(feature = "static")]
+        pub unsafe extern "C" fn register_extension_static(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode {
             let api = unsafe { &*api };
-            #(#scalar_calls)*
+            #(#static_scalars)*
 
-            #(#aggregate_calls)*
+            #(#static_aggregates)*
 
             ::limbo_ext::ResultCode::OK
-        }
+          }
+
+        #[cfg(not(feature = "static"))]
+            #[no_mangle]
+            pub unsafe extern "C" fn register_extension(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode {
+                let api = unsafe { &*api };
+                #(#scalar_calls)*
+
+                #(#aggregate_calls)*
+
+                ::limbo_ext::ResultCode::OK
+            }
     };
 
     TokenStream::from(expanded)

testing/extensions.py

@@ -133,22 +133,7 @@ def assert_specific_time(result):
 
 def test_uuid(pipe):
     specific_time = "01945ca0-3189-76c0-9a8f-caf310fc8b8e"
-    extension_path = "./target/debug/liblimbo_uuid.so"
-
-    # before extension loads, assert no function
-    run_test(
-        pipe,
-        "SELECT uuid4();",
-        returns_error,
-        "uuid functions return null when ext not loaded",
-    )
-    run_test(pipe, "SELECT uuid4_str();", returns_error)
-    run_test(
-        pipe,
-        f".load {extension_path}",
-        returns_null,
-        "load extension command works properly",
-    )
+    # these are built into the binary, so we just test they work
     run_test(
         pipe,
         "SELECT hex(uuid4());",
@@ -286,4 +271,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()