Add basic structure for crypto extension

Cargo.lock

@@ -140,6 +140,12 @@ version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
 
+[[package]]
+name = "arrayref"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
+
 [[package]]
 name = "arrayvec"
 version = "0.7.6"
@@ -204,6 +210,19 @@ version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
 
+[[package]]
+name = "blake3"
+version = "1.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -455,6 +474,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "constant_time_eq"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+
 [[package]]
 name = "core-foundation-sys"
 version = "0.8.7"
@@ -1590,6 +1615,7 @@ dependencies = [
  "julian_day_converter",
  "libc",
  "libloading",
+ "limbo_crypto",
  "limbo_ext",
  "limbo_macros",
  "limbo_percentile",
@@ -1621,6 +1647,16 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "limbo_crypto"
+version = "0.0.14"
+dependencies = [
+ "blake3",
+ "limbo_ext",
+ "mimalloc",
+ "ring",
+]
+
 [[package]]
 name = "limbo_ext"
 version = "0.0.14"
@@ -2538,6 +2574,21 @@ dependencies = [
  "bytemuck",
 ]
 
+[[package]]
+name = "ring"
+version = "0.17.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.15",
+ "libc",
+ "spin",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "rstest"
 version = "0.18.2"
@@ -2761,6 +2812,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
 [[package]]
 name = "sqlite3-parser"
 version = "0.13.0"
@@ -3130,6 +3187,12 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
 
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
 [[package]]
 name = "url"
 version = "2.5.4"

Cargo.toml

@@ -20,6 +20,7 @@ members = [
     "extensions/percentile",
     "extensions/vector", 
     "extensions/time",
+    "extensions/crypto",
 ]
 exclude = ["perf/latency/limbo"]
 

core/Cargo.toml

@@ -14,7 +14,7 @@ name = "limbo_core"
 path = "lib.rs"
 
 [features]
-default = ["fs", "json", "uuid", "vector", "io_uring", "time"]
+default = ["fs", "json", "uuid", "vector", "io_uring", "time", "crypto"]
 fs = []
 json = [
     "dep:jsonb",
@@ -27,6 +27,7 @@ io_uring = ["dep:io-uring", "rustix/io_uring"]
 percentile = ["limbo_percentile/static"]
 regexp = ["limbo_regexp/static"]
 time = ["limbo_time/static"]
+crypto = ["limbo_crypto/static"]
 
 [target.'cfg(target_os = "linux")'.dependencies]
 io-uring = { version = "0.6.1", optional = true }
@@ -67,6 +68,7 @@ limbo_vector = { path = "../extensions/vector", optional = true, features = ["st
 limbo_regexp = { path = "../extensions/regexp", optional = true, features = ["static"] }
 limbo_percentile = { path = "../extensions/percentile", optional = true, features = ["static"] }
 limbo_time = { path = "../extensions/time", optional = true, features = ["static"] }
+limbo_crypto = { path = "../extensions/crypto", optional = true, features = ["static"] }
 miette = "7.4.0"
 strum = "0.26"
 parking_lot = "0.12.3"

core/ext/mod.rs

@@ -96,6 +96,10 @@ impl Database {
         if unsafe { !limbo_time::register_extension_static(&ext_api).is_ok() } {
             return Err("Failed to register time extension".to_string());
         }
+        #[cfg(feature = "crypto")]
+        if unsafe { !limbo_crypto::register_extension_static(&ext_api).is_ok() } {
+            return Err("Failed to register crypto extension".to_string());
+        }
         Ok(())
     }
 }

extensions/core/src/types.rs

@@ -1,4 +1,4 @@
-use std::fmt::Display;
+use std::{fmt::Display, mem};
 
 /// Error type is of type ExtError which can be
 /// either a user defined error or an error code
@@ -204,6 +204,13 @@ impl Blob {
     pub fn new(data: *const u8, size: u64) -> Self {
         Self { data, size }
     }
+
+    pub fn as_bytes(&self) -> &[u8] {
+        if self.data.is_null() {
+            return &[];
+        }
+        unsafe { std::slice::from_raw_parts(self.data, self.size as usize) }
+    }
 }
 
 impl Value {
@@ -303,6 +310,29 @@ impl Value {
         }
     }
 
+    // Return ValueData as raw bytes
+    pub fn as_bytes(&self) -> Vec<u8> {
+        let mut bytes = vec![];
+
+        unsafe {
+            match self.value_type {
+                ValueType::Integer => bytes.extend_from_slice(&self.value.int.to_le_bytes()),
+                ValueType::Float => bytes.extend_from_slice(&self.value.float.to_le_bytes()),
+                ValueType::Text => {
+                    let text = self.value.text.as_ref().expect("Invalid text pointer");
+                    bytes.extend_from_slice(text.as_str().as_bytes());
+                }
+                ValueType::Blob => {
+                    let blob = self.value.blob.as_ref().expect("Invalid blob pointer");
+                    bytes.extend_from_slice(blob.as_bytes());
+                }
+                ValueType::Error | ValueType::Null => {}
+            }
+        }
+
+        bytes
+    }
+
     /// Creates a new integer Value from an i64
     pub fn from_integer(i: i64) -> Self {
         Self {

extensions/crypto/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "limbo_crypto"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[lib]
+crate-type = ["cdylib", "lib"]
+
+[features]
+static= [ "limbo_ext/static" ]
+
+[dependencies]
+blake3 = "1.5.5"
+limbo_ext = { path = "../core", features = ["static"] }
+ring = "0.17.8"
+
+[target.'cfg(not(target_family = "wasm"))'.dependencies]
+mimalloc = { version = "*", default-features = false }

extensions/crypto/src/crypto.rs

@@ -0,0 +1,55 @@
+use crate::Error;
+use blake3::Hasher;
+use limbo_ext::{Value, ValueType};
+use ring::digest::{self, digest};
+
+pub fn sha256(data: &Value) -> Result<Vec<u8>, Error> {
+    match data.value_type() {
+        ValueType::Error | ValueType::Null => Err(Error::InvalidType),
+        _ => {
+            let hash = digest(&digest::SHA256, &data.as_bytes());
+            Ok(hash.as_ref().to_vec())
+        }
+    }
+}
+
+pub fn sha512(data: &Value) -> Result<Vec<u8>, Error> {
+    match data.value_type() {
+        ValueType::Error | ValueType::Null => Err(Error::InvalidType),
+        _ => {
+            let hash = digest(&digest::SHA512, &data.as_bytes());
+            Ok(hash.as_ref().to_vec())
+        }
+    }
+}
+
+pub fn sha384(data: &Value) -> Result<Vec<u8>, Error> {
+    match data.value_type() {
+        ValueType::Error | ValueType::Null => Err(Error::InvalidType),
+        _ => {
+            let hash = digest(&digest::SHA384, &data.as_bytes());
+            Ok(hash.as_ref().to_vec())
+        }
+    }
+}
+
+pub fn blake3(data: &Value) -> Result<Vec<u8>, Error> {
+    match data.value_type() {
+        ValueType::Error | ValueType::Null => Err(Error::InvalidType),
+        _ => {
+            let mut hasher = Hasher::new();
+            hasher.update(data.as_bytes().as_ref());
+            Ok(hasher.finalize().as_bytes().to_vec())
+        }
+    }
+}
+
+pub fn sha1(data: &Value) -> Result<Vec<u8>, Error> {
+    match data.value_type() {
+        ValueType::Error | ValueType::Null => Err(Error::InvalidType),
+        _ => {
+            let hash = digest(&digest::SHA1_FOR_LEGACY_USE_ONLY, &data.as_bytes());
+            Ok(hash.as_ref().to_vec())
+        }
+    }
+}

extensions/crypto/src/lib.rs

@@ -0,0 +1,78 @@
+use crypto::{blake3, sha1, sha256, sha384, sha512};
+use limbo_ext::{register_extension, scalar, ResultCode, Value};
+
+mod crypto;
+
+#[derive(Debug)]
+enum Error {
+    InvalidType,
+}
+
+#[scalar(name = "crypto_sha256", alias = "crypto_sha256")]
+fn crypto_sha256(args: &[Value]) -> Value {
+    if args.len() != 1 {
+        return Value::error(ResultCode::Error);
+    }
+
+    let Ok(hash) = sha256(&args[0]) else {
+        return Value::error(ResultCode::Error);
+    };
+
+    Value::from_blob(hash)
+}
+
+#[scalar(name = "crypto_sha512", alias = "crypto_sha512")]
+fn crypto_sha512(args: &[Value]) -> Value {
+    if args.len() != 1 {
+        return Value::error(ResultCode::Error);
+    }
+
+    let Ok(hash) = sha512(&args[0]) else {
+        return Value::error(ResultCode::Error);
+    };
+
+    Value::from_blob(hash)
+}
+
+#[scalar(name = "crypto_sha384", alias = "crypto_sha384")]
+fn crypto_sha384(args: &[Value]) -> Value {
+    if args.len() != 1 {
+        return Value::error(ResultCode::Error);
+    }
+
+    let Ok(hash) = sha384(&args[0]) else {
+        return Value::error(ResultCode::Error);
+    };
+
+    Value::from_blob(hash)
+}
+
+#[scalar(name = "crypto_blake3", alias = "crypto_blake3")]
+fn crypto_blake3(args: &[Value]) -> Value {
+    if args.len() != 1 {
+        return Value::error(ResultCode::Error);
+    }
+
+    let Ok(hash) = blake3(&args[0]) else {
+        return Value::error(ResultCode::Error);
+    };
+
+    Value::from_blob(hash)
+}
+
+#[scalar(name = "crypto_sha1", alias = "crypto_sha1")]
+fn crypto_sha1(args: &[Value]) -> Value {
+    if args.len() != 1 {
+        return Value::error(ResultCode::Error);
+    }
+
+    let Ok(hash) = sha1(&args[0]) else {
+        return Value::error(ResultCode::Error);
+    };
+
+    Value::from_blob(hash)
+}
+
+register_extension! {
+    scalars: { crypto_sha256, crypto_sha512, crypto_sha384, crypto_blake3, crypto_sha1 },
+}

testing/extensions.py

@@ -255,7 +255,6 @@ def test_aggregates(pipe):
         pipe, "SELECT percentile_disc(value, 0.55) from test;", validate_percentile_disc
     )
 
-
 def main():
     pipe = init_limbo()
     try: