Encryption support for database header page

This commit is contained in:
rajajisai
2025-09-11 16:17:01 -04:00
parent bc4aa63203
commit 89caa868f9
11 changed files with 205 additions and 118 deletions

View File

@@ -183,6 +183,7 @@ impl Limbo {
.with_indexes(indexes_enabled)
.with_views(opts.experimental_views)
.with_strict(opts.experimental_strict),
None,
)?;
let conn = db.connect()?;
(io, conn)

View File

@@ -418,6 +418,7 @@ impl TursoMcpServer {
None::<&str>,
OpenFlags::default(),
DatabaseOpts::new().with_indexes(true),
None,
) {
Ok((_io, db)) => match db.connect() {
Ok(c) => c,

View File

@@ -319,6 +319,7 @@ mod tests {
enable_views: true,
enable_strict: false,
},
None,
)?;
let conn = db.connect()?;

View File

@@ -146,6 +146,18 @@ impl DatabaseOpts {
}
}
#[derive(Clone, Debug, Default)]
pub struct EncryptionOpts {
pub cipher: String,
pub hexkey: String,
}
impl EncryptionOpts {
pub fn new() -> Self {
Self::default()
}
}
pub type Result<T, E = LimboError> = std::result::Result<T, E>;
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
@@ -266,6 +278,7 @@ impl Database {
DatabaseOpts::new()
.with_mvcc(enable_mvcc)
.with_indexes(enable_indexes),
None,
)
}
@@ -275,10 +288,11 @@ impl Database {
path: &str,
flags: OpenFlags,
opts: DatabaseOpts,
encryption_opts: Option<EncryptionOpts>,
) -> Result<Arc<Database>> {
let file = io.open_file(path, flags, true)?;
let db_file = Arc::new(DatabaseFile::new(file));
Self::open_with_flags(io, path, db_file, flags, opts)
Self::open_with_flags(io, path, db_file, flags, opts, encryption_opts)
}
#[allow(clippy::arc_with_non_send_sync)]
@@ -297,6 +311,7 @@ impl Database {
DatabaseOpts::new()
.with_mvcc(enable_mvcc)
.with_indexes(enable_indexes),
None,
)
}
@@ -307,6 +322,7 @@ impl Database {
db_file: Arc<dyn DatabaseStorage>,
flags: OpenFlags,
opts: DatabaseOpts,
encryption_opts: Option<EncryptionOpts>,
) -> Result<Arc<Database>> {
// turso-sync-engine create 2 databases with different names in the same IO if MemoryIO is used
// in this case we need to bypass registry (as this is MemoryIO DB) but also preserve original distinction in names (e.g. :memory:-draft and :memory:-synced)
@@ -318,6 +334,7 @@ impl Database {
db_file,
flags,
opts,
None,
);
}
@@ -338,6 +355,7 @@ impl Database {
db_file,
flags,
opts,
encryption_opts,
)?;
registry.insert(canonical_path, Arc::downgrade(&db));
Ok(db)
@@ -352,8 +370,17 @@ impl Database {
db_file: Arc<dyn DatabaseStorage>,
flags: OpenFlags,
opts: DatabaseOpts,
encryption_opts: Option<EncryptionOpts>,
) -> Result<Arc<Database>> {
Self::open_with_flags_bypass_registry_internal(io, path, wal_path, db_file, flags, opts)
Self::open_with_flags_bypass_registry_internal(
io,
path,
wal_path,
db_file,
flags,
opts,
encryption_opts,
)
}
#[allow(clippy::arc_with_non_send_sync)]
@@ -364,6 +391,7 @@ impl Database {
db_file: Arc<dyn DatabaseStorage>,
flags: OpenFlags,
opts: DatabaseOpts,
encryption_opts: Option<EncryptionOpts>,
) -> Result<Arc<Database>> {
let shared_wal = WalFileShared::open_shared_if_exists(&io, wal_path)?;
@@ -419,6 +447,12 @@ impl Database {
let syms = conn.syms.borrow();
let pager = conn.pager.borrow().clone();
if let Some(encryption_opts) = encryption_opts {
conn.pragma_update("cipher", format!("'{}'", encryption_opts.cipher))?;
conn.pragma_update("hexkey", format!("'{}'", encryption_opts.hexkey))?;
// Clear page cache so the header page can be reread from disk and decrypted using the encryption context.
pager.clear_page_cache();
}
db.with_schema_mut(|schema| {
let header_schema_cookie = pager
.io
@@ -452,7 +486,6 @@ impl Database {
.block(|| pager.with_header(|header| header.default_page_cache_size))
.unwrap_or_default()
.get();
let conn = Arc::new(Connection {
_db: self.clone(),
pager: RefCell::new(Rc::new(pager)),
@@ -712,6 +745,7 @@ impl Database {
vfs: Option<S>,
flags: OpenFlags,
opts: DatabaseOpts,
encryption_opts: Option<EncryptionOpts>,
) -> Result<(Arc<dyn IO>, Arc<Database>)>
where
S: AsRef<str> + std::fmt::Display,
@@ -721,7 +755,7 @@ impl Database {
.or_else(|| Some(Self::io_for_path(path)))
.transpose()?
.unwrap();
let db = Self::open_file_with_flags(io.clone(), path, flags, opts)?;
let db = Self::open_file_with_flags(io.clone(), path, flags, opts, encryption_opts)?;
Ok((io, db))
}
@@ -1288,10 +1322,25 @@ impl Connection {
.with_indexes(use_indexes)
.with_views(views)
.with_strict(strict),
None,
)?;
let conn = db.connect()?;
return Ok((io, conn));
}
let encryption_opts = match (opts.cipher.clone(), opts.hexkey.clone()) {
(Some(cipher), Some(hexkey)) => Some(EncryptionOpts { cipher, hexkey }),
(Some(_), None) => {
return Err(LimboError::InvalidArgument(
"hexkey is required when cipher is provided".to_string(),
))
}
(None, Some(_)) => {
return Err(LimboError::InvalidArgument(
"cipher is required when hexkey is provided".to_string(),
))
}
(None, None) => None,
};
let (io, db) = Database::open_new(
&opts.path,
opts.vfs.as_ref(),
@@ -1301,6 +1350,7 @@ impl Connection {
.with_indexes(use_indexes)
.with_views(views)
.with_strict(strict),
encryption_opts.clone(),
)?;
if let Some(modeof) = opts.modeof {
let perms = std::fs::metadata(modeof)?;
@@ -1313,6 +1363,15 @@ impl Connection {
if let Some(hexkey) = opts.hexkey {
let _ = conn.pragma_update("hexkey", format!("'{hexkey}'"));
}
if let Some(encryption_opts) = encryption_opts {
let _ = conn.pragma_update("cipher", encryption_opts.cipher.to_string());
let _ = conn.pragma_update("hexkey", encryption_opts.hexkey.to_string());
let pager = conn.pager.borrow();
if db.db_state.is_initialized() {
// Clear page cache so the header page can be reread from disk and decrypted using the encryption context.
pager.clear_page_cache();
}
}
Ok((io, conn))
}
@@ -1327,7 +1386,7 @@ impl Connection {
opts.mode = OpenMode::ReadOnly;
let flags = opts.get_flags()?;
let io = opts.vfs.map(Database::io_for_vfs).unwrap_or(Ok(io))?;
let db = Database::open_file_with_flags(io.clone(), &opts.path, flags, db_opts)?;
let db = Database::open_file_with_flags(io.clone(), &opts.path, flags, db_opts, None)?;
if let Some(modeof) = opts.modeof {
let perms = std::fs::metadata(modeof)?;
std::fs::set_permissions(&opts.path, perms.permissions())?;

View File

@@ -1,4 +1,5 @@
#![allow(unused_variables, dead_code)]
use crate::storage::sqlite3_ondisk::DatabaseHeader;
use crate::{LimboError, Result};
use aegis::aegis128l::Aegis128L;
use aegis::aegis128x2::Aegis128X2;
@@ -425,10 +426,6 @@ impl EncryptionContext {
#[cfg(feature = "encryption")]
pub fn encrypt_page(&self, page: &[u8], page_id: usize) -> Result<Vec<u8>> {
if page_id == 1 {
tracing::debug!("skipping encryption for page 1 (database header)");
return Ok(page.to_vec());
}
tracing::debug!("encrypting page {}", page_id);
assert_eq!(
page.len(),
@@ -437,6 +434,10 @@ impl EncryptionContext {
self.page_size
);
let encryption_start_offset = match page_id {
DatabaseHeader::PAGE_ID => DatabaseHeader::SIZE,
_ => 0,
};
let metadata_size = self.cipher_mode.metadata_size();
let reserved_bytes = &page[self.page_size - metadata_size..];
let reserved_bytes_zeroed = reserved_bytes.iter().all(|&b| b == 0);
@@ -445,18 +446,20 @@ impl EncryptionContext {
"last reserved bytes must be empty/zero, but found non-zero bytes"
);
let payload = &page[..self.page_size - metadata_size];
let payload = &page[encryption_start_offset..self.page_size - metadata_size];
let (encrypted, nonce) = self.encrypt_raw(payload)?;
let nonce_size = self.cipher_mode.nonce_size();
assert_eq!(
encrypted.len(),
self.page_size - nonce_size,
self.page_size - nonce_size - encryption_start_offset,
"Encrypted page must be exactly {} bytes",
self.page_size - nonce_size
self.page_size - nonce_size - encryption_start_offset
);
let mut result = Vec::with_capacity(self.page_size);
result.extend_from_slice(&page[..encryption_start_offset]);
result.extend_from_slice(&encrypted);
result.extend_from_slice(&nonce);
assert_eq!(
@@ -470,10 +473,6 @@ impl EncryptionContext {
#[cfg(feature = "encryption")]
pub fn decrypt_page(&self, encrypted_page: &[u8], page_id: usize) -> Result<Vec<u8>> {
if page_id == 1 {
tracing::debug!("skipping decryption for page 1 (database header)");
return Ok(encrypted_page.to_vec());
}
tracing::debug!("decrypting page {}", page_id);
assert_eq!(
encrypted_page.len(),
@@ -481,23 +480,30 @@ impl EncryptionContext {
"Encrypted page data must be exactly {} bytes",
self.page_size
);
// for page 1, the encrypted page starts after the database header
// for other pages, the encrypted page starts at the beginning
let encrypted_page_offset = match page_id {
DatabaseHeader::PAGE_ID => DatabaseHeader::SIZE,
_ => 0,
};
let nonce_size = self.cipher_mode.nonce_size();
let nonce_start = encrypted_page.len() - nonce_size;
let payload = &encrypted_page[..nonce_start];
let nonce = &encrypted_page[nonce_start..];
let nonce_offset = encrypted_page.len() - nonce_size;
let payload = &encrypted_page[encrypted_page_offset..nonce_offset];
let nonce = &encrypted_page[nonce_offset..];
let decrypted_data = self.decrypt_raw(payload, nonce)?;
let metadata_size = self.cipher_mode.metadata_size();
assert_eq!(
decrypted_data.len(),
self.page_size - metadata_size,
self.page_size - metadata_size - encrypted_page_offset,
"Decrypted page data must be exactly {} bytes",
self.page_size - metadata_size
self.page_size - metadata_size - encrypted_page_offset
);
let mut result = Vec::with_capacity(self.page_size);
result.extend_from_slice(&encrypted_page[..encrypted_page_offset]);
result.extend_from_slice(&decrypted_data);
result.resize(self.page_size, 0);
assert_eq!(

View File

@@ -531,6 +531,19 @@ $ cargo run --features encryption -- database.db
PRAGMA cipher = 'aegis256'; -- or 'aes256gcm'
PRAGMA hexkey = '2d7a30108d3eb3e45c90a732041fe54778bdcf707c76749fab7da335d1b39c1d';
```
Alternatively you can provide the encryption parameters directly in a **URI**. For example:
```shell
$ cargo run --features encryption \
"file:database.db?cipher=aegis256&hexkey=2d7a30108d3eb3e45c90a732041fe54778bdcf707c76749fab7da335d1b39c1d"
```
> **Note:** To reopen an already *encrypted database*,the file **must** opened in URI format with the `cipher` and `hexkey` passed as URI parameters. Now, to reopen `database.db` the command below must be run:
```shell
$ cargo run --features encryption \
"file:database.db?cipher=aegis256hexkey=2d7a30108d3eb3e45c90a732041fe54778bdcf707c76749fab7da335d1b39c1d"
```
## CDC (Early Preview)

View File

@@ -138,8 +138,8 @@ impl<P: ProtocolIO> DatabaseSyncEngine<P> {
db_file.clone(),
OpenFlags::Create,
turso_core::DatabaseOpts::new().with_indexes(true),
)
.unwrap();
None,
)?;
let tape_opts = DatabaseTapeOpts {
cdc_table: None,
cdc_mode: Some("full".to_string()),
@@ -184,6 +184,7 @@ impl<P: ProtocolIO> DatabaseSyncEngine<P> {
self.db_file.clone(),
OpenFlags::Create,
turso_core::DatabaseOpts::new().with_indexes(true),
None,
)?;
let conn = db.connect()?;
conn.wal_auto_checkpoint_disable();

View File

@@ -30,6 +30,7 @@ impl TempDatabase {
path.to_str().unwrap(),
turso_core::OpenFlags::default(),
turso_core::DatabaseOpts::new().with_indexes(enable_indexes),
None,
)
.unwrap();
Self { path, io, db }
@@ -44,6 +45,7 @@ impl TempDatabase {
path.to_str().unwrap(),
turso_core::OpenFlags::default(),
opts,
None,
)
.unwrap();
Self {
@@ -72,6 +74,7 @@ impl TempDatabase {
db_path.to_str().unwrap(),
flags,
turso_core::DatabaseOpts::new().with_indexes(enable_indexes),
None,
)
.unwrap();
Self {
@@ -97,6 +100,7 @@ impl TempDatabase {
path.to_str().unwrap(),
turso_core::OpenFlags::default(),
turso_core::DatabaseOpts::new().with_indexes(enable_indexes),
None,
)
.unwrap();

View File

@@ -871,6 +871,7 @@ fn test_db_share_same_file() {
db_file.clone(),
turso_core::OpenFlags::Create,
turso_core::DatabaseOpts::new().with_indexes(true),
None,
)
.unwrap();
let conn1 = db1.connect().unwrap();
@@ -897,6 +898,7 @@ fn test_db_share_same_file() {
db_file.clone(),
turso_core::OpenFlags::empty(),
turso_core::DatabaseOpts::new().with_indexes(true),
None,
)
.unwrap();
let conn2 = db2.connect().unwrap();

View File

@@ -39,92 +39,7 @@ fn test_per_page_encryption() -> anyhow::Result<()> {
}
{
// this should panik because we should not be able to access the encrypted database
// without the key
let conn = tmp_db.connect_limbo();
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |_: &Row| {}).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB without key"
);
// it should also panic if we specify either only key or cipher
let conn = tmp_db.connect_limbo();
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
run_query(&tmp_db, &conn, "PRAGMA cipher = 'aegis256';").unwrap();
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |_: &Row| {}).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB without key"
);
let conn = tmp_db.connect_limbo();
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
run_query(
&tmp_db,
&conn,
"PRAGMA hexkey = 'b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327';",
).unwrap();
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |_: &Row| {}).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB without cipher name"
);
// it should panic if we specify wrong cipher or key
let conn = tmp_db.connect_limbo();
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
run_query(
&tmp_db,
&conn,
"PRAGMA hexkey = 'b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327';",
).unwrap();
run_query(&tmp_db, &conn, "PRAGMA cipher = 'aes256gcm';").unwrap();
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |_: &Row| {}).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB with incorrect cipher"
);
let conn = tmp_db.connect_limbo();
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
run_query(&tmp_db, &conn, "PRAGMA cipher = 'aegis256';").unwrap();
run_query(
&tmp_db,
&conn,
"PRAGMA hexkey = 'b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76377';",
).unwrap();
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |_: &Row| {}).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB with incorrect key"
);
}
{
// let's test the existing db with the key
let existing_db = TempDatabase::new_with_existent(&db_path, false);
let conn = existing_db.connect_limbo();
run_query(&tmp_db, &conn, "PRAGMA cipher = 'aegis256';")?;
run_query(
&existing_db,
&conn,
"PRAGMA hexkey = 'b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327';",
)?;
run_query_on_row(&existing_db, &conn, "SELECT * FROM test", |row: &Row| {
assert_eq!(row.get::<i64>(0).unwrap(), 1);
assert_eq!(row.get::<String>(1).unwrap(), "Hello, World!");
})?;
}
{
// let's test connecting to the encrypted db using URI
//test connecting to the encrypted db using correct URI
let uri = format!(
"file:{}?cipher=aegis256&hexkey=b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327",
db_path.to_str().unwrap()
@@ -138,6 +53,92 @@ fn test_per_page_encryption() -> anyhow::Result<()> {
})?;
assert_eq!(row_count, 1);
}
{
//Try to create a table after reopening the encrypted db.
let uri = format!(
"file:{}?cipher=aegis256&hexkey=b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327",
db_path.to_str().unwrap()
);
let (_io, conn) = turso_core::Connection::from_uri(&uri, true, false, false, false)?;
run_query(
&tmp_db,
&conn,
"CREATE TABLE test1 (id INTEGER PRIMARY KEY, value TEXT);",
)?;
do_flush(&conn, &tmp_db)?;
}
{
//Try to create a table after reopening the encrypted db.
let uri = format!(
"file:{}?cipher=aegis256&hexkey=b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327",
db_path.to_str().unwrap()
);
let (_io, conn) = turso_core::Connection::from_uri(&uri, true, false, false, false)?;
run_query(
&tmp_db,
&conn,
"INSERT INTO test1 (value) VALUES ('Hello, World!')",
)?;
let mut row_count = 0;
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |row: &Row| {
assert_eq!(row.get::<i64>(0).unwrap(), 1);
assert_eq!(row.get::<String>(1).unwrap(), "Hello, World!");
row_count += 1;
})?;
assert_eq!(row_count, 1);
do_flush(&conn, &tmp_db)?;
}
{
// test connecting to encrypted db using wrong key(key is ending with 77.The correct key is ending with 27).This should panic.
let uri = format!(
"file:{}?cipher=aegis256&hexkey=b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76377",
db_path.to_str().unwrap()
);
let (_io, conn) = turso_core::Connection::from_uri(&uri, true, false, false, false)?;
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |_row: &Row| {}).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB with wrong key"
);
}
{
//test connecting to encrypted db using insufficient encryption parameters in URI.This should panic.
let uri = format!("file:{}?cipher=aegis256", db_path.to_str().unwrap());
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
turso_core::Connection::from_uri(&uri, true, false, false, false).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB without passing hexkey in URI"
);
}
{
let uri = format!(
"file:{}?hexkey=b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327",
db_path.to_str().unwrap()
);
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
turso_core::Connection::from_uri(&uri, true, false, false, false).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB without passing cipher in URI"
);
}
{
// Testing connecting to db without using URI.This should panic.
let conn = tmp_db.connect_limbo();
let should_panic = panic::catch_unwind(panic::AssertUnwindSafe(|| {
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |_row: &Row| {}).unwrap();
}));
assert!(
should_panic.is_err(),
"should panic when accessing encrypted DB without using URI"
);
}
Ok(())
}
@@ -182,15 +183,12 @@ fn test_non_4k_page_size_encryption() -> anyhow::Result<()> {
{
// Reopen the existing db with 8k page size and test encryption
let existing_db = TempDatabase::new_with_existent(&db_path, false);
let conn = existing_db.connect_limbo();
run_query(&tmp_db, &conn, "PRAGMA cipher = 'aegis256';")?;
run_query(
&existing_db,
&conn,
"PRAGMA hexkey = 'b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327';",
)?;
run_query_on_row(&existing_db, &conn, "SELECT * FROM test", |row: &Row| {
let uri = format!(
"file:{}?cipher=aegis256&hexkey=b1bbfda4f589dc9daaf004fe21111e00dc00c98237102f5c7002a5669fc76327",
db_path.to_str().unwrap()
);
let (_io, conn) = turso_core::Connection::from_uri(&uri, true, false, false, false)?;
run_query_on_row(&tmp_db, &conn, "SELECT * FROM test", |row: &Row| {
assert_eq!(row.get::<i64>(0).unwrap(), 1);
assert_eq!(row.get::<String>(1).unwrap(), "Hello, World!");
})?;

View File

@@ -707,6 +707,7 @@ fn test_wal_bad_frame() -> anyhow::Result<()> {
db_path.to_str().unwrap(),
turso_core::OpenFlags::default(),
turso_core::DatabaseOpts::new().with_indexes(false),
None,
)
.unwrap();
let tmp_db = TempDatabase {