From 0adb40534c0f02d6712460fbb7546e0ed10a1295 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Mon, 4 Aug 2025 12:40:28 +0400 Subject: [PATCH 1/2] hind dangerous methods behind conn_raw_api feature --- bindings/rust/Cargo.toml | 1 + bindings/rust/src/lib.rs | 10 ++++++++-- core/Cargo.toml | 3 ++- core/lib.rs | 17 +++++++++-------- core/storage/pager.rs | 10 +++++----- core/types.rs | 12 +++++++++--- packages/turso-sync/Cargo.toml | 4 ++-- packages/turso-sync/src/database_inner.rs | 4 ++-- sqlite3/Cargo.toml | 2 +- 9 files changed, 39 insertions(+), 24 deletions(-) diff --git a/bindings/rust/Cargo.toml b/bindings/rust/Cargo.toml index f1d98b403..0f35d0b2f 100644 --- a/bindings/rust/Cargo.toml +++ b/bindings/rust/Cargo.toml @@ -11,6 +11,7 @@ description = "Turso Rust API" [features] default = ["experimental_indexes"] +conn_raw_api = ["turso_core/conn_raw_api"] experimental_indexes = [] antithesis = ["turso_core/antithesis"] diff --git a/bindings/rust/src/lib.rs b/bindings/rust/src/lib.rs index 45d455d1e..72b4096d1 100644 --- a/bindings/rust/src/lib.rs +++ b/bindings/rust/src/lib.rs @@ -38,7 +38,8 @@ pub mod transaction; pub mod value; use transaction::TransactionBehavior; -use turso_core::types::WalInsertInfo; +#[cfg(feature = "conn_raw_api")] +use turso_core::types::WalFrameInfo; pub use value::Value; pub use params::params_from_iter; @@ -181,6 +182,7 @@ impl Connection { stmt.execute(params).await } + #[cfg(feature = "conn_raw_api")] pub fn wal_frame_count(&self) -> Result { let conn = self .inner @@ -190,6 +192,7 @@ impl Connection { .map_err(|e| Error::WalOperationError(format!("wal_insert_begin failed: {e}"))) } + #[cfg(feature = "conn_raw_api")] pub fn wal_insert_begin(&self) -> Result<()> { let conn = self .inner @@ -199,6 +202,7 @@ impl Connection { .map_err(|e| Error::WalOperationError(format!("wal_insert_begin failed: {e}"))) } + #[cfg(feature = "conn_raw_api")] pub fn wal_insert_end(&self) -> Result<()> { let conn = self .inner @@ -208,7 +212,8 @@ impl Connection { .map_err(|e| Error::WalOperationError(format!("wal_insert_end failed: {e}"))) } - pub fn wal_insert_frame(&self, frame_no: u32, frame: &[u8]) -> Result { + #[cfg(feature = "conn_raw_api")] + pub fn wal_insert_frame(&self, frame_no: u32, frame: &[u8]) -> Result { let conn = self .inner .lock() @@ -217,6 +222,7 @@ impl Connection { .map_err(|e| Error::WalOperationError(format!("wal_insert_frame failed: {e}"))) } + #[cfg(feature = "conn_raw_api")] pub fn wal_get_frame(&self, frame_no: u32, frame: &mut [u8]) -> Result<()> { let conn = self .inner diff --git a/core/Cargo.toml b/core/Cargo.toml index 343b63695..29afcd6e6 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -14,8 +14,9 @@ name = "turso_core" path = "lib.rs" [features] -antithesis = ["dep:antithesis_sdk"] default = ["fs", "uuid", "time", "json", "series"] +antithesis = ["dep:antithesis_sdk"] +conn_raw_api = [] fs = ["turso_ext/vfs"] json = [] uuid = ["dep:uuid"] diff --git a/core/lib.rs b/core/lib.rs index 41fe11f36..da7c92b9a 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -44,8 +44,8 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; use crate::translate::optimizer::optimize_plan; use crate::translate::pragma::TURSO_CDC_DEFAULT_TABLE_NAME; -#[cfg(feature = "fs")] -use crate::types::WalInsertInfo; +#[cfg(all(feature = "fs", feature = "conn_raw_api"))] +use crate::types::WalFrameInfo; #[cfg(feature = "fs")] use crate::util::{OpenMode, OpenOptions}; use crate::vtab::VirtualTable; @@ -813,6 +813,7 @@ impl Connection { /// Parse schema from scratch if version of schema for the connection differs from the schema cookie in the root page /// This function must be called outside of any transaction because internally it will start transaction session by itself + #[allow(dead_code)] fn maybe_reparse_schema(self: &Arc) -> Result<()> { let pager = self.pager.borrow().clone(); @@ -1140,12 +1141,12 @@ impl Connection { Ok(()) } - #[cfg(feature = "fs")] + #[cfg(all(feature = "fs", feature = "conn_raw_api"))] pub fn wal_frame_count(&self) -> Result { self.pager.borrow().wal_frame_count() } - #[cfg(feature = "fs")] + #[cfg(all(feature = "fs", feature = "conn_raw_api"))] pub fn wal_get_frame(&self, frame_no: u32, frame: &mut [u8]) -> Result<()> { let c = self.pager.borrow().wal_get_frame(frame_no, frame)?; self._db.io.wait_for_completion(c) @@ -1154,13 +1155,13 @@ impl Connection { /// Insert `frame` (header included) at the position `frame_no` in the WAL /// If WAL already has frame at that position - turso-db will compare content of the page and either report conflict or return OK /// If attempt to write frame at the position `frame_no` will create gap in the WAL - method will return error - #[cfg(feature = "fs")] - pub fn wal_insert_frame(&self, frame_no: u32, frame: &[u8]) -> Result { + #[cfg(all(feature = "fs", feature = "conn_raw_api"))] + pub fn wal_insert_frame(&self, frame_no: u32, frame: &[u8]) -> Result { self.pager.borrow().wal_insert_frame(frame_no, frame) } /// Start WAL session by initiating read+write transaction for this connection - #[cfg(feature = "fs")] + #[cfg(all(feature = "fs", feature = "conn_raw_api"))] pub fn wal_insert_begin(&self) -> Result<()> { let pager = self.pager.borrow(); match pager.begin_read_tx()? { @@ -1181,7 +1182,7 @@ impl Connection { /// Finish WAL session by ending read+write transaction taken in the [Self::wal_insert_begin] method /// All frames written after last commit frame (db_size > 0) within the session will be rolled back - #[cfg(feature = "fs")] + #[cfg(all(feature = "fs", feature = "conn_raw_api"))] pub fn wal_insert_end(self: &Arc) -> Result<()> { { let pager = self.pager.borrow(); diff --git a/core/storage/pager.rs b/core/storage/pager.rs index 6455027ae..759ab2950 100644 --- a/core/storage/pager.rs +++ b/core/storage/pager.rs @@ -6,7 +6,7 @@ use crate::storage::sqlite3_ondisk::{ self, parse_wal_frame_header, DatabaseHeader, PageContent, PageSize, PageType, }; use crate::storage::wal::{CheckpointResult, Wal}; -use crate::types::{IOResult, WalInsertInfo}; +use crate::types::{IOResult, WalFrameInfo}; use crate::util::IOExt as _; use crate::{return_if_io, Completion, TransactionState}; use crate::{turso_assert, Buffer, Connection, LimboError, Result}; @@ -1290,7 +1290,7 @@ impl Pager { } #[instrument(skip_all, level = Level::DEBUG)] - pub fn wal_insert_frame(&self, frame_no: u32, frame: &[u8]) -> Result { + pub fn wal_insert_frame(&self, frame_no: u32, frame: &[u8]) -> Result { let Some(wal) = self.wal.as_ref() else { return Err(LimboError::InternalError( "wal_insert_frame() called on database without WAL".to_string(), @@ -1323,9 +1323,9 @@ impl Pager { } self.dirty_pages.borrow_mut().clear(); } - Ok(WalInsertInfo { - page_no: header.page_number as usize, - is_commit: header.is_commit_frame(), + Ok(WalFrameInfo { + page_no: header.page_number, + db_size: header.db_size, }) } diff --git a/core/types.rs b/core/types.rs index 5cc429584..5c9dbe89e 100644 --- a/core/types.rs +++ b/core/types.rs @@ -2590,9 +2590,15 @@ pub struct DatabaseChange { } #[derive(Debug)] -pub struct WalInsertInfo { - pub page_no: usize, - pub is_commit: bool, +pub struct WalFrameInfo { + pub page_no: u32, + pub db_size: u32, +} + +impl WalFrameInfo { + pub fn is_commit_frame(&self) -> bool { + self.db_size > 0 + } } #[cfg(test)] diff --git a/packages/turso-sync/Cargo.toml b/packages/turso-sync/Cargo.toml index c7079b94b..a9ad0923c 100644 --- a/packages/turso-sync/Cargo.toml +++ b/packages/turso-sync/Cargo.toml @@ -7,8 +7,8 @@ license.workspace = true repository.workspace = true [dependencies] -turso_core = { workspace = true } -turso = { workspace = true } +turso_core = { workspace = true, features = ["conn_raw_api"] } +turso = { workspace = true, features = ["conn_raw_api"] } thiserror = "2.0.12" tracing = "0.1.41" hyper = { version = "1.6.0", features = ["client", "http1"] } diff --git a/packages/turso-sync/src/database_inner.rs b/packages/turso-sync/src/database_inner.rs index 77c0297d9..e51841b39 100644 --- a/packages/turso-sync/src/database_inner.rs +++ b/packages/turso-sync/src/database_inner.rs @@ -369,8 +369,8 @@ impl DatabaseInner { if !wal_session.in_txn() { wal_session.begin()?; } - let wal_insert_info = clean_conn.wal_insert_frame(frame_no as u32, &buffer)?; - if wal_insert_info.is_commit { + let wal_frame_info = clean_conn.wal_insert_frame(frame_no as u32, &buffer)?; + if wal_frame_info.is_commit_frame() { wal_session.end()?; // transaction boundary reached - it's safe to commit progress self.meta = Some(self.write_meta(|m| m.synced_frame_no = frame_no).await?); diff --git a/sqlite3/Cargo.toml b/sqlite3/Cargo.toml index 58d25ec56..76e397141 100644 --- a/sqlite3/Cargo.toml +++ b/sqlite3/Cargo.toml @@ -24,7 +24,7 @@ crate-type = ["lib", "cdylib", "staticlib"] [dependencies] env_logger = { version = "0.11.3", default-features = false } libc = "0.2.169" -turso_core = { path = "../core" } +turso_core = { path = "../core", features = ["conn_raw_api"] } tracing = "0.1.41" tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } From 83b1e99a616d8d12fc75d7ffb6e3b76381ae354b Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Mon, 4 Aug 2025 12:53:07 +0400 Subject: [PATCH 2/2] fix compilation --- packages/turso-sync/src/sync_server/test.rs | 2 +- tests/Cargo.toml | 4 ++-- tests/integration/functions/test_wal_api.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/turso-sync/src/sync_server/test.rs b/packages/turso-sync/src/sync_server/test.rs index 32b9667b4..fed11fc57 100644 --- a/packages/turso-sync/src/sync_server/test.rs +++ b/packages/turso-sync/src/sync_server/test.rs @@ -187,7 +187,7 @@ impl SyncServer for TestSyncServer { let frame = &frames[offset..offset + FRAME_SIZE]; match session.conn.wal_insert_frame(frame_no as u32, frame) { Ok(info) => { - if info.is_commit { + if info.is_commit_frame() { if session.in_txn { session.conn.wal_insert_end()?; session.in_txn = false; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 445e1e335..9353fa47b 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -17,8 +17,8 @@ path = "integration/mod.rs" [dependencies] anyhow.workspace = true env_logger = "0.10.1" -turso_core = { path = "../core" } -turso = { path = "../bindings/rust" } +turso_core = { path = "../core", features = ["conn_raw_api"] } +turso = { path = "../bindings/rust", features = ["conn_raw_api"] } tokio = { version = "1.47", features = ["full"] } rusqlite = { version = "0.34", features = ["bundled"] } tempfile = "3.0.7" diff --git a/tests/integration/functions/test_wal_api.rs b/tests/integration/functions/test_wal_api.rs index 614eb795b..2c1e2c509 100644 --- a/tests/integration/functions/test_wal_api.rs +++ b/tests/integration/functions/test_wal_api.rs @@ -140,7 +140,7 @@ fn test_wal_frame_transfer_schema_changes() { for frame_id in 1..=conn1.wal_frame_count().unwrap() as u32 { conn1.wal_get_frame(frame_id, &mut frame).unwrap(); let info = conn2.wal_insert_frame(frame_id, &frame).unwrap(); - if info.is_commit { + if info.is_commit_frame() { commits += 1; } }