From d9b1cac27bc3a1be7091bd63576bd9e205465c84 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Sat, 3 Aug 2024 12:16:34 +0300 Subject: [PATCH 1/3] testing: Add test database + WAL file --- testing/wal/users.db | Bin 0 -> 4096 bytes testing/wal/users.db-wal | Bin 0 -> 20632 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 testing/wal/users.db create mode 100644 testing/wal/users.db-wal diff --git a/testing/wal/users.db b/testing/wal/users.db new file mode 100644 index 0000000000000000000000000000000000000000..210e705467a7b7c47f0f429110e6f79034d4e95a GIT binary patch literal 4096 zcmWFz^vNtqRY=P(%1ta$FlG>7U}9o$P*7lCU|@t|AVoG{WY8r0Y7zE)iQ6VT;h|x<};qVl5ZsAqSswt%e!G^slde1tAcqMck zUKAJS-dz3~8Eu^&cU}EfDdTbeN3*%T`S0k@^rw8O7RiO>)upf|`_*XE6r)nAcTIIz zddzL;@h|Q=>KcFVQUAGHQ}nHU>2UrF(`g8&;rx8=C2|NLfB*srAbl&YH;cPjoSEJQxd@?^4ysYm@Kl>r?$v7p`rFof~^yO3`H<2wA<+xYui62$M zrAYeTjGxJvwma+^qEwlWHmWOSwXzSwdTlxG2k~cuckxB9>)UQ^^*aB4zrfyU9lWi- zU+>V#o2>+WM<9Rz0tg_000IagfB*srAb>zJ1@s<)byr7_-W}Kr&QEjgdampXB=e6m zD+CZg009ILKmY**5I_I{1ReyQ)C+vw?>2`{#=h%%0mJz#R3i{T009ILKmY**5I_I{ z1Q0+VQ37p-m2$&!X)aRJdQvZN{5t*e+g4!fdI7^}3DpP$5I_I{1Q0*~0R#|0009IL KNT7garBc^RJa8ib literal 0 HcmV?d00001 From ed4116e7c2cad5f26102070fe400f25652eae17f Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Sat, 3 Aug 2024 12:25:58 +0300 Subject: [PATCH 2/3] core: Introduce Wal trait We're going to need it for WebAssembly anyway, which does not have standard filesystem support. --- bindings/wasm/lib.rs | 27 ++++++++++++++++++++++++++- core/lib.rs | 20 +++++++++++++++++--- core/storage/pager.rs | 29 ++++++++++++----------------- core/storage/wal.rs | 41 ++++++++++++++++++++++++++++++----------- 4 files changed, 85 insertions(+), 32 deletions(-) diff --git a/bindings/wasm/lib.rs b/bindings/wasm/lib.rs index 1d36966ca..00d67a814 100644 --- a/bindings/wasm/lib.rs +++ b/bindings/wasm/lib.rs @@ -14,7 +14,8 @@ impl Database { pub fn new(_path: &str) -> Database { let io = Arc::new(IO {}); let page_io = Rc::new(DatabaseStorage {}); - let inner = limbo_core::Database::open(io, page_io).unwrap(); + let wal = Rc::new(Wal {}); + let inner = limbo_core::Database::open(io, page_io, wal).unwrap(); Database { _inner: inner } } @@ -50,3 +51,27 @@ impl limbo_core::DatabaseStorage for DatabaseStorage { todo!() } } + +pub struct Wal {} + +impl limbo_core::Wal for Wal { + fn begin_read_tx(&self) -> Result<()> { + todo!() + } + + fn end_read_tx(&self) -> Result<()> { + todo!() + } + + fn find_frame(&self, _page_id: u64) -> Result> { + todo!() + } + + fn read_frame( + &self, + _frame_id: u64, + _page: Rc>, + ) -> Result<()> { + todo!() + } +} diff --git a/core/lib.rs b/core/lib.rs index 44abf6d5b..8f37f8b1c 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -23,6 +23,8 @@ use std::{cell::RefCell, rc::Rc}; use storage::database::FileStorage; use storage::pager::Pager; use storage::sqlite3_ondisk::DatabaseHeader; +#[cfg(feature = "fs")] +use storage::wal::WalFile; pub use error::LimboError; pub type Result = std::result::Result; @@ -31,6 +33,8 @@ pub type Result = std::result::Result; pub use io::PlatformIO; pub use io::{Buffer, Completion, File, WriteCompletion, IO}; pub use storage::database::DatabaseStorage; +pub use storage::pager::Page; +pub use storage::wal::Wal; pub use types::Value; pub struct Database { @@ -44,13 +48,23 @@ impl Database { pub fn open_file(io: Arc, path: &str) -> Result { let file = io.open_file(path)?; let page_io = Rc::new(FileStorage::new(file)); - Self::open(io, page_io) + let wal = Rc::new(WalFile::new()); + Self::open(io, page_io, wal) } - pub fn open(io: Arc, page_io: Rc) -> Result { + pub fn open( + io: Arc, + page_io: Rc, + wal: Rc, + ) -> Result { let db_header = Pager::begin_open(page_io.clone())?; io.run_once()?; - let pager = Rc::new(Pager::finish_open(db_header.clone(), page_io, io.clone())?); + let pager = Rc::new(Pager::finish_open( + db_header.clone(), + page_io, + wal, + io.clone(), + )?); let bootstrap_schema = Rc::new(Schema::new()); let conn = Connection { pager: pager.clone(), diff --git a/core/storage/pager.rs b/core/storage/pager.rs index 023bfb1aa..783687355 100644 --- a/core/storage/pager.rs +++ b/core/storage/pager.rs @@ -265,7 +265,7 @@ pub struct Pager { /// Source of the database pages. pub page_io: Rc, /// The write-ahead log (WAL) for the database. - wal: Option, + wal: Rc, /// A page cache for the database. page_cache: RefCell, /// Buffer pool for temporary data storage. @@ -286,6 +286,7 @@ impl Pager { pub fn finish_open( db_header_ref: Rc>, page_io: Rc, + wal: Rc, io: Arc, ) -> Result { let db_header = RefCell::borrow(&db_header_ref); @@ -294,7 +295,7 @@ impl Pager { let page_cache = RefCell::new(DumbLruPageCache::new(10)); Ok(Self { page_io, - wal: None, + wal, buffer_pool, page_cache, io, @@ -304,16 +305,12 @@ impl Pager { } pub fn begin_read_tx(&self) -> Result<()> { - if let Some(wal) = &self.wal { - wal.begin_read_tx()?; - } + self.wal.begin_read_tx()?; Ok(()) } pub fn end_read_tx(&self) -> Result<()> { - if let Some(wal) = &self.wal { - wal.end_read_tx()?; - } + self.wal.end_read_tx()?; Ok(()) } @@ -326,16 +323,14 @@ impl Pager { } let page = Rc::new(RefCell::new(Page::new(page_idx))); RefCell::borrow(&page).set_locked(); - if let Some(wal) = &self.wal { - if let Some(frame_id) = wal.find_frame(page_idx as u64)? { - wal.read_frame(frame_id, page.clone())?; - { - let page = page.borrow_mut(); - page.set_uptodate(); - } - page_cache.insert(page_idx, page.clone()); - return Ok(page); + if let Some(frame_id) = self.wal.find_frame(page_idx as u64)? { + self.wal.read_frame(frame_id, page.clone())?; + { + let page = page.borrow_mut(); + page.set_uptodate(); } + page_cache.insert(page_idx, page.clone()); + return Ok(page); } sqlite3_ondisk::begin_read_page( self.page_io.clone(), diff --git a/core/storage/wal.rs b/core/storage/wal.rs index b25a8acbd..ef699bb91 100644 --- a/core/storage/wal.rs +++ b/core/storage/wal.rs @@ -3,30 +3,49 @@ use std::{cell::RefCell, rc::Rc}; use crate::{storage::pager::Page, Result}; /// Write-ahead log (WAL). -pub struct Wal {} - -impl Wal { - pub fn new() -> Self { - Self {} - } - +pub trait Wal { /// Begin a write transaction. - pub fn begin_read_tx(&self) -> Result<()> { + fn begin_read_tx(&self) -> Result<()>; + + /// End a write transaction. + fn end_read_tx(&self) -> Result<()>; + + /// Find the latest frame containing a page. + fn find_frame(&self, page_id: u64) -> Result>; + + /// Read a frame from the WAL. + fn read_frame(&self, frame_id: u64, page: Rc>) -> Result<()>; +} + +#[cfg(feature = "fs")] +pub struct WalFile {} + +#[cfg(feature = "fs")] +impl Wal for WalFile { + /// Begin a write transaction. + fn begin_read_tx(&self) -> Result<()> { Ok(()) } /// End a write transaction. - pub fn end_read_tx(&self) -> Result<()> { + fn end_read_tx(&self) -> Result<()> { Ok(()) } /// Find the latest frame containing a page. - pub fn find_frame(&self, _page_id: u64) -> Result> { + fn find_frame(&self, _page_id: u64) -> Result> { Ok(None) } /// Read a frame from the WAL. - pub fn read_frame(&self, _frame_id: u64, _page: Rc>) -> Result<()> { + fn read_frame(&self, _frame_id: u64, _page: Rc>) -> Result<()> { todo!(); } } + +#[cfg(feature = "fs")] +impl WalFile { + pub fn new() -> Self { + Self {} + } +} From a290b2f1022d625407c754af768884ed1d9c18ab Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Sat, 3 Aug 2024 12:44:36 +0300 Subject: [PATCH 3/3] core: Open WAL file and parse header --- core/lib.rs | 3 ++- core/storage/sqlite3_ondisk.rs | 2 +- core/storage/wal.rs | 35 ++++++++++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index 8f37f8b1c..fb695a3aa 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -48,7 +48,8 @@ impl Database { pub fn open_file(io: Arc, path: &str) -> Result { let file = io.open_file(path)?; let page_io = Rc::new(FileStorage::new(file)); - let wal = Rc::new(WalFile::new()); + let wal_path = format!("{}-wal", path); + let wal = Rc::new(WalFile::new(io.clone(), wal_path)); Self::open(io, page_io, wal) } diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 885c8cd90..4c19577b7 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -830,7 +830,7 @@ pub fn write_varint(buf: &mut [u8], value: u64) -> usize { return n; } -pub fn begin_read_wal_header(io: &Box) -> Result>> { +pub fn begin_read_wal_header(io: Rc) -> Result>> { let drop_fn = Rc::new(|_buf| {}); let buf = Rc::new(RefCell::new(Buffer::allocate(32, drop_fn))); let result = Rc::new(RefCell::new(WalHeader::default())); diff --git a/core/storage/wal.rs b/core/storage/wal.rs index ef699bb91..10583e26b 100644 --- a/core/storage/wal.rs +++ b/core/storage/wal.rs @@ -1,7 +1,10 @@ -use std::{cell::RefCell, rc::Rc}; +use std::{cell::RefCell, rc::Rc, sync::Arc}; +use crate::io::{File, IO}; use crate::{storage::pager::Page, Result}; +use super::sqlite3_ondisk; + /// Write-ahead log (WAL). pub trait Wal { /// Begin a write transaction. @@ -18,7 +21,12 @@ pub trait Wal { } #[cfg(feature = "fs")] -pub struct WalFile {} +pub struct WalFile { + io: Arc, + wal_path: String, + file: RefCell>>, + wal_header: RefCell>>>, +} #[cfg(feature = "fs")] impl Wal for WalFile { @@ -34,6 +42,7 @@ impl Wal for WalFile { /// Find the latest frame containing a page. fn find_frame(&self, _page_id: u64) -> Result> { + self.ensure_init()?; Ok(None) } @@ -45,7 +54,25 @@ impl Wal for WalFile { #[cfg(feature = "fs")] impl WalFile { - pub fn new() -> Self { - Self {} + pub fn new(io: Arc, wal_path: String) -> Self { + Self { + io, + wal_path, + file: RefCell::new(None), + wal_header: RefCell::new(None), + } + } + + fn ensure_init(&self) -> Result<()> { + if self.file.borrow().is_none() { + if let Ok(file) = self.io.open_file(&self.wal_path) { + *self.file.borrow_mut() = Some(file.clone()); + let wal_header = sqlite3_ondisk::begin_read_wal_header(file)?; + // TODO: Return a completion instead. + self.io.run_once()?; + self.wal_header.replace(Some(wal_header)); + } + } + Ok(()) } }