diff --git a/bindings/javascript/src/lib.rs b/bindings/javascript/src/lib.rs index d1045e788..d8a97e958 100644 --- a/bindings/javascript/src/lib.rs +++ b/bindings/javascript/src/lib.rs @@ -6,7 +6,7 @@ use std::num::NonZeroUsize; use std::rc::Rc; use std::sync::Arc; -use limbo_core::{maybe_init_database_file, LimboError, StepResult}; +use limbo_core::{LimboError, StepResult}; use napi::iterator::Generator; use napi::{bindgen_prelude::ObjectFinalize, Env, JsUnknown}; use napi_derive::napi; @@ -65,7 +65,6 @@ impl Database { let file = io .open_file(&path, limbo_core::OpenFlags::Create, false) .map_err(into_napi_error)?; - maybe_init_database_file(&file, &io).map_err(into_napi_error)?; let db_file = Arc::new(DatabaseFile::new(file)); let db = limbo_core::Database::open(io.clone(), &path, db_file, false) .map_err(into_napi_error)?; diff --git a/bindings/wasm/lib.rs b/bindings/wasm/lib.rs index eb78f3b21..7a4fe7ed7 100644 --- a/bindings/wasm/lib.rs +++ b/bindings/wasm/lib.rs @@ -2,7 +2,7 @@ compile_error!("Features 'web' and 'nodejs' cannot be enabled at the same time"); use js_sys::{Array, Object}; -use limbo_core::{maybe_init_database_file, Clock, Instant, OpenFlags, Result}; +use limbo_core::{Clock, Instant, OpenFlags, Result}; use std::cell::RefCell; use std::sync::Arc; use wasm_bindgen::prelude::*; @@ -21,7 +21,6 @@ impl Database { pub fn new(path: &str) -> Database { let io: Arc = Arc::new(PlatformIO { vfs: VFS::new() }); let file = io.open_file(path, OpenFlags::Create, false).unwrap(); - maybe_init_database_file(&file, &io).unwrap(); let db_file = Arc::new(DatabaseFile::new(file)); let db = limbo_core::Database::open(io, path, db_file, false).unwrap(); let conn = db.connect().unwrap(); diff --git a/core/lib.rs b/core/lib.rs index eae944e23..9163e3c0c 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -62,9 +62,10 @@ use std::{ rc::Rc, sync::Arc, }; -use storage::btree::{btree_init_page, BTreePageInner}; +use storage::btree::btree_init_page; #[cfg(feature = "fs")] use storage::database::DatabaseFile; +use storage::page_cache::DumbLruPageCache; pub use storage::pager::PagerCacheflushStatus; pub use storage::{ buffer_pool::BufferPool, @@ -73,11 +74,6 @@ pub use storage::{ pager::{Page, Pager}, wal::{CheckpointMode, CheckpointResult, CheckpointStatus, Wal, WalFile, WalFileShared}, }; -use storage::{ - page_cache::DumbLruPageCache, - pager::allocate_page, - sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE}, -}; use tracing::{instrument, Level}; use translate::select::prepare_select_plan; pub use types::RefValue; @@ -129,7 +125,6 @@ impl Database { enable_mvcc: bool, ) -> Result> { let file = io.open_file(path, flags, true)?; - maybe_init_database_file(&file, &io)?; let db_file = Arc::new(DatabaseFile::new(file)); Self::open_with_flags(io, path, db_file, flags, enable_mvcc) } @@ -154,6 +149,7 @@ impl Database { ) -> Result> { let wal_path = format!("{}-wal", path); let maybe_shared_wal = WalFileShared::open_shared_if_exists(&io, wal_path.as_str())?; + let db_size = db_file.size()?; let mv_store = if enable_mvcc { Some(Rc::new(MvStore::new( @@ -177,7 +173,7 @@ impl Database { open_flags: flags, }; let db = Arc::new(db); - { + if db_size > 0 { // parse schema let conn = db.connect()?; let rows = conn.query("SELECT * FROM sqlite_schema")?; @@ -299,59 +295,6 @@ impl Database { } } -pub fn maybe_init_database_file(file: &Arc, io: &Arc) -> Result<()> { - if file.size()? == 0 { - // init db - let db_header = DatabaseHeader::default(); - let page1 = allocate_page( - 1, - &Rc::new(BufferPool::new(Some(db_header.get_page_size() as usize))), - DATABASE_HEADER_SIZE, - ); - let page1 = Arc::new(BTreePageInner { - page: RefCell::new(page1), - }); - { - // Create the sqlite_schema table, for this we just need to create the btree page - // for the first page of the database which is basically like any other btree page - // but with a 100 byte offset, so we just init the page so that sqlite understands - // this is a correct page. - btree_init_page( - &page1, - storage::sqlite3_ondisk::PageType::TableLeaf, - DATABASE_HEADER_SIZE, - (db_header.get_page_size() - db_header.reserved_space as u32) as u16, - ); - - let page1 = page1.get(); - let contents = page1.get().contents.as_mut().unwrap(); - contents.write_database_header(&db_header); - // write the first page to disk synchronously - let flag_complete = Rc::new(RefCell::new(false)); - { - let flag_complete = flag_complete.clone(); - let completion = Completion::Write(WriteCompletion::new(Box::new(move |_| { - *flag_complete.borrow_mut() = true; - }))); - #[allow(clippy::arc_with_non_send_sync)] - file.pwrite(0, contents.buffer.clone(), Arc::new(completion))?; - } - let mut limit = 100; - loop { - io.run_once()?; - if *flag_complete.borrow() { - break; - } - limit -= 1; - if limit == 0 { - panic!("Database file couldn't be initialized, io loop run for {} iterations and write didn't finish", limit); - } - } - } - }; - Ok(()) -} - pub struct Connection { _db: Arc, pager: Rc, diff --git a/core/storage/pager.rs b/core/storage/pager.rs index cabc7aba8..13d6c7823 100644 --- a/core/storage/pager.rs +++ b/core/storage/pager.rs @@ -1,3 +1,4 @@ +use crate::fast_lock::SpinLock; use crate::result::LimboResult; use crate::storage::btree::BTreePageInner; use crate::storage::buffer_pool::BufferPool; @@ -236,6 +237,12 @@ pub enum PagerCacheflushResult { } impl Pager { + /// Begins opening a database by reading the database header. + pub fn begin_open(db_file: Arc) -> Result>> { + assert!(db_file.size()? > 0); + sqlite3_ondisk::begin_read_database_header(db_file) + } + pub fn new( db_file: Arc, wal: Rc>, @@ -243,6 +250,16 @@ impl Pager { page_cache: Arc>, buffer_pool: Rc, ) -> Result { + let npages = if db_file.size().unwrap() > 0 { + let db_header = Pager::begin_open(db_file.clone()).unwrap(); + // ensure db header is there + io.run_once().unwrap(); + let size = db_header.lock().database_size; + size as usize + } else { + 0 + }; + Ok(Self { db_file, wal, @@ -258,7 +275,7 @@ impl Pager { checkpoint_inflight: Rc::new(RefCell::new(0)), buffer_pool, auto_vacuum_mode: RefCell::new(AutoVacuumMode::None), - npages: AtomicUsize::new(0), + npages: AtomicUsize::new(npages), }) } @@ -1381,8 +1398,9 @@ mod ptrmap_tests { // Helper to create a Pager for testing fn test_pager_setup(page_size: u32, initial_db_pages: u32) -> Pager { let io: Arc = Arc::new(MemoryIO::new()); - let db_file_raw = io.open_file("test.db", OpenFlags::Create, true).unwrap(); - let db_storage: Arc = Arc::new(DatabaseFile::new(db_file_raw)); + let db_file: Arc = Arc::new(DatabaseFile::new( + io.open_file("test.db", OpenFlags::Create, true).unwrap(), + )); // Initialize a minimal header in autovacuum mode let mut header_data = DatabaseHeader::default(); @@ -1408,7 +1426,7 @@ mod ptrmap_tests { buffer_pool.clone(), ))); - let pager = Pager::new(db_storage, wal, io, page_cache, buffer_pool).unwrap(); + let pager = Pager::new(db_file, wal, io, page_cache, buffer_pool).unwrap(); pager.set_auto_vacuum_mode(AutoVacuumMode::Full); // Allocate all the pages as btree root pages diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index c12fc1406..8b3832957 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -287,6 +287,60 @@ impl DatabaseHeader { } } +pub fn begin_read_database_header( + db_file: Arc, +) -> Result>> { + let drop_fn = Rc::new(|_buf| {}); + #[allow(clippy::arc_with_non_send_sync)] + let buf = Arc::new(RefCell::new(Buffer::allocate(512, drop_fn))); + let result = Arc::new(SpinLock::new(DatabaseHeader::default())); + let header = result.clone(); + let complete = Box::new(move |buf: Arc>| { + let header = header.clone(); + finish_read_database_header(buf, header).unwrap(); + }); + let c = Completion::Read(ReadCompletion::new(buf, complete)); + #[allow(clippy::arc_with_non_send_sync)] + db_file.read_page(DATABASE_HEADER_PAGE_ID, Arc::new(c))?; + Ok(result) +} + +fn finish_read_database_header( + buf: Arc>, + header: Arc>, +) -> Result<()> { + let buf = buf.borrow(); + let buf = buf.as_slice(); + let mut header = header.lock(); + header.magic.copy_from_slice(&buf[0..16]); + header.page_size = u16::from_be_bytes([buf[16], buf[17]]); + header.write_version = buf[18]; + header.read_version = buf[19]; + header.reserved_space = buf[20]; + header.max_embed_frac = buf[21]; + header.min_embed_frac = buf[22]; + header.min_leaf_frac = buf[23]; + header.change_counter = u32::from_be_bytes([buf[24], buf[25], buf[26], buf[27]]); + header.database_size = u32::from_be_bytes([buf[28], buf[29], buf[30], buf[31]]); + header.freelist_trunk_page = u32::from_be_bytes([buf[32], buf[33], buf[34], buf[35]]); + header.freelist_pages = u32::from_be_bytes([buf[36], buf[37], buf[38], buf[39]]); + header.schema_cookie = u32::from_be_bytes([buf[40], buf[41], buf[42], buf[43]]); + header.schema_format = u32::from_be_bytes([buf[44], buf[45], buf[46], buf[47]]); + header.default_page_cache_size = i32::from_be_bytes([buf[48], buf[49], buf[50], buf[51]]); + if header.default_page_cache_size == 0 { + header.default_page_cache_size = DEFAULT_CACHE_SIZE; + } + header.vacuum_mode_largest_root_page = u32::from_be_bytes([buf[52], buf[53], buf[54], buf[55]]); + header.text_encoding = u32::from_be_bytes([buf[56], buf[57], buf[58], buf[59]]); + header.user_version = i32::from_be_bytes([buf[60], buf[61], buf[62], buf[63]]); + header.incremental_vacuum_enabled = u32::from_be_bytes([buf[64], buf[65], buf[66], buf[67]]); + header.application_id = u32::from_be_bytes([buf[68], buf[69], buf[70], buf[71]]); + header.reserved_for_expansion.copy_from_slice(&buf[72..92]); + header.version_valid_for = u32::from_be_bytes([buf[92], buf[93], buf[94], buf[95]]); + header.version_number = u32::from_be_bytes([buf[96], buf[97], buf[98], buf[99]]); + Ok(()) +} + pub fn write_header_to_buf(buf: &mut [u8], header: &DatabaseHeader) { buf[0..16].copy_from_slice(&header.magic); buf[16..18].copy_from_slice(&header.page_size.to_be_bytes()); diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index 6c5769a9b..d1e175305 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -51,8 +51,7 @@ use crate::{ }; use crate::{ - info, maybe_init_database_file, BufferPool, MvCursor, OpenFlags, RefValue, Row, StepResult, - TransactionState, IO, + info, BufferPool, MvCursor, OpenFlags, RefValue, Row, StepResult, TransactionState, IO, }; use super::{ @@ -5211,7 +5210,6 @@ pub fn op_open_ephemeral( let io = conn.pager.io.get_memory_io(); let file = io.open_file("", OpenFlags::Create, true)?; - maybe_init_database_file(&file, &(io.clone() as Arc))?; let db_file = Arc::new(FileMemoryStorage::new(file)); let buffer_pool = Rc::new(BufferPool::new(None));