From 6cd0f036439b9c83020991d3bad7f427ebdda065 Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Fri, 15 Nov 2024 12:09:07 +0100 Subject: [PATCH] core: create databases from limbo --- bindings/wasm/lib.rs | 43 +++++++++++++++++++++---------- bindings/wasm/vfs.js | 8 ++++-- core/lib.rs | 47 ++++++++++++++++++++++++++++++++-- core/storage/btree.rs | 37 +++++++++++++------------- core/storage/pager.rs | 41 ++++++++++++++++++----------- core/storage/sqlite3_ondisk.rs | 32 ++++++++++++++++++++++- 6 files changed, 156 insertions(+), 52 deletions(-) diff --git a/bindings/wasm/lib.rs b/bindings/wasm/lib.rs index 47302d6bc..c4a224905 100644 --- a/bindings/wasm/lib.rs +++ b/bindings/wasm/lib.rs @@ -1,4 +1,4 @@ -use limbo_core::{OpenFlags, Page, Pager, Result, WalFile, IO}; +use limbo_core::{maybe_init_database_file, OpenFlags, Pager, Result, WalFile}; use std::cell::RefCell; use std::rc::Rc; use std::sync::Arc; @@ -15,10 +15,11 @@ pub struct Database { impl Database { #[wasm_bindgen(constructor)] pub fn new(path: &str) -> Database { - let io = Arc::new(PlatformIO { vfs: VFS::new() }); + let io: Arc = Arc::new(PlatformIO { vfs: VFS::new() }); let file = io - .open_file(path, limbo_core::OpenFlags::None, false) + .open_file(path, limbo_core::OpenFlags::Create, false) .unwrap(); + maybe_init_database_file(&file, &io).unwrap(); let page_io = Rc::new(DatabaseStorage::new(file)); let db_header = Pager::begin_open(page_io.clone()).unwrap(); let wal_path = format!("{}-wal", path); @@ -111,14 +112,24 @@ impl limbo_core::File for File { fn pwrite( &self, - _pos: usize, - _buffer: Rc>, - _c: Rc, + pos: usize, + buffer: Rc>, + c: Rc, ) -> Result<()> { + let w = match &*c { + limbo_core::Completion::Write(w) => w, + _ => unreachable!(), + }; + let buf = buffer.borrow(); + let buf: &[u8] = buf.as_slice(); + self.vfs.pwrite(self.fd, buf, pos); + w.complete(buf.len() as i32); Ok(()) } - fn sync(&self, _c: Rc) -> Result<()> { + fn sync(&self, c: Rc) -> Result<()> { + self.vfs.sync(self.fd); + c.complete(0); Ok(()) } @@ -138,7 +149,7 @@ impl limbo_core::IO for PlatformIO { _flags: OpenFlags, _direct: bool, ) -> Result> { - let fd = self.vfs.open(path); + let fd = self.vfs.open(path, "w+"); Ok(Rc::new(File { vfs: VFS::new(), fd, @@ -207,11 +218,14 @@ impl limbo_core::DatabaseStorage for DatabaseStorage { fn write_page( &self, - _page_idx: usize, - _buffer: Rc>, - _c: Rc, + page_idx: usize, + buffer: Rc>, + c: Rc, ) -> Result<()> { - todo!() + let size = buffer.borrow().len(); + let pos = (page_idx - 1) * size; + self.file.pwrite(pos, buffer, c)?; + Ok(()) } fn sync(&self, _c: Rc) -> Result<()> { @@ -227,7 +241,7 @@ extern "C" { fn new() -> VFS; #[wasm_bindgen(method)] - fn open(this: &VFS, path: &str) -> i32; + fn open(this: &VFS, path: &str, flags: &str) -> i32; #[wasm_bindgen(method)] fn close(this: &VFS, fd: i32) -> bool; @@ -240,6 +254,9 @@ extern "C" { #[wasm_bindgen(method)] fn size(this: &VFS, fd: i32) -> u64; + + #[wasm_bindgen(method)] + fn sync(this: &VFS, fd: i32) -> u64; } #[wasm_bindgen(start)] diff --git a/bindings/wasm/vfs.js b/bindings/wasm/vfs.js index 19527d288..143945aec 100644 --- a/bindings/wasm/vfs.js +++ b/bindings/wasm/vfs.js @@ -4,8 +4,8 @@ class VFS { constructor() { } - open(path) { - return fs.openSync(path, 'r'); + open(path, flags) { + return fs.openSync(path, flags); } close(fd) { @@ -24,6 +24,10 @@ class VFS { let stats = fs.fstatSync(fd); return BigInt(stats.size); } + + sync(fd) { + return fs.fsyncSync(fd); + } } module.exports = { VFS }; diff --git a/core/lib.rs b/core/lib.rs index 7fdc257a4..980bc96fd 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -22,9 +22,11 @@ use sqlite3_parser::{ast::Cmd, lexer::sql::Parser}; use std::rc::Weak; use std::sync::{Arc, OnceLock}; use std::{cell::RefCell, rc::Rc}; +use storage::btree::btree_init_page; #[cfg(feature = "fs")] use storage::database::FileStorage; -use storage::sqlite3_ondisk::DatabaseHeader; +use storage::pager::allocate_page; +use storage::sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE}; pub use storage::wal::WalFile; use translate::optimizer::optimize_plan; @@ -64,7 +66,8 @@ pub struct Database { impl Database { #[cfg(feature = "fs")] pub fn open_file(io: Arc, path: &str) -> Result> { - let file = io.open_file(path, io::OpenFlags::None, true)?; + let file = io.open_file(path, io::OpenFlags::Create, true)?; + maybe_init_database_file(&file, &io)?; let page_io = Rc::new(FileStorage::new(file)); let wal_path = format!("{}-wal", path); let db_header = Pager::begin_open(page_io.clone())?; @@ -162,6 +165,46 @@ impl Database { } } +pub fn maybe_init_database_file(file: &Rc, io: &Arc) -> Result<()> { + if file.size().unwrap() == 0 { + // init db + let db_header = DatabaseHeader::default(); + let page1 = allocate_page( + 1, + &Rc::new(BufferPool::new(db_header.page_size as usize)), + DATABASE_HEADER_SIZE, + ); + { + // Create the sqlite_schema table, for this we just need to create the btree page + // for the first page of the database which is basically like any other btree page + // but with a 100 byte offset, so we just init the page so that sqlite understands + // this is a correct page. + btree_init_page( + &page1, + storage::sqlite3_ondisk::PageType::TableLeaf, + &db_header, + ); + + let mut page = page1.borrow_mut(); + let contents = page.contents.as_mut().unwrap(); + contents.write_database_header(&db_header); + // write the first page to disk synchronously + let flag_complete = Rc::new(RefCell::new(false)); + { + let flag_complete = flag_complete.clone(); + let completion = Completion::Write(WriteCompletion::new(Box::new(move |_| { + *flag_complete.borrow_mut() = true; + }))); + file.pwrite(0, contents.buffer.clone(), Rc::new(completion)) + .unwrap(); + } + io.run_once()?; + assert!(*flag_complete.borrow()); + } + }; + Ok(()) +} + pub struct Connection { pager: Rc, schema: Rc, diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 28e76fbd1..18ceb5f93 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -1086,25 +1086,7 @@ impl BTreeCursor { fn allocate_page(&self, page_type: PageType) -> Rc> { let page = self.pager.allocate_page().unwrap(); - - { - // setup btree page - let mut contents = page.borrow_mut(); - debug!("allocating page {}", contents.id); - let contents = contents.contents.as_mut().unwrap(); - let id = page_type as u8; - contents.write_u8(BTREE_HEADER_OFFSET_TYPE, id); - contents.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, 0); - contents.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, 0); - - let db_header = RefCell::borrow(&self.database_header); - let cell_content_area_start = db_header.page_size - db_header.unused_space as u16; - contents.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, cell_content_area_start); - - contents.write_u8(BTREE_HEADER_OFFSET_FRAGMENTED, 0); - contents.write_u32(BTREE_HEADER_OFFSET_RIGHTMOST, 0); - } - + btree_init_page(&page, page_type, &*self.database_header.borrow()); page } @@ -1699,6 +1681,23 @@ impl Cursor for BTreeCursor { } } +pub fn btree_init_page(page: &Rc>, page_type: PageType, db_header: &DatabaseHeader) { + // setup btree page + let mut contents = page.borrow_mut(); + debug!("allocating page {}", contents.id); + let contents = contents.contents.as_mut().unwrap(); + let id = page_type as u8; + contents.write_u8(BTREE_HEADER_OFFSET_TYPE, id); + contents.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, 0); + contents.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, 0); + + let cell_content_area_start = db_header.page_size - db_header.unused_space as u16; + contents.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, cell_content_area_start); + + contents.write_u8(BTREE_HEADER_OFFSET_FRAGMENTED, 0); + contents.write_u32(BTREE_HEADER_OFFSET_RIGHTMOST, 0); +} + fn to_static_buf(buf: &[u8]) -> &'static [u8] { unsafe { std::mem::transmute::<&[u8], &'static [u8]>(buf) } } diff --git a/core/storage/pager.rs b/core/storage/pager.rs index 7e71fedb9..c51ea31ab 100644 --- a/core/storage/pager.rs +++ b/core/storage/pager.rs @@ -621,25 +621,13 @@ impl Pager { } } - let page_ref = Rc::new(RefCell::new(Page::new(0))); + let page_ref = allocate_page(header.database_size as usize, &self.buffer_pool, 0); { // setup page and add to cache - let mut page = RefCell::borrow_mut(&page_ref); - page.id = header.database_size as usize; + let page = page_ref.borrow_mut(); page.set_dirty(); self.add_dirty(page.id); - let buffer = self.buffer_pool.get(); - let bp = self.buffer_pool.clone(); - let drop_fn = Rc::new(move |buf| { - bp.put(buf); - }); - let buffer = Rc::new(RefCell::new(Buffer::new(buffer, drop_fn))); - page.contents = Some(PageContent { - offset: 0, - buffer, - overflow_cells: Vec::new(), - }); - let mut cache = RefCell::borrow_mut(&self.page_cache); + let mut cache = self.page_cache.borrow_mut(); cache.insert(page.id, page_ref.clone()); } Ok(page_ref) @@ -657,3 +645,26 @@ impl Pager { (db_header.page_size - db_header.unused_space as u16) as usize } } + +pub fn allocate_page( + page_id: usize, + buffer_pool: &Rc, + offset: usize, +) -> Rc> { + let page_ref = Rc::new(RefCell::new(Page::new(page_id))); + { + let mut page = RefCell::borrow_mut(&page_ref); + let buffer = buffer_pool.get(); + let bp = buffer_pool.clone(); + let drop_fn = Rc::new(move |buf| { + bp.put(buf); + }); + let buffer = Rc::new(RefCell::new(Buffer::new(buffer, drop_fn))); + page.contents = Some(PageContent { + offset, + buffer, + overflow_cells: Vec::new(), + }); + } + page_ref +} diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 00e70adbd..da91b815b 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -61,7 +61,7 @@ const DEFAULT_CACHE_SIZE: i32 = -2000; // Minimum number of pages that cache can hold. pub const MIN_PAGE_CACHE_SIZE: usize = 10; -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] pub struct DatabaseHeader { magic: [u8; 16], pub page_size: u16, @@ -114,6 +114,36 @@ pub struct WalFrameHeader { checksum_2: u32, } +impl Default for DatabaseHeader { + fn default() -> Self { + Self { + magic: *b"SQLite format 3\0", + page_size: 4096, + write_version: 2, + read_version: 2, + unused_space: 0, + max_embed_frac: 64, + min_embed_frac: 32, + min_leaf_frac: 32, + change_counter: 1, + database_size: 1, + freelist_trunk_page: 0, + freelist_pages: 0, + schema_cookie: 0, + schema_format: 4, // latest format, new sqlite3 databases use this format + default_cache_size: 500, // pages + vacuum: 0, + text_encoding: 1, // utf-8 + user_version: 1, + incremental_vacuum: 0, + application_id: 0, + reserved: [0; 20], + version_valid_for: 3047000, + version_number: 3047000, + } + } +} + pub fn begin_read_database_header( page_io: Rc, ) -> Result>> {