aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-02-23 17:05:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

core: create databases from limbo

Browse Source
This commit is contained in:
Pere Diaz Bou
2024-11-15 12:09:07 +01:00
parent 87c80b2a07
commit 6cd0f03643
6 changed files with 156 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
bindings/wasm/lib.rs
View File

@@ -1,4 +1,4 @@
use limbo_core::{OpenFlags, Page, Pager, Result, WalFile, IO};
use limbo_core::{maybe_init_database_file, OpenFlags, Pager, Result, WalFile};
use std::cell::RefCell;
use std::rc::Rc;
use std::sync::Arc;
@@ -15,10 +15,11 @@ pub struct Database {
impl Database {
#[wasm_bindgen(constructor)]
pub fn new(path: &str) -> Database {
let io = Arc::new(PlatformIO { vfs: VFS::new() });
let io: Arc<dyn limbo_core::IO> = Arc::new(PlatformIO { vfs: VFS::new() });
let file = io
.open_file(path, limbo_core::OpenFlags::None, false)
.open_file(path, limbo_core::OpenFlags::Create, false)
.unwrap();
maybe_init_database_file(&file, &io).unwrap();
let page_io = Rc::new(DatabaseStorage::new(file));
let db_header = Pager::begin_open(page_io.clone()).unwrap();
let wal_path = format!("{}-wal", path);
@@ -111,14 +112,24 @@ impl limbo_core::File for File {
fn pwrite(
&self,
_pos: usize,
_buffer: Rc<std::cell::RefCell<limbo_core::Buffer>>,
_c: Rc<limbo_core::Completion>,
pos: usize,
buffer: Rc<std::cell::RefCell<limbo_core::Buffer>>,
c: Rc<limbo_core::Completion>,
) -> Result<()> {
let w = match &*c {
limbo_core::Completion::Write(w) => w,
_ => unreachable!(),
};
let buf = buffer.borrow();
let buf: &[u8] = buf.as_slice();
self.vfs.pwrite(self.fd, buf, pos);
w.complete(buf.len() as i32);
Ok(())
}
fn sync(&self, _c: Rc<limbo_core::Completion>) -> Result<()> {
fn sync(&self, c: Rc<limbo_core::Completion>) -> Result<()> {
self.vfs.sync(self.fd);
c.complete(0);
Ok(())
}
@@ -138,7 +149,7 @@ impl limbo_core::IO for PlatformIO {
_flags: OpenFlags,
_direct: bool,
) -> Result<Rc<dyn limbo_core::File>> {
let fd = self.vfs.open(path);
let fd = self.vfs.open(path, "w+");
Ok(Rc::new(File {
vfs: VFS::new(),
fd,
@@ -207,11 +218,14 @@ impl limbo_core::DatabaseStorage for DatabaseStorage {
fn write_page(
&self,
_page_idx: usize,
_buffer: Rc<std::cell::RefCell<limbo_core::Buffer>>,
_c: Rc<limbo_core::Completion>,
page_idx: usize,
buffer: Rc<std::cell::RefCell<limbo_core::Buffer>>,
c: Rc<limbo_core::Completion>,
) -> Result<()> {
todo!()
let size = buffer.borrow().len();
let pos = (page_idx - 1) * size;
self.file.pwrite(pos, buffer, c)?;
Ok(())
}
fn sync(&self, _c: Rc<limbo_core::Completion>) -> Result<()> {
@@ -227,7 +241,7 @@ extern "C" {
fn new() -> VFS;
#[wasm_bindgen(method)]
fn open(this: &VFS, path: &str) -> i32;
fn open(this: &VFS, path: &str, flags: &str) -> i32;
#[wasm_bindgen(method)]
fn close(this: &VFS, fd: i32) -> bool;
@@ -240,6 +254,9 @@ extern "C" {
#[wasm_bindgen(method)]
fn size(this: &VFS, fd: i32) -> u64;
#[wasm_bindgen(method)]
fn sync(this: &VFS, fd: i32) -> u64;
}
#[wasm_bindgen(start)]

8
bindings/wasm/vfs.js
View File

@@ -4,8 +4,8 @@ class VFS {
constructor() {
}
open(path) {
return fs.openSync(path, 'r');
open(path, flags) {
return fs.openSync(path, flags);
}
close(fd) {
@@ -24,6 +24,10 @@ class VFS {
let stats = fs.fstatSync(fd);
return BigInt(stats.size);
}
sync(fd) {
return fs.fsyncSync(fd);
}
}
module.exports = { VFS };

47
core/lib.rs
View File

@@ -22,9 +22,11 @@ use sqlite3_parser::{ast::Cmd, lexer::sql::Parser};
use std::rc::Weak;
use std::sync::{Arc, OnceLock};
use std::{cell::RefCell, rc::Rc};
use storage::btree::btree_init_page;
#[cfg(feature = "fs")]
use storage::database::FileStorage;
use storage::sqlite3_ondisk::DatabaseHeader;
use storage::pager::allocate_page;
use storage::sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE};
pub use storage::wal::WalFile;
use translate::optimizer::optimize_plan;
@@ -64,7 +66,8 @@ pub struct Database {
impl Database {
#[cfg(feature = "fs")]
pub fn open_file(io: Arc<dyn IO>, path: &str) -> Result<Rc<Database>> {
let file = io.open_file(path, io::OpenFlags::None, true)?;
let file = io.open_file(path, io::OpenFlags::Create, true)?;
maybe_init_database_file(&file, &io)?;
let page_io = Rc::new(FileStorage::new(file));
let wal_path = format!("{}-wal", path);
let db_header = Pager::begin_open(page_io.clone())?;
@@ -162,6 +165,46 @@ impl Database {
}
}
pub fn maybe_init_database_file(file: &Rc<dyn File>, io: &Arc<dyn IO>) -> Result<()> {
if file.size().unwrap() == 0 {
// init db
let db_header = DatabaseHeader::default();
let page1 = allocate_page(
1,
&Rc::new(BufferPool::new(db_header.page_size as usize)),
DATABASE_HEADER_SIZE,
);
{
// Create the sqlite_schema table, for this we just need to create the btree page
// for the first page of the database which is basically like any other btree page
// but with a 100 byte offset, so we just init the page so that sqlite understands
// this is a correct page.
btree_init_page(
&page1,
storage::sqlite3_ondisk::PageType::TableLeaf,
&db_header,
);
let mut page = page1.borrow_mut();
let contents = page.contents.as_mut().unwrap();
contents.write_database_header(&db_header);
// write the first page to disk synchronously
let flag_complete = Rc::new(RefCell::new(false));
{
let flag_complete = flag_complete.clone();
let completion = Completion::Write(WriteCompletion::new(Box::new(move |_| {
*flag_complete.borrow_mut() = true;
})));
file.pwrite(0, contents.buffer.clone(), Rc::new(completion))
.unwrap();
}
io.run_once()?;
assert!(*flag_complete.borrow());
}
};
Ok(())
}
pub struct Connection {
pager: Rc<Pager>,
schema: Rc<Schema>,

37
core/storage/btree.rs
View File

@@ -1086,25 +1086,7 @@ impl BTreeCursor {
fn allocate_page(&self, page_type: PageType) -> Rc<RefCell<Page>> {
let page = self.pager.allocate_page().unwrap();
{
// setup btree page
let mut contents = page.borrow_mut();
debug!("allocating page {}", contents.id);
let contents = contents.contents.as_mut().unwrap();
let id = page_type as u8;
contents.write_u8(BTREE_HEADER_OFFSET_TYPE, id);
contents.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, 0);
contents.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, 0);
let db_header = RefCell::borrow(&self.database_header);
let cell_content_area_start = db_header.page_size - db_header.unused_space as u16;
contents.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, cell_content_area_start);
contents.write_u8(BTREE_HEADER_OFFSET_FRAGMENTED, 0);
contents.write_u32(BTREE_HEADER_OFFSET_RIGHTMOST, 0);
}
btree_init_page(&page, page_type, &*self.database_header.borrow());
page
}
@@ -1699,6 +1681,23 @@ impl Cursor for BTreeCursor {
}
}
pub fn btree_init_page(page: &Rc<RefCell<Page>>, page_type: PageType, db_header: &DatabaseHeader) {
// setup btree page
let mut contents = page.borrow_mut();
debug!("allocating page {}", contents.id);
let contents = contents.contents.as_mut().unwrap();
let id = page_type as u8;
contents.write_u8(BTREE_HEADER_OFFSET_TYPE, id);
contents.write_u16(BTREE_HEADER_OFFSET_FREEBLOCK, 0);
contents.write_u16(BTREE_HEADER_OFFSET_CELL_COUNT, 0);
let cell_content_area_start = db_header.page_size - db_header.unused_space as u16;
contents.write_u16(BTREE_HEADER_OFFSET_CELL_CONTENT, cell_content_area_start);
contents.write_u8(BTREE_HEADER_OFFSET_FRAGMENTED, 0);
contents.write_u32(BTREE_HEADER_OFFSET_RIGHTMOST, 0);
}
fn to_static_buf(buf: &[u8]) -> &'static [u8] {
unsafe { std::mem::transmute::<&[u8], &'static [u8]>(buf) }
}

41
core/storage/pager.rs
View File

@@ -621,25 +621,13 @@ impl Pager {
}
}
let page_ref = Rc::new(RefCell::new(Page::new(0)));
let page_ref = allocate_page(header.database_size as usize, &self.buffer_pool, 0);
{
// setup page and add to cache
let mut page = RefCell::borrow_mut(&page_ref);
page.id = header.database_size as usize;
let page = page_ref.borrow_mut();
page.set_dirty();
self.add_dirty(page.id);
let buffer = self.buffer_pool.get();
let bp = self.buffer_pool.clone();
let drop_fn = Rc::new(move |buf| {
bp.put(buf);
});
let buffer = Rc::new(RefCell::new(Buffer::new(buffer, drop_fn)));
page.contents = Some(PageContent {
offset: 0,
buffer,
overflow_cells: Vec::new(),
});
let mut cache = RefCell::borrow_mut(&self.page_cache);
let mut cache = self.page_cache.borrow_mut();
cache.insert(page.id, page_ref.clone());
}
Ok(page_ref)
@@ -657,3 +645,26 @@ impl Pager {
(db_header.page_size - db_header.unused_space as u16) as usize
}
}
pub fn allocate_page(
page_id: usize,
buffer_pool: &Rc<BufferPool>,
offset: usize,
) -> Rc<RefCell<Page>> {
let page_ref = Rc::new(RefCell::new(Page::new(page_id)));
{
let mut page = RefCell::borrow_mut(&page_ref);
let buffer = buffer_pool.get();
let bp = buffer_pool.clone();
let drop_fn = Rc::new(move |buf| {
bp.put(buf);
});
let buffer = Rc::new(RefCell::new(Buffer::new(buffer, drop_fn)));
page.contents = Some(PageContent {
offset,
buffer,
overflow_cells: Vec::new(),
});
}
page_ref
}

32
core/storage/sqlite3_ondisk.rs
View File

@@ -61,7 +61,7 @@ const DEFAULT_CACHE_SIZE: i32 = -2000;
// Minimum number of pages that cache can hold.
pub const MIN_PAGE_CACHE_SIZE: usize = 10;
#[derive(Debug, Default, Clone)]
#[derive(Debug, Clone)]
pub struct DatabaseHeader {
magic: [u8; 16],
pub page_size: u16,
@@ -114,6 +114,36 @@ pub struct WalFrameHeader {
checksum_2: u32,
}
impl Default for DatabaseHeader {
fn default() -> Self {
Self {
magic: *b"SQLite format 3\0",
page_size: 4096,
write_version: 2,
read_version: 2,
unused_space: 0,
max_embed_frac: 64,
min_embed_frac: 32,
min_leaf_frac: 32,
change_counter: 1,
database_size: 1,
freelist_trunk_page: 0,
freelist_pages: 0,
schema_cookie: 0,
schema_format: 4, // latest format, new sqlite3 databases use this format
default_cache_size: 500, // pages
vacuum: 0,
text_encoding: 1, // utf-8
user_version: 1,
incremental_vacuum: 0,
application_id: 0,
reserved: [0; 20],
version_valid_for: 3047000,
version_number: 3047000,
}
}
}
pub fn begin_read_database_header(
page_io: Rc<dyn DatabaseStorage>,
) -> Result<Rc<RefCell<DatabaseHeader>>> {