aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-02-23 17:05:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Use sieve algorithm in page cache in place of full LRU

Browse Source
This commit is contained in:
PThorpe92
2025-09-01 22:57:25 -04:00
parent d60628dc8e
commit 03d5598cfb
5 changed files with 1009 additions and 876 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
core/lib.rs
View File

@@ -80,7 +80,7 @@ use std::{
use storage::database::DatabaseFile;
pub use storage::database::IOContext;
pub use storage::encryption::{EncryptionContext, EncryptionKey};
use storage::page_cache::DumbLruPageCache;
use storage::page_cache::PageCache;
use storage::pager::{AtomicDbState, DbState};
use storage::sqlite3_ondisk::PageSize;
pub use storage::{
@@ -187,7 +187,7 @@ pub struct Database {
buffer_pool: Arc<BufferPool>,
// Shared structures of a Database are the parts that are common to multiple threads that might
// create DB connections.
_shared_page_cache: Arc<RwLock<DumbLruPageCache>>,
_shared_page_cache: Arc<RwLock<PageCache>>,
shared_wal: Arc<RwLock<WalFileShared>>,
db_state: Arc<AtomicDbState>,
init_lock: Arc<Mutex<()>>,
@@ -385,7 +385,7 @@ impl Database {
DbState::Initialized
};
let shared_page_cache = Arc::new(RwLock::new(DumbLruPageCache::default()));
let shared_page_cache = Arc::new(RwLock::new(PageCache::default()));
let syms = SymbolTable::new();
let arena_size = if std::env::var("TESTING").is_ok_and(|v| v.eq_ignore_ascii_case("true")) {
BufferPool::TEST_ARENA_SIZE
@@ -576,7 +576,7 @@ impl Database {
self.db_file.clone(),
Some(wal),
self.io.clone(),
Arc::new(RwLock::new(DumbLruPageCache::default())),
Arc::new(RwLock::new(PageCache::default())),
buffer_pool.clone(),
db_state,
self.init_lock.clone(),
@@ -599,7 +599,7 @@ impl Database {
self.db_file.clone(),
None,
self.io.clone(),
Arc::new(RwLock::new(DumbLruPageCache::default())),
Arc::new(RwLock::new(PageCache::default())),
buffer_pool.clone(),
db_state,
Arc::new(Mutex::new(())),

4
core/storage/btree.rs
View File

@@ -7376,7 +7376,7 @@ mod tests {
schema::IndexColumn,
storage::{
database::DatabaseFile,
page_cache::DumbLruPageCache,
page_cache::PageCache,
pager::{AtomicDbState, DbState},
sqlite3_ondisk::PageSize,
},
@@ -8631,7 +8631,7 @@ mod tests {
db_file,
Some(wal),
io,
Arc::new(parking_lot::RwLock::new(DumbLruPageCache::new(10))),
Arc::new(parking_lot::RwLock::new(PageCache::new(10))),
buffer_pool,
Arc::new(AtomicDbState::new(DbState::Uninitialized)),
Arc::new(Mutex::new(())),

1845
core/storage/page_cache.rs
View File

File diff suppressed because it is too large Load Diff

22
core/storage/pager.rs
View File

@@ -25,7 +25,7 @@ use std::sync::{Arc, Mutex};
use tracing::{instrument, trace, Level};
use super::btree::btree_init_page;
use super::page_cache::{CacheError, CacheResizeResult, DumbLruPageCache, PageCacheKey};
use super::page_cache::{CacheError, CacheResizeResult, PageCache, PageCacheKey};
use super::sqlite3_ondisk::begin_write_btree_page;
use super::wal::CheckpointMode;
use crate::storage::encryption::{CipherMode, EncryptionContext, EncryptionKey};
@@ -129,7 +129,7 @@ pub struct PageInner {
/// requests unpinning via [Page::unpin], the pin count will still be >0 if the outer
/// code path has not yet requested to unpin the page as well.
///
/// Note that [DumbLruPageCache::clear] evicts the pages even if pinned, so as long as
/// Note that [PageCache::clear] evicts the pages even if pinned, so as long as
/// we clear the page cache on errors, pins will not 'leak'.
pub pin_count: AtomicUsize,
/// The WAL frame number this page was loaded from (0 if loaded from main DB file)
@@ -464,7 +464,7 @@ pub struct Pager {
/// in-memory databases, ephemeral tables and ephemeral indexes do not have a WAL.
pub(crate) wal: Option<Rc<RefCell<dyn Wal>>>,
/// A page cache for the database.
page_cache: Arc<RwLock<DumbLruPageCache>>,
page_cache: Arc<RwLock<PageCache>>,
/// Buffer pool for temporary data storage.
pub buffer_pool: Arc<BufferPool>,
/// I/O interface for input/output operations.
@@ -564,7 +564,7 @@ impl Pager {
db_file: Arc<dyn DatabaseStorage>,
wal: Option<Rc<RefCell<dyn Wal>>>,
io: Arc<dyn crate::io::IO>,
page_cache: Arc<RwLock<DumbLruPageCache>>,
page_cache: Arc<RwLock<PageCache>>,
buffer_pool: Arc<BufferPool>,
db_state: Arc<AtomicDbState>,
init_lock: Arc<Mutex<()>>,
@@ -1153,7 +1153,7 @@ impl Pager {
&self,
page_idx: usize,
page: PageRef,
page_cache: &mut DumbLruPageCache,
page_cache: &mut PageCache,
) -> Result<()> {
let page_key = PageCacheKey::new(page_idx);
match page_cache.insert(page_key, page.clone()) {
@@ -1981,7 +1981,7 @@ impl Pager {
trunk_page.get_contents().as_ptr().fill(0);
let page_key = PageCacheKey::new(trunk_page.get().id);
{
let mut page_cache = self.page_cache.write();
let page_cache = self.page_cache.read();
turso_assert!(
page_cache.contains_key(&page_key),
"page {} is not in cache",
@@ -2013,7 +2013,7 @@ impl Pager {
leaf_page.get_contents().as_ptr().fill(0);
let page_key = PageCacheKey::new(leaf_page.get().id);
{
let mut page_cache = self.page_cache.write();
let page_cache = self.page_cache.read();
turso_assert!(
page_cache.contains_key(&page_key),
"page {} is not in cache",
@@ -2402,14 +2402,14 @@ mod tests {
use parking_lot::RwLock;
use crate::storage::page_cache::{DumbLruPageCache, PageCacheKey};
use crate::storage::page_cache::{PageCache, PageCacheKey};
use super::Page;
#[test]
fn test_shared_cache() {
// ensure cache can be shared between threads
let cache = Arc::new(RwLock::new(DumbLruPageCache::new(10)));
let cache = Arc::new(RwLock::new(PageCache::new(10)));
let thread = {
let cache = cache.clone();
@@ -2442,7 +2442,7 @@ mod ptrmap_tests {
use crate::io::{MemoryIO, OpenFlags, IO};
use crate::storage::buffer_pool::BufferPool;
use crate::storage::database::{DatabaseFile, DatabaseStorage};
use crate::storage::page_cache::DumbLruPageCache;
use crate::storage::page_cache::PageCache;
use crate::storage::pager::Pager;
use crate::storage::sqlite3_ondisk::PageSize;
use crate::storage::wal::{WalFile, WalFileShared};
@@ -2471,7 +2471,7 @@ mod ptrmap_tests {
let pages = initial_db_pages + 10;
let sz = std::cmp::max(std::cmp::min(pages, 64), pages);
let buffer_pool = BufferPool::begin_init(&io, (sz * page_size) as usize);
let page_cache = Arc::new(RwLock::new(DumbLruPageCache::new(sz as usize)));
let page_cache = Arc::new(RwLock::new(PageCache::new(sz as usize)));
let wal = Rc::new(RefCell::new(WalFile::new(
io.clone(),

4
core/vdbe/execute.rs
View File

@@ -6,7 +6,7 @@ use crate::storage::btree::{
integrity_check, IntegrityCheckError, IntegrityCheckState, PageCategory,
};
use crate::storage::database::DatabaseFile;
use crate::storage::page_cache::DumbLruPageCache;
use crate::storage::page_cache::PageCache;
use crate::storage::pager::{AtomicDbState, CreateBTreeFlags, DbState};
use crate::storage::sqlite3_ondisk::read_varint;
use crate::translate::collate::CollationSeq;
@@ -6996,7 +6996,7 @@ pub fn op_open_ephemeral(
.get();
let buffer_pool = program.connection._db.buffer_pool.clone();
let page_cache = Arc::new(RwLock::new(DumbLruPageCache::default()));
let page_cache = Arc::new(RwLock::new(PageCache::default()));
let pager = Rc::new(Pager::new(
db_file,