diff --git a/core/lib.rs b/core/lib.rs index d242fed4f..b5a6f3e10 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -43,6 +43,7 @@ mod numeric; use crate::index_method::IndexMethod; use crate::storage::checksum::CHECKSUM_REQUIRED_RESERVED_BYTES; use crate::storage::encryption::AtomicCipherMode; +use crate::storage::pager::{AutoVacuumMode, HeaderRef}; use crate::translate::display::PlanContext; use crate::translate::pragma::TURSO_CDC_DEFAULT_TABLE_NAME; #[cfg(all(feature = "fs", feature = "conn_raw_api"))] @@ -566,6 +567,29 @@ impl Database { pager.enable_encryption(self.opts.enable_encryption); let pager = Arc::new(pager); + if self.db_state.get().is_initialized() { + let header_ref = pager.io.block(|| HeaderRef::from_pager(&pager))?; + + let header = header_ref.borrow(); + + let mode = if header.vacuum_mode_largest_root_page.get() > 0 { + if header.incremental_vacuum_enabled.get() > 0 { + AutoVacuumMode::Incremental + } else { + AutoVacuumMode::Full + } + } else { + AutoVacuumMode::None + }; + + pager.set_auto_vacuum_mode(mode); + + tracing::debug!( + "Opened existing database. Detected auto_vacuum_mode from header: {:?}", + mode + ); + } + let page_size = pager.get_page_size_unchecked(); let default_cache_size = pager diff --git a/core/storage/btree.rs b/core/storage/btree.rs index bebe35a71..ed1fbc84e 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -5784,6 +5784,8 @@ pub(crate) enum PageCategory { Overflow, FreeListTrunk, FreePage, + #[cfg(not(feature = "omit_autovacuum"))] + PointerMap, } #[derive(Clone)] diff --git a/core/storage/pager.rs b/core/storage/pager.rs index 28257c1b1..2d37f9d74 100644 --- a/core/storage/pager.rs +++ b/core/storage/pager.rs @@ -1222,6 +1222,21 @@ impl Pager { BtreePageAllocMode::Exact(root_page_num), )); let allocated_page_id = page.get().id as u32; + + return_if_io!(self.with_header_mut(|header| { + if allocated_page_id + > header.vacuum_mode_largest_root_page.get() + { + tracing::debug!( + "Updating largest root page in header from {} to {}", + header.vacuum_mode_largest_root_page.get(), + allocated_page_id + ); + header.vacuum_mode_largest_root_page = + allocated_page_id.into(); + } + })); + if allocated_page_id != root_page_num { // TODO(Zaid): Handle swapping the allocated page with the desired root page } @@ -2886,7 +2901,7 @@ impl CreateBTreeFlags { ** identifies the parent page in the btree. */ #[cfg(not(feature = "omit_autovacuum"))] -mod ptrmap { +pub(crate) mod ptrmap { use crate::{storage::sqlite3_ondisk::PageSize, LimboError, Result}; // Constants diff --git a/core/translate/pragma.rs b/core/translate/pragma.rs index 8e996bae9..8d1db5ae3 100644 --- a/core/translate/pragma.rs +++ b/core/translate/pragma.rs @@ -241,6 +241,18 @@ fn update_pragma( Ok((program, TransactionMode::None)) } PragmaName::AutoVacuum => { + let is_empty = is_database_empty(resolver.schema, &pager)?; + tracing::debug!( + "Checking if database is empty for auto_vacuum pragma: {}", + is_empty + ); + + if !is_empty { + // SQLite's behavior is to silently ignore this pragma if the database is not empty. + tracing::debug!("Attempted to set auto_vacuum, database is not empty so we are ignoring pragma."); + return Ok((program, TransactionMode::None)); + } + let auto_vacuum_mode = match value { Expr::Name(name) => { let name = name.as_str().as_bytes(); @@ -894,3 +906,30 @@ fn update_page_size(connection: Arc, page_size: u32) -> crate connection.reset_page_size(page_size)?; Ok(()) } + +fn is_database_empty(schema: &Schema, pager: &Arc) -> crate::Result { + if schema.tables.len() > 1 { + return Ok(false); + } + if let Some(table_arc) = schema.tables.values().next() { + let table_name = match table_arc.as_ref() { + crate::schema::Table::BTree(tbl) => &tbl.name, + crate::schema::Table::Virtual(tbl) => &tbl.name, + crate::schema::Table::FromClauseSubquery(tbl) => &tbl.name, + }; + + if table_name != "sqlite_schema" { + return Ok(false); + } + } + + let db_size_result = pager + .io + .block(|| pager.with_header(|header| header.database_size.get())); + + match db_size_result { + Err(_) => Ok(true), + Ok(0 | 1) => Ok(true), + Ok(_) => Ok(false), + } +} diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index 7672a142e..51a848c30 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -8140,6 +8140,33 @@ pub fn op_integrity_check( expected_count: integrity_check_state.freelist_count.expected_count, }); } + + #[cfg(not(feature = "omit_autovacuum"))] + { + let auto_vacuum_mode = pager.get_auto_vacuum_mode(); + if !matches!( + auto_vacuum_mode, + crate::storage::pager::AutoVacuumMode::None + ) { + tracing::debug!("Integrity check: auto-vacuum mode detected ({:?}). Scanning for pointer-map pages.", auto_vacuum_mode); + let page_size = pager.get_page_size_unchecked().get() as usize; + + for page_number in 2..=integrity_check_state.db_size { + if crate::storage::pager::ptrmap::is_ptrmap_page( + page_number as u32, + page_size, + ) { + tracing::debug!("Integrity check: Found and marking pointer-map page as visited: page_id={}", page_number); + + integrity_check_state.start( + page_number as i64, + PageCategory::PointerMap, + errors, + ); + } + } + } + } for page_number in 2..=integrity_check_state.db_size { if !integrity_check_state .page_reference diff --git a/simulator/generation/plan.rs b/simulator/generation/plan.rs index 6c7e66384..42aab9df6 100644 --- a/simulator/generation/plan.rs +++ b/simulator/generation/plan.rs @@ -237,6 +237,7 @@ impl InteractionPlan { Query::AlterTable(_) => stats.alter_table_count += 1, Query::DropIndex(_) => stats.drop_index_count += 1, Query::Placeholder => {} + Query::Pragma(_) => stats.pragma_count += 1, } } for interactions in &self.plan { @@ -771,6 +772,7 @@ pub(crate) struct InteractionStats { pub rollback_count: u32, pub alter_table_count: u32, pub drop_index_count: u32, + pub pragma_count: u32, } impl Display for InteractionStats { diff --git a/simulator/generation/property.rs b/simulator/generation/property.rs index f2530dfeb..8f10b96a2 100644 --- a/simulator/generation/property.rs +++ b/simulator/generation/property.rs @@ -302,7 +302,6 @@ impl Property { let rows = insert.rows(); let row = &rows[*row_index]; - match &query { Query::Delete(Delete { table: t, @@ -1381,6 +1380,7 @@ pub(super) struct Remaining { pub drop: u32, pub alter_table: u32, pub drop_index: u32, + pub pragma_count: u32, } pub(super) fn remaining( @@ -1401,6 +1401,7 @@ pub(super) fn remaining( let total_drop = (max_interactions * opts.drop_table_weight) / total_weight; let total_alter_table = (max_interactions * opts.alter_table_weight) / total_weight; let total_drop_index = (max_interactions * opts.drop_index) / total_weight; + let total_pragma = (max_interactions * opts.pragma_weight) / total_weight; let remaining_select = total_select .checked_sub(stats.select_count) @@ -1421,6 +1422,9 @@ pub(super) fn remaining( .checked_sub(stats.update_count) .unwrap_or_default(); let remaining_drop = total_drop.checked_sub(stats.drop_count).unwrap_or_default(); + let remaining_pragma = total_pragma + .checked_sub(stats.pragma_count) + .unwrap_or_default(); let remaining_alter_table = total_alter_table .checked_sub(stats.alter_table_count) @@ -1455,6 +1459,7 @@ pub(super) fn remaining( update: remaining_update, alter_table: remaining_alter_table, drop_index: remaining_drop_index, + pragma_count: remaining_pragma, } } diff --git a/simulator/generation/query.rs b/simulator/generation/query.rs index 7445bd744..02d82c17f 100644 --- a/simulator/generation/query.rs +++ b/simulator/generation/query.rs @@ -5,12 +5,15 @@ use crate::{ use rand::{ Rng, distr::{Distribution, weighted::WeightedIndex}, + seq::IndexedRandom, }; use sql_generation::{ generation::{Arbitrary, ArbitraryFrom, GenerationContext, query::SelectFree}, model::{ query::{ - Create, CreateIndex, Delete, DropIndex, Insert, Select, alter_table::AlterTable, + Create, CreateIndex, Delete, DropIndex, Insert, Select, + alter_table::AlterTable, + pragma::{Pragma, VacuumMode}, update::Update, }, table::Table, @@ -82,6 +85,18 @@ fn random_create_index( Query::CreateIndex(create_index) } +fn random_pragma(rng: &mut R, _conn_ctx: &impl GenerationContext) -> Query { + const ALL_MODES: [VacuumMode; 2] = [ + VacuumMode::None, + // VacuumMode::Incremental, not implemented yet + VacuumMode::Full, + ]; + + let mode = ALL_MODES.choose(rng).unwrap(); + + Query::Pragma(Pragma::AutoVacuumMode(mode.clone())) +} + fn random_alter_table( rng: &mut R, conn_ctx: &impl GenerationContext, @@ -140,6 +155,7 @@ impl QueryDiscriminants { QueryDiscriminants::Placeholder => { unreachable!("Query Placeholders should not be generated") } + QueryDiscriminants::Pragma => random_pragma, } } @@ -164,6 +180,7 @@ impl QueryDiscriminants { QueryDiscriminants::Placeholder => { unreachable!("Query Placeholders should not be generated") } + QueryDiscriminants::Pragma => remaining.pragma_count, } } } diff --git a/simulator/model/mod.rs b/simulator/model/mod.rs index cdebd76fe..1ffa5a161 100644 --- a/simulator/model/mod.rs +++ b/simulator/model/mod.rs @@ -9,6 +9,7 @@ use sql_generation::model::{ query::{ Create, CreateIndex, Delete, Drop, DropIndex, Insert, Select, alter_table::{AlterTable, AlterTableType}, + pragma::Pragma, select::{CompoundOperator, FromClause, ResultColumn, SelectInner}, transaction::{Begin, Commit, Rollback}, update::Update, @@ -34,6 +35,7 @@ pub enum Query { Begin(Begin), Commit(Commit), Rollback(Rollback), + Pragma(Pragma), /// Placeholder query that still needs to be generated Placeholder, } @@ -81,8 +83,11 @@ impl Query { | Query::DropIndex(DropIndex { table_name: table, .. }) => IndexSet::from_iter([table.clone()]), - Query::Begin(_) | Query::Commit(_) | Query::Rollback(_) => IndexSet::new(), - Query::Placeholder => IndexSet::new(), + Query::Begin(_) + | Query::Commit(_) + | Query::Rollback(_) + | Query::Placeholder + | Query::Pragma(_) => IndexSet::new(), } } pub fn uses(&self) -> Vec { @@ -107,6 +112,7 @@ impl Query { }) => vec![table.clone()], Query::Begin(..) | Query::Commit(..) | Query::Rollback(..) => vec![], Query::Placeholder => vec![], + Query::Pragma(_) => vec![], } } @@ -147,6 +153,7 @@ impl Display for Query { Self::Commit(commit) => write!(f, "{commit}"), Self::Rollback(rollback) => write!(f, "{rollback}"), Self::Placeholder => Ok(()), + Query::Pragma(pragma) => write!(f, "{pragma}"), } } } @@ -169,12 +176,14 @@ impl Shadow for Query { Query::Commit(commit) => Ok(commit.shadow(env)), Query::Rollback(rollback) => Ok(rollback.shadow(env)), Query::Placeholder => Ok(vec![]), + Query::Pragma(Pragma::AutoVacuumMode(_)) => Ok(vec![]), } } } bitflags! { pub struct QueryCapabilities: u32 { + const NONE = 0; const CREATE = 1 << 0; const SELECT = 1 << 1; const INSERT = 1 << 2; @@ -222,6 +231,7 @@ impl From for QueryCapabilities { QueryDiscriminants::Placeholder => { unreachable!("QueryCapabilities do not apply to query Placeholder") } + QueryDiscriminants::Pragma => QueryCapabilities::NONE, } } } @@ -237,6 +247,7 @@ impl QueryDiscriminants { QueryDiscriminants::CreateIndex, QueryDiscriminants::AlterTable, QueryDiscriminants::DropIndex, + QueryDiscriminants::Pragma, ]; } diff --git a/simulator/profiles/query.rs b/simulator/profiles/query.rs index 95bcf146a..3f0de4bd9 100644 --- a/simulator/profiles/query.rs +++ b/simulator/profiles/query.rs @@ -26,6 +26,8 @@ pub struct QueryProfile { pub alter_table_weight: u32, #[garde(skip)] pub drop_index: u32, + #[garde(skip)] + pub pragma_weight: u32, } impl Default for QueryProfile { @@ -41,6 +43,7 @@ impl Default for QueryProfile { drop_table_weight: 2, alter_table_weight: 2, drop_index: 2, + pragma_weight: 2, } } } @@ -56,6 +59,7 @@ impl QueryProfile { + self.delete_weight + self.drop_table_weight + self.alter_table_weight + + self.pragma_weight } } diff --git a/sql_generation/model/query/mod.rs b/sql_generation/model/query/mod.rs index 9876ffe54..f6aa57abe 100644 --- a/sql_generation/model/query/mod.rs +++ b/sql_generation/model/query/mod.rs @@ -13,6 +13,7 @@ pub mod delete; pub mod drop; pub mod drop_index; pub mod insert; +pub mod pragma; pub mod predicate; pub mod select; pub mod transaction; diff --git a/sql_generation/model/query/pragma.rs b/sql_generation/model/query/pragma.rs new file mode 100644 index 000000000..0bee9cbff --- /dev/null +++ b/sql_generation/model/query/pragma.rs @@ -0,0 +1,32 @@ +use std::fmt::Display; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Pragma { + AutoVacuumMode(VacuumMode), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum VacuumMode { + None, + Incremental, + Full, +} + +impl Display for Pragma { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Pragma::AutoVacuumMode(vacuum_mode) => { + let mode = match vacuum_mode { + VacuumMode::None => "none", + VacuumMode::Incremental => "incremental", + VacuumMode::Full => "full", + }; + + write!(f, "PRAGMA auto_vacuum={mode}")?; + Ok(()) + } + } + } +}