Merge 'Introduce instruction VTABLE' from Lâm Hoàng Phúc

this PR improves 3-6% for `prepare` benchmark without slowing down
others.  After this PR we don't have to store `InsnFunction`  in
`Program` and `ProgramBuilder` anymore, because `to_function` will
return result without matching.

Reviewed-by: Preston Thorpe <preston@turso.tech>

Closes #3098
This commit is contained in:
Jussi Saurio
2025-09-18 09:18:48 +03:00
committed by GitHub
5 changed files with 232 additions and 161 deletions

View File

@@ -411,7 +411,8 @@ impl CompiledExpression {
// Execute the program
let mut pc = 0usize;
while pc < program.insns.len() {
let (insn, insn_fn) = &program.insns[pc];
let (insn, _) = &program.insns[pc];
let insn_fn = insn.to_function();
state.pc = pc as u32;
// Execute the instruction

View File

@@ -34,7 +34,7 @@ impl TableRefIdCounter {
}
}
use super::{BranchOffset, CursorID, Insn, InsnFunction, InsnReference, JumpTarget, Program};
use super::{BranchOffset, CursorID, Insn, InsnReference, JumpTarget, Program};
/// A key that uniquely identifies a cursor.
/// The key is a pair of table reference id and index.
@@ -85,7 +85,7 @@ pub struct ProgramBuilder {
next_free_register: usize,
next_free_cursor_id: usize,
/// Instruction, the function to execute it with, and its original index in the vector.
insns: Vec<(Insn, InsnFunction, usize)>,
insns: Vec<(Insn, usize)>,
/// A span of instructions from (offset_start_inclusive, offset_end_exclusive),
/// that are deemed to be compile-time constant and can be hoisted out of loops
/// so that they get evaluated only once at the start of the program.
@@ -328,10 +328,9 @@ impl ProgramBuilder {
#[instrument(skip(self), level = Level::DEBUG)]
pub fn emit_insn(&mut self, insn: Insn) {
let function = insn.to_function();
// This seemingly empty trace here is needed so that a function span is emmited with it
tracing::trace!("");
self.insns.push((insn, function, self.insns.len()));
self.insns.push((insn, self.insns.len()));
}
pub fn close_cursors(&mut self, cursors: &[CursorID]) {
@@ -419,7 +418,7 @@ impl ProgramBuilder {
pub fn pop_current_parent_explain(&mut self) {
if let QueryMode::ExplainQueryPlan = self.query_mode {
if let Some(current) = self.current_parent_explain_idx {
let (Insn::Explain { p2, .. }, _, _) = &self.insns[current] else {
let (Insn::Explain { p2, .. }, _) = &self.insns[current] else {
unreachable!("current_parent_explain_idx must point to an Explain insn");
};
self.current_parent_explain_idx = *p2;
@@ -447,7 +446,7 @@ impl ProgramBuilder {
// 1. if insn not in any constant span, it stays where it is
// 2. if insn is in a constant span, it is after other insns, except those that are in a later constant span
// 3. within a single constant span the order is preserver
self.insns.sort_by(|(_, _, index_a), (_, _, index_b)| {
self.insns.sort_by(|(_, index_a), (_, index_b)| {
let a_span = self
.constant_spans
.iter()
@@ -471,7 +470,7 @@ impl ProgramBuilder {
let new_offset = self
.insns
.iter()
.position(|(_, _, index)| *old_offset == *index as u32)
.position(|(_, index)| *old_offset == *index as u32)
.unwrap() as u32;
*resolved_offset = Some((new_offset, *target));
}
@@ -482,7 +481,7 @@ impl ProgramBuilder {
let new_offset = self
.insns
.iter()
.position(|(_, _, index)| *old_offset == *index as u32)
.position(|(_, index)| *old_offset == *index as u32)
.expect("comment must exist") as u32;
*old_offset = new_offset;
}
@@ -492,25 +491,23 @@ impl ProgramBuilder {
if let Some(old_parent) = self.current_parent_explain_idx {
self.insns
.iter()
.position(|(_, _, index)| old_parent == *index)
.position(|(_, index)| old_parent == *index)
} else {
None
};
for i in 0..self.insns.len() {
let (Insn::Explain { p2, .. }, _, _) = &self.insns[i] else {
let (Insn::Explain { p2, .. }, _) = &self.insns[i] else {
continue;
};
let new_p2 = if p2.is_some() {
self.insns
.iter()
.position(|(_, _, index)| *p2 == Some(*index))
self.insns.iter().position(|(_, index)| *p2 == Some(*index))
} else {
None
};
let (Insn::Explain { p1, p2, .. }, _, _) = &mut self.insns[i] else {
let (Insn::Explain { p1, p2, .. }, _) = &mut self.insns[i] else {
unreachable!();
};
@@ -591,7 +588,7 @@ impl ProgramBuilder {
);
}
};
for (insn, _, _) in self.insns.iter_mut() {
for (insn, _) in self.insns.iter_mut() {
match insn {
Insn::Init { target_pc } => {
resolve(target_pc, "Init");
@@ -997,11 +994,7 @@ impl ProgramBuilder {
self.parameters.list.dedup();
Program {
max_registers: self.next_free_register,
insns: self
.insns
.into_iter()
.map(|(insn, function, _)| (insn, function))
.collect(),
insns: self.insns,
cursor_ref: self.cursor_ref,
comments: self.comments,
connection,

View File

@@ -10,6 +10,8 @@ use crate::{
translate::{collate::CollationSeq, emitter::TransactionMode},
Value,
};
use strum::EnumCount;
use strum_macros::{EnumDiscriminants, FromRepr, VariantArray};
use turso_macros::Description;
use turso_parser::ast::SortOrder;
@@ -150,7 +152,12 @@ impl<T: Copy + std::fmt::Display> std::fmt::Display for RegisterOrLiteral<T> {
}
}
#[derive(Description, Debug)]
// There are currently 190 opcodes in sqlite
#[repr(u8)]
#[derive(Description, Debug, EnumDiscriminants)]
#[strum_discriminants(vis(pub(crate)))]
#[strum_discriminants(derive(VariantArray, EnumCount, FromRepr))]
#[strum_discriminants(name(InsnVariants))]
pub enum Insn {
/// Initialize the program state and jump to the given PC.
Init {
@@ -1127,149 +1134,187 @@ pub enum Insn {
},
}
impl Insn {
pub fn to_function(&self) -> InsnFunction {
const fn get_insn_virtual_table() -> [InsnFunction; InsnVariants::COUNT] {
let mut result: [InsnFunction; InsnVariants::COUNT] = [execute::op_init; InsnVariants::COUNT];
let mut insn = 0;
while insn < InsnVariants::COUNT {
result[insn] = InsnVariants::from_repr(insn as u8).unwrap().to_function();
insn += 1;
}
result
}
const INSN_VTABLE: [InsnFunction; InsnVariants::COUNT] = get_insn_virtual_table();
impl InsnVariants {
// This function is used for testing
#[allow(dead_code)]
#[inline(always)]
pub(crate) const fn to_function_fast(self) -> InsnFunction {
INSN_VTABLE[self as usize]
}
// This function is used for generating `INSN_VTABLE`.
// We need to keep this function to make sure we implement all opcodes
pub(crate) const fn to_function(self) -> InsnFunction {
match self {
Insn::Init { .. } => execute::op_init,
Insn::Null { .. } => execute::op_null,
Insn::BeginSubrtn { .. } => execute::op_null,
Insn::NullRow { .. } => execute::op_null_row,
Insn::Add { .. } => execute::op_add,
Insn::Subtract { .. } => execute::op_subtract,
Insn::Multiply { .. } => execute::op_multiply,
Insn::Divide { .. } => execute::op_divide,
Insn::DropIndex { .. } => execute::op_drop_index,
Insn::Compare { .. } => execute::op_compare,
Insn::BitAnd { .. } => execute::op_bit_and,
Insn::BitOr { .. } => execute::op_bit_or,
Insn::BitNot { .. } => execute::op_bit_not,
Insn::Checkpoint { .. } => execute::op_checkpoint,
Insn::Remainder { .. } => execute::op_remainder,
Insn::Jump { .. } => execute::op_jump,
Insn::Move { .. } => execute::op_move,
Insn::IfPos { .. } => execute::op_if_pos,
Insn::NotNull { .. } => execute::op_not_null,
Insn::Eq { .. }
| Insn::Ne { .. }
| Insn::Lt { .. }
| Insn::Le { .. }
| Insn::Gt { .. }
| Insn::Ge { .. } => execute::op_comparison,
Insn::If { .. } => execute::op_if,
Insn::IfNot { .. } => execute::op_if_not,
Insn::OpenRead { .. } => execute::op_open_read,
Insn::VOpen { .. } => execute::op_vopen,
Insn::VCreate { .. } => execute::op_vcreate,
Insn::VFilter { .. } => execute::op_vfilter,
Insn::VColumn { .. } => execute::op_vcolumn,
Insn::VUpdate { .. } => execute::op_vupdate,
Insn::VNext { .. } => execute::op_vnext,
Insn::VDestroy { .. } => execute::op_vdestroy,
InsnVariants::Init => execute::op_init,
InsnVariants::Null => execute::op_null,
InsnVariants::BeginSubrtn => execute::op_null,
InsnVariants::NullRow => execute::op_null_row,
InsnVariants::Add => execute::op_add,
InsnVariants::Subtract => execute::op_subtract,
InsnVariants::Multiply => execute::op_multiply,
InsnVariants::Divide => execute::op_divide,
InsnVariants::DropIndex => execute::op_drop_index,
InsnVariants::Compare => execute::op_compare,
InsnVariants::BitAnd => execute::op_bit_and,
InsnVariants::BitOr => execute::op_bit_or,
InsnVariants::BitNot => execute::op_bit_not,
InsnVariants::Checkpoint => execute::op_checkpoint,
InsnVariants::Remainder => execute::op_remainder,
InsnVariants::Jump => execute::op_jump,
InsnVariants::Move => execute::op_move,
InsnVariants::IfPos => execute::op_if_pos,
InsnVariants::NotNull => execute::op_not_null,
InsnVariants::Eq
| InsnVariants::Ne
| InsnVariants::Lt
| InsnVariants::Le
| InsnVariants::Gt
| InsnVariants::Ge => execute::op_comparison,
InsnVariants::If => execute::op_if,
InsnVariants::IfNot => execute::op_if_not,
InsnVariants::OpenRead => execute::op_open_read,
InsnVariants::VOpen => execute::op_vopen,
InsnVariants::VCreate => execute::op_vcreate,
InsnVariants::VFilter => execute::op_vfilter,
InsnVariants::VColumn => execute::op_vcolumn,
InsnVariants::VUpdate => execute::op_vupdate,
InsnVariants::VNext => execute::op_vnext,
InsnVariants::VDestroy => execute::op_vdestroy,
InsnVariants::OpenPseudo => execute::op_open_pseudo,
InsnVariants::Rewind => execute::op_rewind,
InsnVariants::Last => execute::op_last,
InsnVariants::Column => execute::op_column,
InsnVariants::TypeCheck => execute::op_type_check,
InsnVariants::MakeRecord => execute::op_make_record,
InsnVariants::ResultRow => execute::op_result_row,
InsnVariants::Next => execute::op_next,
InsnVariants::Prev => execute::op_prev,
InsnVariants::Halt => execute::op_halt,
InsnVariants::HaltIfNull => execute::op_halt_if_null,
InsnVariants::Transaction => execute::op_transaction,
InsnVariants::AutoCommit => execute::op_auto_commit,
InsnVariants::Goto => execute::op_goto,
InsnVariants::Gosub => execute::op_gosub,
InsnVariants::Return => execute::op_return,
InsnVariants::Integer => execute::op_integer,
InsnVariants::Real => execute::op_real,
InsnVariants::RealAffinity => execute::op_real_affinity,
InsnVariants::String8 => execute::op_string8,
InsnVariants::Blob => execute::op_blob,
InsnVariants::RowData => execute::op_row_data,
InsnVariants::RowId => execute::op_row_id,
InsnVariants::IdxRowId => execute::op_idx_row_id,
InsnVariants::SeekRowid => execute::op_seek_rowid,
InsnVariants::DeferredSeek => execute::op_deferred_seek,
InsnVariants::SeekGE
| InsnVariants::SeekGT
| InsnVariants::SeekLE
| InsnVariants::SeekLT => execute::op_seek,
InsnVariants::SeekEnd => execute::op_seek_end,
InsnVariants::IdxGE => execute::op_idx_ge,
InsnVariants::IdxGT => execute::op_idx_gt,
InsnVariants::IdxLE => execute::op_idx_le,
InsnVariants::IdxLT => execute::op_idx_lt,
InsnVariants::DecrJumpZero => execute::op_decr_jump_zero,
InsnVariants::AggStep => execute::op_agg_step,
InsnVariants::AggFinal | InsnVariants::AggValue => execute::op_agg_final,
InsnVariants::SorterOpen => execute::op_sorter_open,
InsnVariants::SorterInsert => execute::op_sorter_insert,
InsnVariants::SorterSort => execute::op_sorter_sort,
InsnVariants::SorterData => execute::op_sorter_data,
InsnVariants::SorterNext => execute::op_sorter_next,
InsnVariants::Function => execute::op_function,
InsnVariants::Cast => execute::op_cast,
InsnVariants::InitCoroutine => execute::op_init_coroutine,
InsnVariants::EndCoroutine => execute::op_end_coroutine,
InsnVariants::Yield => execute::op_yield,
InsnVariants::Insert => execute::op_insert,
InsnVariants::Int64 => execute::op_int_64,
InsnVariants::IdxInsert => execute::op_idx_insert,
InsnVariants::Delete => execute::op_delete,
InsnVariants::NewRowid => execute::op_new_rowid,
InsnVariants::MustBeInt => execute::op_must_be_int,
InsnVariants::SoftNull => execute::op_soft_null,
InsnVariants::NoConflict => execute::op_no_conflict,
InsnVariants::NotExists => execute::op_not_exists,
InsnVariants::OffsetLimit => execute::op_offset_limit,
InsnVariants::OpenWrite => execute::op_open_write,
InsnVariants::Copy => execute::op_copy,
InsnVariants::CreateBtree => execute::op_create_btree,
InsnVariants::Destroy => execute::op_destroy,
InsnVariants::ResetSorter => execute::op_reset_sorter,
Insn::OpenPseudo { .. } => execute::op_open_pseudo,
Insn::Rewind { .. } => execute::op_rewind,
Insn::Last { .. } => execute::op_last,
Insn::Column { .. } => execute::op_column,
Insn::TypeCheck { .. } => execute::op_type_check,
Insn::MakeRecord { .. } => execute::op_make_record,
Insn::ResultRow { .. } => execute::op_result_row,
Insn::Next { .. } => execute::op_next,
Insn::Prev { .. } => execute::op_prev,
Insn::Halt { .. } => execute::op_halt,
Insn::HaltIfNull { .. } => execute::op_halt_if_null,
Insn::Transaction { .. } => execute::op_transaction,
Insn::AutoCommit { .. } => execute::op_auto_commit,
Insn::Goto { .. } => execute::op_goto,
Insn::Gosub { .. } => execute::op_gosub,
Insn::Return { .. } => execute::op_return,
Insn::Integer { .. } => execute::op_integer,
Insn::Real { .. } => execute::op_real,
Insn::RealAffinity { .. } => execute::op_real_affinity,
Insn::String8 { .. } => execute::op_string8,
Insn::Blob { .. } => execute::op_blob,
Insn::RowData { .. } => execute::op_row_data,
Insn::RowId { .. } => execute::op_row_id,
Insn::IdxRowId { .. } => execute::op_idx_row_id,
Insn::SeekRowid { .. } => execute::op_seek_rowid,
Insn::DeferredSeek { .. } => execute::op_deferred_seek,
Insn::SeekGE { .. }
| Insn::SeekGT { .. }
| Insn::SeekLE { .. }
| Insn::SeekLT { .. } => execute::op_seek,
Insn::SeekEnd { .. } => execute::op_seek_end,
Insn::IdxGE { .. } => execute::op_idx_ge,
Insn::IdxGT { .. } => execute::op_idx_gt,
Insn::IdxLE { .. } => execute::op_idx_le,
Insn::IdxLT { .. } => execute::op_idx_lt,
Insn::DecrJumpZero { .. } => execute::op_decr_jump_zero,
Insn::AggStep { .. } => execute::op_agg_step,
Insn::AggFinal { .. } | Insn::AggValue { .. } => execute::op_agg_final,
Insn::SorterOpen { .. } => execute::op_sorter_open,
Insn::SorterInsert { .. } => execute::op_sorter_insert,
Insn::SorterSort { .. } => execute::op_sorter_sort,
Insn::SorterData { .. } => execute::op_sorter_data,
Insn::SorterNext { .. } => execute::op_sorter_next,
Insn::Function { .. } => execute::op_function,
Insn::Cast { .. } => execute::op_cast,
Insn::InitCoroutine { .. } => execute::op_init_coroutine,
Insn::EndCoroutine { .. } => execute::op_end_coroutine,
Insn::Yield { .. } => execute::op_yield,
Insn::Insert { .. } => execute::op_insert,
Insn::Int64 { .. } => execute::op_int_64,
Insn::IdxInsert { .. } => execute::op_idx_insert,
Insn::Delete { .. } => execute::op_delete,
Insn::NewRowid { .. } => execute::op_new_rowid,
Insn::MustBeInt { .. } => execute::op_must_be_int,
Insn::SoftNull { .. } => execute::op_soft_null,
Insn::NoConflict { .. } => execute::op_no_conflict,
Insn::NotExists { .. } => execute::op_not_exists,
Insn::OffsetLimit { .. } => execute::op_offset_limit,
Insn::OpenWrite { .. } => execute::op_open_write,
Insn::Copy { .. } => execute::op_copy,
Insn::CreateBtree { .. } => execute::op_create_btree,
Insn::Destroy { .. } => execute::op_destroy,
Insn::ResetSorter { .. } => execute::op_reset_sorter,
Insn::DropTable { .. } => execute::op_drop_table,
Insn::DropView { .. } => execute::op_drop_view,
Insn::Close { .. } => execute::op_close,
Insn::IsNull { .. } => execute::op_is_null,
Insn::CollSeq { .. } => execute::op_coll_seq,
Insn::ParseSchema { .. } => execute::op_parse_schema,
Insn::PopulateMaterializedViews { .. } => execute::op_populate_materialized_views,
Insn::ShiftRight { .. } => execute::op_shift_right,
Insn::ShiftLeft { .. } => execute::op_shift_left,
Insn::AddImm { .. } => execute::op_add_imm,
Insn::Variable { .. } => execute::op_variable,
Insn::ZeroOrNull { .. } => execute::op_zero_or_null,
Insn::Not { .. } => execute::op_not,
Insn::Concat { .. } => execute::op_concat,
Insn::And { .. } => execute::op_and,
Insn::Or { .. } => execute::op_or,
Insn::Noop => execute::op_noop,
Insn::PageCount { .. } => execute::op_page_count,
Insn::ReadCookie { .. } => execute::op_read_cookie,
Insn::SetCookie { .. } => execute::op_set_cookie,
Insn::OpenEphemeral { .. } | Insn::OpenAutoindex { .. } => execute::op_open_ephemeral,
Insn::OpenDup { .. } => execute::op_open_dup,
Insn::Once { .. } => execute::op_once,
Insn::Found { .. } | Insn::NotFound { .. } => execute::op_found,
Insn::Affinity { .. } => execute::op_affinity,
Insn::IdxDelete { .. } => execute::op_idx_delete,
Insn::Count { .. } => execute::op_count,
Insn::IntegrityCk { .. } => execute::op_integrity_check,
Insn::RenameTable { .. } => execute::op_rename_table,
Insn::DropColumn { .. } => execute::op_drop_column,
Insn::AddColumn { .. } => execute::op_add_column,
Insn::AlterColumn { .. } => execute::op_alter_column,
Insn::MaxPgcnt { .. } => execute::op_max_pgcnt,
Insn::JournalMode { .. } => execute::op_journal_mode,
Insn::IfNeg { .. } => execute::op_if_neg,
Insn::Explain { .. } => execute::op_noop,
InsnVariants::DropTable => execute::op_drop_table,
InsnVariants::DropView => execute::op_drop_view,
InsnVariants::Close => execute::op_close,
InsnVariants::IsNull => execute::op_is_null,
InsnVariants::CollSeq => execute::op_coll_seq,
InsnVariants::ParseSchema => execute::op_parse_schema,
InsnVariants::PopulateMaterializedViews => execute::op_populate_materialized_views,
InsnVariants::ShiftRight => execute::op_shift_right,
InsnVariants::ShiftLeft => execute::op_shift_left,
InsnVariants::AddImm => execute::op_add_imm,
InsnVariants::Variable => execute::op_variable,
InsnVariants::ZeroOrNull => execute::op_zero_or_null,
InsnVariants::Not => execute::op_not,
InsnVariants::Concat => execute::op_concat,
InsnVariants::And => execute::op_and,
InsnVariants::Or => execute::op_or,
InsnVariants::Noop => execute::op_noop,
InsnVariants::PageCount => execute::op_page_count,
InsnVariants::ReadCookie => execute::op_read_cookie,
InsnVariants::SetCookie => execute::op_set_cookie,
InsnVariants::OpenEphemeral | InsnVariants::OpenAutoindex => execute::op_open_ephemeral,
InsnVariants::Once => execute::op_once,
InsnVariants::Found | InsnVariants::NotFound => execute::op_found,
InsnVariants::Affinity => execute::op_affinity,
InsnVariants::IdxDelete => execute::op_idx_delete,
InsnVariants::Count => execute::op_count,
InsnVariants::IntegrityCk => execute::op_integrity_check,
InsnVariants::RenameTable => execute::op_rename_table,
InsnVariants::DropColumn => execute::op_drop_column,
InsnVariants::AddColumn => execute::op_add_column,
InsnVariants::AlterColumn => execute::op_alter_column,
InsnVariants::MaxPgcnt => execute::op_max_pgcnt,
InsnVariants::JournalMode => execute::op_journal_mode,
InsnVariants::IfNeg => execute::op_if_neg,
InsnVariants::Explain => execute::op_noop,
InsnVariants::OpenDup => execute::op_open_dup,
}
}
}
impl Insn {
// SAFETY: If the enumeration specifies a primitive representation,
// then the discriminant may be reliably accessed via unsafe pointer casting
#[inline(always)]
fn discriminant(&self) -> u8 {
unsafe { *(self as *const Self as *const u8) }
}
#[inline(always)]
pub fn to_function(&self) -> InsnFunction {
// dont use this because its still using match
// InsnVariants::from(self).to_function_fast()
INSN_VTABLE[self.discriminant() as usize]
}
}
// TODO: Add remaining cookies.
#[derive(Description, Debug, Clone, Copy)]
pub enum Cookie {
@@ -1290,3 +1335,21 @@ pub enum Cookie {
/// The application ID as set by the application_id pragma.
ApplicationId = 8,
}
#[cfg(test)]
mod tests {
use strum::VariantArray;
#[test]
fn test_make_sure_correct_insn_table() {
for variant in super::InsnVariants::VARIANTS {
let func1 = variant.to_function();
let func2 = variant.to_function_fast();
assert_eq!(
func1 as usize, func2 as usize,
"Variant {:?} does not match in fast table at index {}",
variant, *variant as usize
);
}
}
}

View File

@@ -475,7 +475,9 @@ macro_rules! get_cursor {
pub struct Program {
pub max_registers: usize,
pub insns: Vec<(Insn, InsnFunction)>,
// we store original indices because we don't want to create new vec from
// ProgramBuilder
pub insns: Vec<(Insn, usize)>,
pub cursor_ref: Vec<(Option<CursorKey>, CursorType)>,
pub comments: Vec<(InsnReference, &'static str)>,
pub parameters: crate::parameters::Parameters,
@@ -644,7 +646,8 @@ impl Program {
}
// invalidate row
let _ = state.result_row.take();
let (insn, insn_function) = &self.insns[state.pc as usize];
let (insn, _) = &self.insns[state.pc as usize];
let insn_function = insn.to_function();
if enable_tracing {
trace_insn(self, state.pc as InsnReference, insn);
}