From 8e107ab18e5197e2d4d4ffcc7e0774d81e59c6e9 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 15 Oct 2025 14:53:36 +0400 Subject: [PATCH 01/25] slight reorder of operations --- core/storage/btree.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 0b7489530..909bb5de3 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -5773,30 +5773,26 @@ impl CursorTrait for BTreeCursor { first_overflow_page, .. }) => (payload, payload_size, first_overflow_page), + BTreeCell::IndexLeafCell(IndexLeafCell { + payload, + payload_size, + first_overflow_page, + }) => (payload, payload_size, first_overflow_page), BTreeCell::IndexInteriorCell(IndexInteriorCell { payload, payload_size, first_overflow_page, .. }) => (payload, payload_size, first_overflow_page), - BTreeCell::IndexLeafCell(IndexLeafCell { - payload, - first_overflow_page, - payload_size, - }) => (payload, payload_size, first_overflow_page), _ => unreachable!("unexpected page_type"), }; if let Some(next_page) = first_overflow_page { return_if_io!(self.process_overflow_read(payload, next_page, payload_size)) } else { - self.get_immutable_record_or_create() - .as_mut() - .unwrap() - .invalidate(); - self.get_immutable_record_or_create() - .as_mut() - .unwrap() - .start_serialization(payload); + let mut record = self.get_immutable_record_or_create(); + let record = record.as_mut().unwrap(); + record.invalidate(); + record.start_serialization(payload); self.record_cursor.borrow_mut().invalidate(); }; From ae8adc044958145dfc3f1224c55f1b1b3be45422 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 15 Oct 2025 14:53:53 +0400 Subject: [PATCH 02/25] faster extend_from_slice --- core/types.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/types.rs b/core/types.rs index bfcbb004e..f61d7f371 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1060,7 +1060,15 @@ impl ImmutableRecord { } pub fn start_serialization(&mut self, payload: &[u8]) { - self.as_blob_mut().extend_from_slice(payload); + let blob = self.as_blob_mut(); + blob.reserve(payload.len()); + + let len = blob.len(); + unsafe { + let dst = blob.as_mut_ptr().add(len); + std::ptr::copy_nonoverlapping(payload.as_ptr(), dst, payload.len()); + blob.set_len(len + payload.len()); + } } pub fn invalidate(&mut self) { From dba195bdfa331ab3e3533031f96c1632517d24eb Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 15 Oct 2025 15:43:37 +0400 Subject: [PATCH 03/25] avoid allocations --- core/types.rs | 34 +++++++++++++++++++++++----------- core/vdbe/execute.rs | 23 +++++++---------------- core/vdbe/mod.rs | 12 ++++++------ 3 files changed, 36 insertions(+), 33 deletions(-) diff --git a/core/types.rs b/core/types.rs index f61d7f371..bd0f8ff06 100644 --- a/core/types.rs +++ b/core/types.rs @@ -218,6 +218,12 @@ pub enum ValueRef<'a> { Blob(&'a [u8]), } +impl<'a, 'b> From<&'b ValueRef<'a>> for ValueRef<'a> { + fn from(value: &'b ValueRef<'a>) -> Self { + *value + } +} + impl Debug for ValueRef<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -1813,12 +1819,15 @@ fn compare_records_int( /// 4. **Length comparison**: If strings are equal, compares lengths /// 5. **Remaining fields**: If first field is equal and more fields exist, /// delegates to `compare_records_generic()` with `skip=1` -fn compare_records_string( +fn compare_records_string<'a, T>( serialized: &ImmutableRecord, - unpacked: &[ValueRef], + unpacked: &'a [T], index_info: &IndexInfo, tie_breaker: std::cmp::Ordering, -) -> Result { +) -> Result +where + ValueRef<'a>: From<&'a T>, +{ turso_assert!( index_info.key_info.len() >= unpacked.len(), "index_info.key_info.len() < unpacked.len()" @@ -1846,7 +1855,7 @@ fn compare_records_string( return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker); } - let ValueRef::Text(rhs_text, _) = &unpacked[0] else { + let ValueRef::Text(rhs_text, _) = (&unpacked[0]).into() else { return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker); }; @@ -1925,13 +1934,16 @@ fn compare_records_string( /// The serialized and unpacked records do not have to contain the same number /// of fields. If all fields that appear in both records are equal, then /// `tie_breaker` is returned. -pub fn compare_records_generic( +pub fn compare_records_generic<'a, T>( serialized: &ImmutableRecord, - unpacked: &[ValueRef], + unpacked: &'a [T], index_info: &IndexInfo, skip: usize, tie_breaker: std::cmp::Ordering, -) -> Result { +) -> Result +where + ValueRef<'a>: From<&'a T>, +{ turso_assert!( index_info.key_info.len() >= unpacked.len(), "index_info.key_info.len() < unpacked.len()" @@ -1971,7 +1983,7 @@ pub fn compare_records_generic( header_pos += bytes_read; let serial_type = SerialType::try_from(serial_type_raw)?; - let rhs_value = &unpacked[field_idx]; + let rhs_value = (&unpacked[field_idx]).into(); let lhs_value = match serial_type.kind() { SerialTypeKind::ConstInt0 => ValueRef::Integer(0), @@ -1993,14 +2005,14 @@ pub fn compare_records_generic( } (ValueRef::Integer(lhs_int), ValueRef::Float(rhs_float)) => { - sqlite_int_float_compare(*lhs_int, *rhs_float) + sqlite_int_float_compare(*lhs_int, rhs_float) } (ValueRef::Float(lhs_float), ValueRef::Integer(rhs_int)) => { - sqlite_int_float_compare(*rhs_int, *lhs_float).reverse() + sqlite_int_float_compare(rhs_int, *lhs_float).reverse() } - _ => lhs_value.partial_cmp(rhs_value).unwrap(), + _ => lhs_value.partial_cmp(&rhs_value).unwrap(), }; let final_comparison = match index_info.key_info[field_idx].sort_order { diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index e56bd4fdb..fa0878c64 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -19,7 +19,7 @@ use crate::types::{ }; use crate::util::normalize_ident; use crate::vdbe::insn::InsertFlags; -use crate::vdbe::{registers_to_ref_values, TxnCleanup}; +use crate::vdbe::TxnCleanup; use crate::vector::{vector32_sparse, vector_concat, vector_distance_jaccard, vector_slice}; use crate::{ error::{ @@ -3405,13 +3405,11 @@ pub fn op_idx_ge( let pc = if let Some(idx_record) = return_if_io!(cursor.record()) { // Create the comparison record from registers - let values = - registers_to_ref_values(&state.registers[*start_reg..*start_reg + *num_regs]); let tie_breaker = get_tie_breaker_from_idx_comp_op(insn); let ord = compare_records_generic( - &idx_record, // The serialized record from the index - &values, // The record built from registers - cursor.get_index_info(), // Sort order flags + &idx_record, // The serialized record from the index + &state.registers[*start_reg..*start_reg + *num_regs], // The record built from registers + cursor.get_index_info(), // Sort order flags 0, tie_breaker, )?; @@ -3473,12 +3471,10 @@ pub fn op_idx_le( let cursor = cursor.as_btree_mut(); let pc = if let Some(idx_record) = return_if_io!(cursor.record()) { - let values = - registers_to_ref_values(&state.registers[*start_reg..*start_reg + *num_regs]); let tie_breaker = get_tie_breaker_from_idx_comp_op(insn); let ord = compare_records_generic( &idx_record, - &values, + &state.registers[*start_reg..*start_reg + *num_regs], cursor.get_index_info(), 0, tie_breaker, @@ -3524,12 +3520,10 @@ pub fn op_idx_gt( let cursor = cursor.as_btree_mut(); let pc = if let Some(idx_record) = return_if_io!(cursor.record()) { - let values = - registers_to_ref_values(&state.registers[*start_reg..*start_reg + *num_regs]); let tie_breaker = get_tie_breaker_from_idx_comp_op(insn); let ord = compare_records_generic( &idx_record, - &values, + &state.registers[*start_reg..*start_reg + *num_regs], cursor.get_index_info(), 0, tie_breaker, @@ -3575,13 +3569,10 @@ pub fn op_idx_lt( let cursor = cursor.as_btree_mut(); let pc = if let Some(idx_record) = return_if_io!(cursor.record()) { - let values = - registers_to_ref_values(&state.registers[*start_reg..*start_reg + *num_regs]); - let tie_breaker = get_tie_breaker_from_idx_comp_op(insn); let ord = compare_records_generic( &idx_record, - &values, + &state.registers[*start_reg..*start_reg + *num_regs], cursor.get_index_info(), 0, tie_breaker, diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index bcb4372d5..a7b5cbd46 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -250,6 +250,12 @@ pub enum Register { Record(ImmutableRecord), } +impl<'a> From<&'a Register> for ValueRef<'a> { + fn from(value: &'a Register) -> Self { + value.get_value().as_ref() + } +} + impl Register { #[inline] pub fn is_null(&self) -> bool { @@ -1002,12 +1008,6 @@ fn make_record(registers: &[Register], start_reg: &usize, count: &usize) -> Immu ImmutableRecord::from_registers(regs, regs.len()) } -pub fn registers_to_ref_values<'a>(registers: &'a [Register]) -> Vec> { - registers - .iter() - .map(|reg| reg.get_value().as_ref()) - .collect() -} #[instrument(skip(program), level = Level::DEBUG)] fn trace_insn(program: &Program, addr: InsnReference, insn: &Insn) { From a6a5ffd8212495fee5369a716b26eba18780a62f Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 15 Oct 2025 15:44:25 +0400 Subject: [PATCH 04/25] move read_varint_fast closer to the read_varint impl --- core/storage/sqlite3_ondisk.rs | 38 ++++++++++++++++++++++++++++++++ core/vdbe/execute.rs | 40 +--------------------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index 08f7addc3..f7e440fda 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -1486,6 +1486,44 @@ pub fn read_integer(buf: &[u8], serial_type: u8) -> Result { } } +/// Fast varint reader optimized for the common cases of 1-byte and 2-byte varints. +/// +/// This function is a performance-optimized version of `read_varint()` that handles +/// the most common varint cases inline before falling back to the full implementation. +/// It follows the same varint encoding as SQLite. +/// +/// # Optimized Cases +/// +/// - **Single-byte case**: Values 0-127 (0x00-0x7F) are returned immediately +/// - **Two-byte case**: Values 128-16383 (0x80-0x3FFF) are handled inline +/// - **Multi-byte case**: Larger values fall back to the full `read_varint()` implementation +/// +/// This function is similar to `sqlite3GetVarint32` +#[inline(always)] +pub fn read_varint_fast(buf: &[u8]) -> Result<(u64, usize)> { + // Fast path: Single-byte varint + if let Some(&first_byte) = buf.first() { + if first_byte & 0x80 == 0 { + return Ok((first_byte as u64, 1)); + } + } else { + crate::bail_corrupt_error!("Invalid varint"); + } + + // Fast path: Two-byte varint + if let Some(&second_byte) = buf.get(1) { + if second_byte & 0x80 == 0 { + let v = (((buf[0] & 0x7f) as u64) << 7) + (second_byte as u64); + return Ok((v, 2)); + } + } else { + crate::bail_corrupt_error!("Invalid varint"); + } + + //Fallback: Multi-byte varint + read_varint(buf) +} + #[inline(always)] pub fn read_varint(buf: &[u8]) -> Result<(u64, usize)> { let mut v: u64 = 0; diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index fa0878c64..4677c48e7 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -11,7 +11,7 @@ use crate::storage::btree::{ use crate::storage::database::DatabaseFile; use crate::storage::page_cache::PageCache; use crate::storage::pager::{AtomicDbState, CreateBTreeFlags, DbState}; -use crate::storage::sqlite3_ondisk::{read_varint, DatabaseHeader, PageSize}; +use crate::storage::sqlite3_ondisk::{read_varint_fast, DatabaseHeader, PageSize}; use crate::translate::collate::CollationSeq; use crate::types::{ compare_immutable, compare_records_generic, Extendable, IOCompletions, ImmutableRecord, @@ -1469,44 +1469,6 @@ pub fn op_last( Ok(InsnFunctionStepResult::Step) } -/// Fast varint reader optimized for the common cases of 1-byte and 2-byte varints. -/// -/// This function is a performance-optimized version of `read_varint()` that handles -/// the most common varint cases inline before falling back to the full implementation. -/// It follows the same varint encoding as SQLite. -/// -/// # Optimized Cases -/// -/// - **Single-byte case**: Values 0-127 (0x00-0x7F) are returned immediately -/// - **Two-byte case**: Values 128-16383 (0x80-0x3FFF) are handled inline -/// - **Multi-byte case**: Larger values fall back to the full `read_varint()` implementation -/// -/// This function is similar to `sqlite3GetVarint32` -#[inline(always)] -fn read_varint_fast(buf: &[u8]) -> Result<(u64, usize)> { - // Fast path: Single-byte varint - if let Some(&first_byte) = buf.first() { - if first_byte & 0x80 == 0 { - return Ok((first_byte as u64, 1)); - } - } else { - crate::bail_corrupt_error!("Invalid varint"); - } - - // Fast path: Two-byte varint - if let Some(&second_byte) = buf.get(1) { - if second_byte & 0x80 == 0 { - let v = (((buf[0] & 0x7f) as u64) << 7) + (second_byte as u64); - return Ok((v, 2)); - } - } else { - crate::bail_corrupt_error!("Invalid varint"); - } - - //Fallback: Multi-byte varint - read_varint(buf) -} - #[derive(Debug, Clone, Copy)] pub enum OpColumnState { Start, From f19c73822ee9afbb9c859fe6b7f3f9e0ee3bc576 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 15 Oct 2025 17:26:23 +0400 Subject: [PATCH 05/25] simplify serial_type size calculation --- core/vdbe/execute.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index 4677c48e7..46cc68d79 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -1642,17 +1642,14 @@ pub fn op_column( 8 => 0, // CONST_INT1 9 => 0, - // BLOB - n if n >= 12 && n & 1 == 0 => (n - 12) >> 1, - // TEXT - n if n >= 13 && n & 1 == 1 => (n - 13) >> 1, // Reserved 10 | 11 => { return Err(LimboError::Corrupt(format!( "Reserved serial type: {serial_type}" ))) } - _ => unreachable!("Invalid serial type: {serial_type}"), + // BLOB or TEXT + n => (n - 12) / 2, } as usize; data_offset += data_size; record_cursor.offsets.push(data_offset); From c0fdaeb4755fa49b8fab928a566cd536b476c55f Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 15 Oct 2025 17:26:35 +0400 Subject: [PATCH 06/25] move more possible option higher --- core/vdbe/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index a7b5cbd46..2d60d854f 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -133,8 +133,8 @@ impl BranchOffset { /// Returns the offset value. Panics if the branch offset is a label or placeholder. pub fn as_offset_int(&self) -> InsnReference { match self { - BranchOffset::Label(v) => unreachable!("Unresolved label: {}", v), BranchOffset::Offset(v) => *v, + BranchOffset::Label(v) => unreachable!("Unresolved label: {}", v), BranchOffset::Placeholder => unreachable!("Unresolved placeholder"), } } From 68650cf594799d4c5a97de3670bfe157b5229698 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 15 Oct 2025 17:25:57 +0400 Subject: [PATCH 07/25] alternative read_variant implementation - it faster in benchmark (who knows why) - also seems bit faster for some my query - let's test on CI --- core/storage/sqlite3_ondisk.rs | 85 +++++++++++----------------------- core/vdbe/execute.rs | 7 ++- 2 files changed, 30 insertions(+), 62 deletions(-) diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index f7e440fda..8e82ffa13 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -1486,71 +1486,40 @@ pub fn read_integer(buf: &[u8], serial_type: u8) -> Result { } } -/// Fast varint reader optimized for the common cases of 1-byte and 2-byte varints. -/// -/// This function is a performance-optimized version of `read_varint()` that handles -/// the most common varint cases inline before falling back to the full implementation. -/// It follows the same varint encoding as SQLite. -/// -/// # Optimized Cases -/// -/// - **Single-byte case**: Values 0-127 (0x00-0x7F) are returned immediately -/// - **Two-byte case**: Values 128-16383 (0x80-0x3FFF) are handled inline -/// - **Multi-byte case**: Larger values fall back to the full `read_varint()` implementation -/// -/// This function is similar to `sqlite3GetVarint32` -#[inline(always)] -pub fn read_varint_fast(buf: &[u8]) -> Result<(u64, usize)> { - // Fast path: Single-byte varint - if let Some(&first_byte) = buf.first() { - if first_byte & 0x80 == 0 { - return Ok((first_byte as u64, 1)); - } - } else { - crate::bail_corrupt_error!("Invalid varint"); - } - - // Fast path: Two-byte varint - if let Some(&second_byte) = buf.get(1) { - if second_byte & 0x80 == 0 { - let v = (((buf[0] & 0x7f) as u64) << 7) + (second_byte as u64); - return Ok((v, 2)); - } - } else { - crate::bail_corrupt_error!("Invalid varint"); - } - - //Fallback: Multi-byte varint - read_varint(buf) -} - #[inline(always)] pub fn read_varint(buf: &[u8]) -> Result<(u64, usize)> { let mut v: u64 = 0; - for i in 0..8 { - match buf.get(i) { - Some(c) => { - v = (v << 7) + (c & 0x7f) as u64; - if (c & 0x80) == 0 { - return Ok((v, i + 1)); - } - } - None => { - crate::bail_corrupt_error!("Invalid varint"); - } + let mut i = 0; + let chunks = buf.chunks_exact(2); + for chunk in chunks { + let c1 = chunk[0]; + v = (v << 7) + (c1 & 0x7f) as u64; + i += 1; + if (c1 & 0x80) == 0 { + return Ok((v, i)); + } + let c2 = chunk[1]; + v = (v << 7) + (c2 & 0x7f) as u64; + i += 1; + if (c2 & 0x80) == 0 { + return Ok((v, i)); + } + if i == 8 { + break; } } - match buf.get(8) { + match buf.get(i) { Some(&c) => { - // Values requiring 9 bytes must have non-zero in the top 8 bits (value >= 1<<56). - // Since the final value is `(v<<8) + c`, the top 8 bits (v >> 48) must not be 0. - // If those are zero, this should be treated as corrupt. - // Perf? the comparison + branching happens only in parsing 9-byte varint which is rare. - if (v >> 48) == 0 { - bail_corrupt_error!("Invalid varint"); + if i < 8 && (c & 0x80) == 0 { + return Ok(((v << 7) + c as u64, i + 1)); + } else if i == 8 && (v >> 48) > 0 { + // Values requiring 9 bytes must have non-zero in the top 8 bits (value >= 1<<56). + // Since the final value is `(v<<8) + c`, the top 8 bits (v >> 48) must not be 0. + // If those are zero, this should be treated as corrupt. + // Perf? the comparison + branching happens only in parsing 9-byte varint which is rare. + return Ok(((v << 8) + c as u64, i + 1)); } - v = (v << 8) + c as u64; - Ok((v, 9)) + bail_corrupt_error!("Invalid varint"); } None => { bail_corrupt_error!("Invalid varint"); diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index 46cc68d79..55c37e0a9 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -11,7 +11,7 @@ use crate::storage::btree::{ use crate::storage::database::DatabaseFile; use crate::storage::page_cache::PageCache; use crate::storage::pager::{AtomicDbState, CreateBTreeFlags, DbState}; -use crate::storage::sqlite3_ondisk::{read_varint_fast, DatabaseHeader, PageSize}; +use crate::storage::sqlite3_ondisk::{read_varint, DatabaseHeader, PageSize}; use crate::translate::collate::CollationSeq; use crate::types::{ compare_immutable, compare_records_generic, Extendable, IOCompletions, ImmutableRecord, @@ -1595,7 +1595,7 @@ pub fn op_column( let mut record_cursor = cursor.record_cursor_mut(); if record_cursor.offsets.is_empty() { - let (header_size, header_len_bytes) = read_varint_fast(payload)?; + let (header_size, header_len_bytes) = read_varint(payload)?; let header_size = header_size as usize; debug_assert!(header_size <= payload.len() && header_size <= 98307, "header_size: {header_size}, header_len_bytes: {header_len_bytes}, payload.len(): {}", payload.len()); @@ -1617,8 +1617,7 @@ pub fn op_column( while record_cursor.serial_types.len() <= target_column && parse_pos < record_cursor.header_size { - let (serial_type, varint_len) = - read_varint_fast(&payload[parse_pos..])?; + let (serial_type, varint_len) = read_varint(&payload[parse_pos..])?; record_cursor.serial_types.push(serial_type); parse_pos += varint_len; let data_size = match serial_type { From 7c919314a99778f11f842454ee70bb3cd29e82ef Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Tue, 14 Oct 2025 12:27:57 +0400 Subject: [PATCH 08/25] use heap-sort style algorithm for order by ... limit k queries --- core/translate/emitter.rs | 1 + core/translate/order_by.rs | 246 +++++++++++++++++++++++++++---------- 2 files changed, 184 insertions(+), 63 deletions(-) diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index 2dabd4b82..5ae59fc57 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -304,6 +304,7 @@ pub fn emit_query<'a>( &plan.order_by, &plan.table_references, plan.group_by.is_some(), + plan.distinctness != Distinctness::NonDistinct, &plan.aggregates, )?; } diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index aafb32a21..09de9c3d3 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -1,8 +1,10 @@ +use std::sync::Arc; + use turso_parser::ast::{self, SortOrder}; use crate::{ emit_explain, - schema::PseudoCursorType, + schema::{Index, IndexColumn, PseudoCursorType}, translate::{ collate::{get_collseq_from_expr, CollationSeq}, group_by::is_orderby_agg_or_const, @@ -11,7 +13,7 @@ use crate::{ util::exprs_are_equivalent, vdbe::{ builder::{CursorType, ProgramBuilder}, - insn::Insn, + insn::{IdxInsertFlags, Insn}, }, QueryMode, Result, }; @@ -39,6 +41,8 @@ pub struct SortMetadata { /// aggregates/constants, so that rows that tie on ORDER BY terms are output in /// the same relative order the underlying row stream produced them. pub has_sequence: bool, + /// Whether to use heap-sort with BTreeIndex instead of full-collection sort through Sorter + pub use_heap_sort: bool, } /// Initialize resources needed for ORDER BY processing @@ -49,54 +53,106 @@ pub fn init_order_by( order_by: &[(Box, SortOrder)], referenced_tables: &TableReferences, has_group_by: bool, + has_distinct: bool, aggregates: &[Aggregate], ) -> Result<()> { - let sort_cursor = program.alloc_cursor_id(CursorType::Sorter); let only_aggs = order_by .iter() .all(|(e, _)| is_orderby_agg_or_const(&t_ctx.resolver, e, aggregates)); // only emit sequence column if we have GROUP BY and ORDER BY is not only aggregates or constants let has_sequence = has_group_by && !only_aggs; + + let use_heap_sort = !has_distinct && !has_group_by && t_ctx.limit_ctx.is_some(); + if use_heap_sort { + assert!(!has_sequence); + } + let remappings = order_by_deduplicate_result_columns(order_by, result_columns, has_sequence); + let sort_cursor = if use_heap_sort { + let index_name = format!("heap_sort_{}", program.offset().as_offset_int()); // we don't really care about the name that much, just enough that we don't get name collisions + let mut index_columns = Vec::with_capacity(order_by.len() + result_columns.len()); + for (column, order) in order_by { + let collation = get_collseq_from_expr(column, referenced_tables)?; + let pos_in_table = index_columns.len(); + index_columns.push(IndexColumn { + name: pos_in_table.to_string(), + order: *order, + pos_in_table, + collation, + default: None, + }) + } + for _ in remappings.iter().filter(|r| !r.deduplicated) { + let pos_in_table = index_columns.len(); + index_columns.push(IndexColumn { + name: pos_in_table.to_string(), + order: SortOrder::Asc, + pos_in_table, + collation: None, + default: None, + }) + } + let index = Arc::new(Index { + name: index_name.clone(), + table_name: String::new(), + ephemeral: true, + root_page: 0, + columns: index_columns, + unique: false, + has_rowid: false, + where_clause: None, + }); + program.alloc_cursor_id(CursorType::BTreeIndex(index)) + } else { + program.alloc_cursor_id(CursorType::Sorter) + }; t_ctx.meta_sort = Some(SortMetadata { sort_cursor, reg_sorter_data: program.alloc_register(), - remappings: order_by_deduplicate_result_columns(order_by, result_columns, has_sequence), + remappings, has_sequence, + use_heap_sort, }); - /* - * Terms of the ORDER BY clause that is part of a SELECT statement may be assigned a collating sequence using the COLLATE operator, - * in which case the specified collating function is used for sorting. - * Otherwise, if the expression sorted by an ORDER BY clause is a column, - * then the collating sequence of the column is used to determine sort order. - * If the expression is not a column and has no COLLATE clause, then the BINARY collating sequence is used. - */ - let mut collations = order_by - .iter() - .map(|(expr, _)| get_collseq_from_expr(expr, referenced_tables)) - .collect::>>()?; + if use_heap_sort { + program.emit_insn(Insn::OpenEphemeral { + cursor_id: sort_cursor, + is_table: false, + }); + } else { + /* + * Terms of the ORDER BY clause that is part of a SELECT statement may be assigned a collating sequence using the COLLATE operator, + * in which case the specified collating function is used for sorting. + * Otherwise, if the expression sorted by an ORDER BY clause is a column, + * then the collating sequence of the column is used to determine sort order. + * If the expression is not a column and has no COLLATE clause, then the BINARY collating sequence is used. + */ + let mut collations = order_by + .iter() + .map(|(expr, _)| get_collseq_from_expr(expr, referenced_tables)) + .collect::>>()?; - if has_sequence { - // sequence column uses BINARY collation - collations.push(Some(CollationSeq::default())); + if has_sequence { + // sequence column uses BINARY collation + collations.push(Some(CollationSeq::default())); + } + + let key_len = order_by.len() + if has_sequence { 1 } else { 0 }; + + program.emit_insn(Insn::SorterOpen { + cursor_id: sort_cursor, + columns: key_len, + order: { + let mut ord: Vec = order_by.iter().map(|(_, d)| *d).collect(); + if has_sequence { + // sequence is ascending tiebreaker + ord.push(SortOrder::Asc); + } + ord + }, + collations, + }); } - - let key_len = order_by.len() + if has_sequence { 1 } else { 0 }; - - program.emit_insn(Insn::SorterOpen { - cursor_id: sort_cursor, - columns: key_len, - order: { - let mut ord: Vec = order_by.iter().map(|(_, d)| *d).collect(); - if has_sequence { - // sequence is ascending tiebreaker - ord.push(SortOrder::Asc); - } - ord - }, - collations, - }); Ok(()) } @@ -118,6 +174,7 @@ pub fn emit_order_by( reg_sorter_data, ref remappings, has_sequence, + use_heap_sort, } = *t_ctx.meta_sort.as_ref().unwrap(); let sorter_column_count = order_by.len() @@ -128,33 +185,44 @@ pub fn emit_order_by( // to emit correct explain output. emit_explain!(program, false, "USE TEMP B-TREE FOR ORDER BY".to_owned()); - let pseudo_cursor = program.alloc_cursor_id(CursorType::Pseudo(PseudoCursorType { - column_count: sorter_column_count, - })); + let cursor_id = if !use_heap_sort { + let pseudo_cursor = program.alloc_cursor_id(CursorType::Pseudo(PseudoCursorType { + column_count: sorter_column_count, + })); - program.emit_insn(Insn::OpenPseudo { - cursor_id: pseudo_cursor, - content_reg: reg_sorter_data, - num_fields: sorter_column_count, - }); + program.emit_insn(Insn::OpenPseudo { + cursor_id: pseudo_cursor, + content_reg: reg_sorter_data, + num_fields: sorter_column_count, + }); + + program.emit_insn(Insn::SorterSort { + cursor_id: sort_cursor, + pc_if_empty: sort_loop_end_label, + }); + pseudo_cursor + } else { + program.emit_insn(Insn::Rewind { + cursor_id: sort_cursor, + pc_if_empty: sort_loop_end_label, + }); + sort_cursor + }; - program.emit_insn(Insn::SorterSort { - cursor_id: sort_cursor, - pc_if_empty: sort_loop_end_label, - }); program.preassign_label_to_next_insn(sort_loop_start_label); emit_offset(program, sort_loop_next_label, t_ctx.reg_offset); - program.emit_insn(Insn::SorterData { - cursor_id: sort_cursor, - dest_reg: reg_sorter_data, - pseudo_cursor, - }); + if !use_heap_sort { + program.emit_insn(Insn::SorterData { + cursor_id: sort_cursor, + dest_reg: reg_sorter_data, + pseudo_cursor: cursor_id, + }); + } // We emit the columns in SELECT order, not sorter order (sorter always has the sort keys first). // This is tracked in sort_metadata.remappings. - let cursor_id = pseudo_cursor; let start_reg = t_ctx.reg_result_cols_start.unwrap(); for i in 0..result_columns.len() { let reg = start_reg + i; @@ -175,10 +243,17 @@ pub fn emit_order_by( )?; program.resolve_label(sort_loop_next_label, program.offset()); - program.emit_insn(Insn::SorterNext { - cursor_id: sort_cursor, - pc_if_next: sort_loop_start_label, - }); + if !use_heap_sort { + program.emit_insn(Insn::SorterNext { + cursor_id: sort_cursor, + pc_if_next: sort_loop_start_label, + }); + } else { + program.emit_insn(Insn::Next { + cursor_id: sort_cursor, + pc_if_next: sort_loop_start_label, + }); + } program.preassign_label_to_next_insn(sort_loop_end_label); Ok(()) @@ -333,16 +408,61 @@ pub fn order_by_sorter_insert( let SortMetadata { sort_cursor, reg_sorter_data, + use_heap_sort, .. } = sort_metadata; - sorter_insert( - program, - start_reg, - orderby_sorter_column_count, - *sort_cursor, - *reg_sorter_data, - ); + if *use_heap_sort { + // maintain top-k records in the index instead of materializing the whole sequence + let insert_label = program.allocate_label(); + let skip_label = program.allocate_label(); + let limit = t_ctx.limit_ctx.as_ref().expect("limit must be set"); + program.emit_insn(Insn::IfPos { + reg: limit.reg_limit, + target_pc: insert_label, + decrement_by: 1, + }); + program.emit_insn(Insn::Last { + cursor_id: *sort_cursor, + pc_if_empty: insert_label, + }); + program.emit_insn(Insn::IdxLE { + cursor_id: *sort_cursor, + start_reg: start_reg, + num_regs: orderby_sorter_column_count, + target_pc: skip_label, + }); + + program.emit_insn(Insn::Delete { + cursor_id: *sort_cursor, + table_name: "".to_string(), + }); + + program.preassign_label_to_next_insn(insert_label); + program.emit_insn(Insn::MakeRecord { + start_reg, + count: orderby_sorter_column_count, + dest_reg: *reg_sorter_data, + index_name: None, + affinity_str: None, + }); + program.emit_insn(Insn::IdxInsert { + cursor_id: *sort_cursor, + record_reg: *reg_sorter_data, + unpacked_start: None, + unpacked_count: None, + flags: IdxInsertFlags::new(), + }); + program.preassign_label_to_next_insn(skip_label); + } else { + sorter_insert( + program, + start_reg, + orderby_sorter_column_count, + *sort_cursor, + *reg_sorter_data, + ); + } Ok(()) } From 1a24139359e6b65dcf66696ae49fb1294babe160 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Tue, 14 Oct 2025 13:23:22 +0400 Subject: [PATCH 09/25] fix limit for order by queries with heap-sort style execution --- core/translate/order_by.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index 09de9c3d3..7a91bc160 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -239,7 +239,11 @@ pub fn emit_order_by( plan, start_reg, t_ctx.limit_ctx, - Some(sort_loop_end_label), + if !use_heap_sort { + Some(sort_loop_end_label) + } else { + None + }, )?; program.resolve_label(sort_loop_next_label, program.offset()); From 5868270b0644818769468083aa9443391d535fcd Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Tue, 14 Oct 2025 13:29:49 +0400 Subject: [PATCH 10/25] fix clippy --- core/translate/order_by.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index 7a91bc160..dfd218ef7 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -46,6 +46,7 @@ pub struct SortMetadata { } /// Initialize resources needed for ORDER BY processing +#[allow(clippy::too_many_arguments)] pub fn init_order_by( program: &mut ProgramBuilder, t_ctx: &mut TranslateCtx, @@ -432,7 +433,7 @@ pub fn order_by_sorter_insert( }); program.emit_insn(Insn::IdxLE { cursor_id: *sort_cursor, - start_reg: start_reg, + start_reg, num_regs: orderby_sorter_column_count, target_pc: skip_label, }); From b065e7d3801beda895294b7fefb7adc3f82a316f Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Tue, 14 Oct 2025 14:40:01 +0400 Subject: [PATCH 11/25] emit Sequence column for heap-sort in order to distinguish between rows with same order by key and result columns --- core/translate/order_by.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index dfd218ef7..e4c72191d 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -61,13 +61,11 @@ pub fn init_order_by( .iter() .all(|(e, _)| is_orderby_agg_or_const(&t_ctx.resolver, e, aggregates)); - // only emit sequence column if we have GROUP BY and ORDER BY is not only aggregates or constants - let has_sequence = has_group_by && !only_aggs; - let use_heap_sort = !has_distinct && !has_group_by && t_ctx.limit_ctx.is_some(); - if use_heap_sort { - assert!(!has_sequence); - } + + // only emit sequence column if (we have GROUP BY and ORDER BY is not only aggregates or constants) OR (we decided to use heap-sort) + let has_sequence = (has_group_by && !only_aggs) || use_heap_sort; + let remappings = order_by_deduplicate_result_columns(order_by, result_columns, has_sequence); let sort_cursor = if use_heap_sort { let index_name = format!("heap_sort_{}", program.offset().as_offset_int()); // we don't really care about the name that much, just enough that we don't get name collisions @@ -83,6 +81,15 @@ pub fn init_order_by( default: None, }) } + let pos_in_table = index_columns.len(); + // add sequence number between ORDER BY columns and result column + index_columns.push(IndexColumn { + name: pos_in_table.to_string(), + order: SortOrder::Asc, + pos_in_table, + collation: Some(CollationSeq::Binary), + default: None, + }); for _ in remappings.iter().filter(|r| !r.deduplicated) { let pos_in_table = index_columns.len(); index_columns.push(IndexColumn { From a1260ca8c7030c3634babf0582efe3f92868cb91 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Tue, 14 Oct 2025 14:49:09 +0400 Subject: [PATCH 12/25] implement Sequence opcodes for any type of cursors --- core/vdbe/execute.rs | 11 +++++++---- core/vdbe/mod.rs | 4 ++++ core/vdbe/sorter.rs | 17 ----------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index 55c37e0a9..dcfa26046 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -5553,8 +5553,9 @@ pub fn op_sequence( }, insn ); - let cursor = state.get_cursor(*cursor_id).as_sorter_mut(); - let seq_num = cursor.next_sequence(); + let cursor_seq = state.cursor_seqs.get_mut(*cursor_id).unwrap(); + let seq_num = *cursor_seq; + *cursor_seq += 1; state.registers[*target_reg] = Register::Value(Value::Integer(seq_num)); state.pc += 1; Ok(InsnFunctionStepResult::Step) @@ -5575,8 +5576,10 @@ pub fn op_sequence_test( }, insn ); - let cursor = state.get_cursor(*cursor_id).as_sorter_mut(); - state.pc = if cursor.seq_beginning() { + let cursor_seq = state.cursor_seqs.get_mut(*cursor_id).unwrap(); + let was_zero = *cursor_seq == 0; + *cursor_seq += 1; + state.pc = if was_zero { target_pc.as_offset_int() } else { state.pc + 1 diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 2d60d854f..94ae80c41 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -282,6 +282,7 @@ pub struct ProgramState { pub io_completions: Option, pub pc: InsnReference, cursors: Vec>, + cursor_seqs: Vec, registers: Vec, pub(crate) result_row: Option, last_compare: Option, @@ -325,11 +326,13 @@ pub struct ProgramState { impl ProgramState { pub fn new(max_registers: usize, max_cursors: usize) -> Self { let cursors: Vec> = (0..max_cursors).map(|_| None).collect(); + let cursor_seqs = vec![0i64; max_cursors]; let registers = vec![Register::Value(Value::Null); max_registers]; Self { io_completions: None, pc: 0, cursors, + cursor_seqs, registers, result_row: None, last_compare: None, @@ -411,6 +414,7 @@ impl ProgramState { if let Some(max_cursors) = max_cursors { self.cursors.resize_with(max_cursors, || None); + self.cursor_seqs.resize(max_cursors, 0); } if let Some(max_resgisters) = max_registers { self.registers diff --git a/core/vdbe/sorter.rs b/core/vdbe/sorter.rs index ac7e07ed4..ef0864bec 100644 --- a/core/vdbe/sorter.rs +++ b/core/vdbe/sorter.rs @@ -86,7 +86,6 @@ pub struct Sorter { insert_state: InsertState, /// State machine for [Sorter::init_chunk_heap] init_chunk_heap_state: InitChunkHeapState, - seq_count: i64, pending_completions: Vec, } @@ -125,7 +124,6 @@ impl Sorter { sort_state: SortState::Start, insert_state: InsertState::Start, init_chunk_heap_state: InitChunkHeapState::Start, - seq_count: 0, pending_completions: Vec::new(), } } @@ -138,21 +136,6 @@ impl Sorter { self.current.is_some() } - /// Get current sequence count and increment it - pub fn next_sequence(&mut self) -> i64 { - let current = self.seq_count; - self.seq_count += 1; - current - } - - /// Test if at beginning of sequence (count == 0) and increment - /// Returns true if this was the first call (seq_count was 0) - pub fn seq_beginning(&mut self) -> bool { - let was_zero = self.seq_count == 0; - self.seq_count += 1; - was_zero - } - // We do the sorting here since this is what is called by the SorterSort instruction pub fn sort(&mut self) -> Result> { loop { From a2dbaafe69385d1849af646d6941cf986d812380 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Tue, 14 Oct 2025 14:49:27 +0400 Subject: [PATCH 13/25] add explicit test for multiple rows heap-sort bug --- testing/orderby.test | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/testing/orderby.test b/testing/orderby.test index e173d946b..ca929978a 100755 --- a/testing/orderby.test +++ b/testing/orderby.test @@ -239,4 +239,16 @@ do_execsql_test_on_specific_db {:memory:} orderby_alias_precedence { INSERT INTO t VALUES (1,200),(2,100); SELECT x AS y, y AS x FROM t ORDER BY x; } {2|100 -1|200} \ No newline at end of file +1|200} + +# Check that ORDER BY with heap-sort properly handle multiple rows with same order key + result values +do_execsql_test_on_specific_db {:memory:} orderby_same_rows { + CREATE TABLE t(x,y,z); + INSERT INTO t VALUES (1,2,3),(1,2,6),(1,2,9),(1,2,10),(1,3,-1),(1,3,-2); + SELECT x, y FROM t ORDER BY x, y LIMIT 10; +} {1|2 +1|2 +1|2 +1|2 +1|3 +1|3} \ No newline at end of file From af4c1e8bd4a1203b66e664e47797bcde3ba64ff1 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Tue, 14 Oct 2025 15:14:01 +0400 Subject: [PATCH 14/25] use proper register for limit --- core/translate/order_by.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index e4c72191d..bd419e016 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -429,8 +429,9 @@ pub fn order_by_sorter_insert( let insert_label = program.allocate_label(); let skip_label = program.allocate_label(); let limit = t_ctx.limit_ctx.as_ref().expect("limit must be set"); + let limit_reg = t_ctx.reg_limit_offset_sum.unwrap_or(limit.reg_limit); program.emit_insn(Insn::IfPos { - reg: limit.reg_limit, + reg: limit_reg, target_pc: insert_label, decrement_by: 1, }); From dd34f7fd504fbbd57e7802da4e76447c4911ab09 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Tue, 14 Oct 2025 22:19:01 +0400 Subject: [PATCH 15/25] wip --- core/lib.rs | 1 + core/storage/btree.rs | 12 +++++++----- core/vdbe/execute.rs | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index 2a0a558cf..9b93d64ad 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -15,6 +15,7 @@ mod json; pub mod mvcc; mod parameters; mod pragma; +pub mod primitives; mod pseudo; mod schema; #[cfg(feature = "series")] diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 909bb5de3..5a37ba568 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -41,7 +41,7 @@ use super::{ use parking_lot::RwLock; use std::{ any::Any, - cell::{Cell, Ref, RefCell}, + cell::Cell, cmp::{Ordering, Reverse}, collections::{BinaryHeap, HashMap}, fmt::Debug, @@ -50,6 +50,8 @@ use std::{ sync::Arc, }; +use crate::primitives::{Ref, RefCell, RefMut}; + /// The B-Tree page header is 12 bytes for interior pages and 8 bytes for leaf pages. /// /// +--------+-----------------+-----------------+-----------------+--------+----- ..... ----+ @@ -552,7 +554,7 @@ pub trait CursorTrait: Any { // --- start: BTreeCursor specific functions ---- fn invalidate_record(&mut self); fn has_rowid(&self) -> bool; - fn record_cursor_mut(&self) -> std::cell::RefMut<'_, RecordCursor>; + fn record_cursor_mut(&self) -> RefMut<'_, RecordCursor>; fn get_pager(&self) -> Arc; fn get_skip_advance(&self) -> bool; @@ -5422,7 +5424,7 @@ impl BTreeCursor { Ok(IOResult::Done(())) } - fn get_immutable_record_or_create(&self) -> std::cell::RefMut<'_, Option> { + fn get_immutable_record_or_create(&self) -> RefMut<'_, Option> { let mut reusable_immutable_record = self.reusable_immutable_record.borrow_mut(); if reusable_immutable_record.is_none() { let page_size = self.pager.get_page_size_unchecked().get(); @@ -5432,7 +5434,7 @@ impl BTreeCursor { reusable_immutable_record } - fn get_immutable_record(&self) -> std::cell::RefMut<'_, Option> { + fn get_immutable_record(&self) -> RefMut<'_, Option> { self.reusable_immutable_record.borrow_mut() } @@ -6390,7 +6392,7 @@ impl CursorTrait for BTreeCursor { .invalidate(); self.record_cursor.borrow_mut().invalidate(); } - fn record_cursor_mut(&self) -> std::cell::RefMut<'_, RecordCursor> { + fn record_cursor_mut(&self) -> RefMut<'_, RecordCursor> { self.record_cursor.borrow_mut() } diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index dcfa26046..e74f64a1c 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -110,7 +110,7 @@ macro_rules! load_insn { }; #[cfg(not(debug_assertions))] let Insn::$variant { $($field $(: $binding)?),*} = $insn else { - // this will optimize away the branch + // this will optimize away the branch unsafe { std::hint::unreachable_unchecked() }; }; }; From 4b3689e9e7d0263410112e4fd7420abf1d22b398 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 15 Oct 2025 16:27:02 +0400 Subject: [PATCH 16/25] avoid doing work in case of heap-sort optimization --- core/translate/order_by.rs | 66 +++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index bd419e016..781482df5 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -324,6 +324,40 @@ pub fn order_by_sorter_insert( )?; } } + + let SortMetadata { + sort_cursor, + reg_sorter_data, + use_heap_sort, + .. + } = sort_metadata; + + let (insert_label, skip_label) = if *use_heap_sort { + // skip records which greater than current top-k maintained in a separate BTreeIndex + let insert_label = program.allocate_label(); + let skip_label = program.allocate_label(); + let limit = t_ctx.limit_ctx.as_ref().expect("limit must be set"); + let limit_reg = t_ctx.reg_limit_offset_sum.unwrap_or(limit.reg_limit); + program.emit_insn(Insn::IfPos { + reg: limit_reg, + target_pc: insert_label, + decrement_by: 1, + }); + program.emit_insn(Insn::Last { + cursor_id: *sort_cursor, + pc_if_empty: insert_label, + }); + program.emit_insn(Insn::IdxLE { + cursor_id: *sort_cursor, + start_reg, + num_regs: orderby_sorter_column_count, + target_pc: skip_label, + }); + (Some(insert_label), Some(skip_label)) + } else { + (None, None) + }; + let mut cur_reg = start_reg + order_by_len; if sort_metadata.has_sequence { program.emit_insn(Insn::Sequence { @@ -417,41 +451,13 @@ pub fn order_by_sorter_insert( } } - let SortMetadata { - sort_cursor, - reg_sorter_data, - use_heap_sort, - .. - } = sort_metadata; - if *use_heap_sort { - // maintain top-k records in the index instead of materializing the whole sequence - let insert_label = program.allocate_label(); - let skip_label = program.allocate_label(); - let limit = t_ctx.limit_ctx.as_ref().expect("limit must be set"); - let limit_reg = t_ctx.reg_limit_offset_sum.unwrap_or(limit.reg_limit); - program.emit_insn(Insn::IfPos { - reg: limit_reg, - target_pc: insert_label, - decrement_by: 1, - }); - program.emit_insn(Insn::Last { - cursor_id: *sort_cursor, - pc_if_empty: insert_label, - }); - program.emit_insn(Insn::IdxLE { - cursor_id: *sort_cursor, - start_reg, - num_regs: orderby_sorter_column_count, - target_pc: skip_label, - }); - program.emit_insn(Insn::Delete { cursor_id: *sort_cursor, table_name: "".to_string(), }); - program.preassign_label_to_next_insn(insert_label); + program.preassign_label_to_next_insn(insert_label.unwrap()); program.emit_insn(Insn::MakeRecord { start_reg, count: orderby_sorter_column_count, @@ -466,7 +472,7 @@ pub fn order_by_sorter_insert( unpacked_count: None, flags: IdxInsertFlags::new(), }); - program.preassign_label_to_next_insn(skip_label); + program.preassign_label_to_next_insn(skip_label.unwrap()); } else { sorter_insert( program, From 671d266dd662690ebda0d47763cbd04b48ea89ba Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 11:47:46 +0400 Subject: [PATCH 17/25] Revert "wip" This reverts commit dd34f7fd504fbbd57e7802da4e76447c4911ab09. --- core/lib.rs | 1 - core/storage/btree.rs | 12 +++++------- core/vdbe/execute.rs | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/core/lib.rs b/core/lib.rs index 8a057e326..637e91c92 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -15,7 +15,6 @@ mod json; pub mod mvcc; mod parameters; mod pragma; -pub mod primitives; mod pseudo; mod schema; #[cfg(feature = "series")] diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 38a4511b7..3124050cd 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -41,7 +41,7 @@ use super::{ }; use std::{ any::Any, - cell::Cell, + cell::{Cell, Ref, RefCell}, cmp::{Ordering, Reverse}, collections::{BinaryHeap, HashMap}, fmt::Debug, @@ -50,8 +50,6 @@ use std::{ sync::Arc, }; -use crate::primitives::{Ref, RefCell, RefMut}; - /// The B-Tree page header is 12 bytes for interior pages and 8 bytes for leaf pages. /// /// +--------+-----------------+-----------------+-----------------+--------+----- ..... ----+ @@ -554,7 +552,7 @@ pub trait CursorTrait: Any { // --- start: BTreeCursor specific functions ---- fn invalidate_record(&mut self); fn has_rowid(&self) -> bool; - fn record_cursor_mut(&self) -> RefMut<'_, RecordCursor>; + fn record_cursor_mut(&self) -> std::cell::RefMut<'_, RecordCursor>; fn get_pager(&self) -> Arc; fn get_skip_advance(&self) -> bool; // --- end: BTreeCursor specific functions ---- @@ -4795,7 +4793,7 @@ impl BTreeCursor { Ok(IOResult::Done(())) } - fn get_immutable_record_or_create(&self) -> RefMut<'_, Option> { + fn get_immutable_record_or_create(&self) -> std::cell::RefMut<'_, Option> { let mut reusable_immutable_record = self.reusable_immutable_record.borrow_mut(); if reusable_immutable_record.is_none() { let page_size = self.pager.get_page_size_unchecked().get(); @@ -4805,7 +4803,7 @@ impl BTreeCursor { reusable_immutable_record } - fn get_immutable_record(&self) -> RefMut<'_, Option> { + fn get_immutable_record(&self) -> std::cell::RefMut<'_, Option> { self.reusable_immutable_record.borrow_mut() } @@ -5613,7 +5611,7 @@ impl CursorTrait for BTreeCursor { .invalidate(); self.record_cursor.borrow_mut().invalidate(); } - fn record_cursor_mut(&self) -> RefMut<'_, RecordCursor> { + fn record_cursor_mut(&self) -> std::cell::RefMut<'_, RecordCursor> { self.record_cursor.borrow_mut() } diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index fa42f8a4e..dadc7a063 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -112,7 +112,7 @@ macro_rules! load_insn { }; #[cfg(not(debug_assertions))] let Insn::$variant { $($field $(: $binding)?),*} = $insn else { - // this will optimize away the branch + // this will optimize away the branch unsafe { std::hint::unreachable_unchecked() }; }; }; From 0fb149c4c9924b2666b1c90db883a6d9eda86a4c Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 17:44:02 +0400 Subject: [PATCH 18/25] fix bug --- core/translate/order_by.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index c80a504f7..2dd7a3c8a 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -332,7 +332,7 @@ pub fn order_by_sorter_insert( .. } = sort_metadata; - let (insert_label, skip_label) = if *use_heap_sort { + let skip_label = if *use_heap_sort { // skip records which greater than current top-k maintained in a separate BTreeIndex let insert_label = program.allocate_label(); let skip_label = program.allocate_label(); @@ -353,9 +353,15 @@ pub fn order_by_sorter_insert( num_regs: orderby_sorter_column_count, target_pc: skip_label, }); - (Some(insert_label), Some(skip_label)) + program.emit_insn(Insn::Delete { + cursor_id: *sort_cursor, + table_name: "".to_string(), + is_part_of_update: false, + }); + program.preassign_label_to_next_insn(insert_label); + Some(skip_label) } else { - (None, None) + None }; let mut cur_reg = start_reg + order_by_len; @@ -452,13 +458,6 @@ pub fn order_by_sorter_insert( } if *use_heap_sort { - program.emit_insn(Insn::Delete { - cursor_id: *sort_cursor, - table_name: "".to_string(), - is_part_of_update: false, - }); - - program.preassign_label_to_next_insn(insert_label.unwrap()); program.emit_insn(Insn::MakeRecord { start_reg, count: orderby_sorter_column_count, From 689c11a21a5d1b19d881925a972cd848861746e9 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 17:45:49 +0400 Subject: [PATCH 19/25] cargo fmt --- core/vdbe/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index bc9a159b1..847f53004 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -1028,7 +1028,6 @@ fn make_record(registers: &[Register], start_reg: &usize, count: &usize) -> Immu ImmutableRecord::from_registers(regs, regs.len()) } - #[instrument(skip(program), level = Level::DEBUG)] fn trace_insn(program: &Program, addr: InsnReference, insn: &Insn) { tracing::trace!( From 8e1cec5104e4f1c6c3226b4858a0421c200a9609 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 19:30:43 +0400 Subject: [PATCH 20/25] Revert "alternative read_variant implementation" This reverts commit 68650cf594799d4c5a97de3670bfe157b5229698. --- core/storage/sqlite3_ondisk.rs | 85 +++++++++++++++++++++++----------- core/vdbe/execute.rs | 7 +-- 2 files changed, 62 insertions(+), 30 deletions(-) diff --git a/core/storage/sqlite3_ondisk.rs b/core/storage/sqlite3_ondisk.rs index d62d7aa9f..190b94479 100644 --- a/core/storage/sqlite3_ondisk.rs +++ b/core/storage/sqlite3_ondisk.rs @@ -1486,40 +1486,71 @@ pub fn read_integer(buf: &[u8], serial_type: u8) -> Result { } } +/// Fast varint reader optimized for the common cases of 1-byte and 2-byte varints. +/// +/// This function is a performance-optimized version of `read_varint()` that handles +/// the most common varint cases inline before falling back to the full implementation. +/// It follows the same varint encoding as SQLite. +/// +/// # Optimized Cases +/// +/// - **Single-byte case**: Values 0-127 (0x00-0x7F) are returned immediately +/// - **Two-byte case**: Values 128-16383 (0x80-0x3FFF) are handled inline +/// - **Multi-byte case**: Larger values fall back to the full `read_varint()` implementation +/// +/// This function is similar to `sqlite3GetVarint32` +#[inline(always)] +pub fn read_varint_fast(buf: &[u8]) -> Result<(u64, usize)> { + // Fast path: Single-byte varint + if let Some(&first_byte) = buf.first() { + if first_byte & 0x80 == 0 { + return Ok((first_byte as u64, 1)); + } + } else { + crate::bail_corrupt_error!("Invalid varint"); + } + + // Fast path: Two-byte varint + if let Some(&second_byte) = buf.get(1) { + if second_byte & 0x80 == 0 { + let v = (((buf[0] & 0x7f) as u64) << 7) + (second_byte as u64); + return Ok((v, 2)); + } + } else { + crate::bail_corrupt_error!("Invalid varint"); + } + + //Fallback: Multi-byte varint + read_varint(buf) +} + #[inline(always)] pub fn read_varint(buf: &[u8]) -> Result<(u64, usize)> { let mut v: u64 = 0; - let mut i = 0; - let chunks = buf.chunks_exact(2); - for chunk in chunks { - let c1 = chunk[0]; - v = (v << 7) + (c1 & 0x7f) as u64; - i += 1; - if (c1 & 0x80) == 0 { - return Ok((v, i)); - } - let c2 = chunk[1]; - v = (v << 7) + (c2 & 0x7f) as u64; - i += 1; - if (c2 & 0x80) == 0 { - return Ok((v, i)); - } - if i == 8 { - break; + for i in 0..8 { + match buf.get(i) { + Some(c) => { + v = (v << 7) + (c & 0x7f) as u64; + if (c & 0x80) == 0 { + return Ok((v, i + 1)); + } + } + None => { + crate::bail_corrupt_error!("Invalid varint"); + } } } - match buf.get(i) { + match buf.get(8) { Some(&c) => { - if i < 8 && (c & 0x80) == 0 { - return Ok(((v << 7) + c as u64, i + 1)); - } else if i == 8 && (v >> 48) > 0 { - // Values requiring 9 bytes must have non-zero in the top 8 bits (value >= 1<<56). - // Since the final value is `(v<<8) + c`, the top 8 bits (v >> 48) must not be 0. - // If those are zero, this should be treated as corrupt. - // Perf? the comparison + branching happens only in parsing 9-byte varint which is rare. - return Ok(((v << 8) + c as u64, i + 1)); + // Values requiring 9 bytes must have non-zero in the top 8 bits (value >= 1<<56). + // Since the final value is `(v<<8) + c`, the top 8 bits (v >> 48) must not be 0. + // If those are zero, this should be treated as corrupt. + // Perf? the comparison + branching happens only in parsing 9-byte varint which is rare. + if (v >> 48) == 0 { + bail_corrupt_error!("Invalid varint"); } - bail_corrupt_error!("Invalid varint"); + v = (v << 8) + c as u64; + Ok((v, 9)) } None => { bail_corrupt_error!("Invalid varint"); diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index dadc7a063..f989d23f0 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -11,7 +11,7 @@ use crate::storage::btree::{ use crate::storage::database::DatabaseFile; use crate::storage::page_cache::PageCache; use crate::storage::pager::{AtomicDbState, CreateBTreeFlags, DbState}; -use crate::storage::sqlite3_ondisk::{read_varint, DatabaseHeader, PageSize}; +use crate::storage::sqlite3_ondisk::{read_varint_fast, DatabaseHeader, PageSize}; use crate::translate::collate::CollationSeq; use crate::types::{ compare_immutable, compare_records_generic, Extendable, IOCompletions, ImmutableRecord, @@ -1612,7 +1612,7 @@ pub fn op_column( let mut record_cursor = cursor.record_cursor_mut(); if record_cursor.offsets.is_empty() { - let (header_size, header_len_bytes) = read_varint(payload)?; + let (header_size, header_len_bytes) = read_varint_fast(payload)?; let header_size = header_size as usize; debug_assert!(header_size <= payload.len() && header_size <= 98307, "header_size: {header_size}, header_len_bytes: {header_len_bytes}, payload.len(): {}", payload.len()); @@ -1634,7 +1634,8 @@ pub fn op_column( while record_cursor.serial_types.len() <= target_column && parse_pos < record_cursor.header_size { - let (serial_type, varint_len) = read_varint(&payload[parse_pos..])?; + let (serial_type, varint_len) = + read_varint_fast(&payload[parse_pos..])?; record_cursor.serial_types.push(serial_type); parse_pos += varint_len; let data_size = match serial_type { From b32d22a2fdc35bcafa3b0babd58fb86ae1610244 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 20:20:54 +0400 Subject: [PATCH 21/25] Revert "move more possible option higher" This reverts commit c0fdaeb4755fa49b8fab928a566cd536b476c55f. --- core/vdbe/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 847f53004..3aec5900c 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -134,8 +134,8 @@ impl BranchOffset { /// Returns the offset value. Panics if the branch offset is a label or placeholder. pub fn as_offset_int(&self) -> InsnReference { match self { - BranchOffset::Offset(v) => *v, BranchOffset::Label(v) => unreachable!("Unresolved label: {}", v), + BranchOffset::Offset(v) => *v, BranchOffset::Placeholder => unreachable!("Unresolved placeholder"), } } From 53957b6d2214154f30acc7d63b398e23e1cf4a9f Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 20:21:00 +0400 Subject: [PATCH 22/25] Revert "simplify serial_type size calculation" This reverts commit f19c73822ee9afbb9c859fe6b7f3f9e0ee3bc576. --- core/vdbe/execute.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index f989d23f0..c671a81a7 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -1659,14 +1659,17 @@ pub fn op_column( 8 => 0, // CONST_INT1 9 => 0, + // BLOB + n if n >= 12 && n & 1 == 0 => (n - 12) >> 1, + // TEXT + n if n >= 13 && n & 1 == 1 => (n - 13) >> 1, // Reserved 10 | 11 => { return Err(LimboError::Corrupt(format!( "Reserved serial type: {serial_type}" ))) } - // BLOB or TEXT - n => (n - 12) / 2, + _ => unreachable!("Invalid serial type: {serial_type}"), } as usize; data_offset += data_size; record_cursor.offsets.push(data_offset); From 91ffb4e249e726593f1cfc726071fd0a2340f8cc Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 20:21:39 +0400 Subject: [PATCH 23/25] Revert "avoid allocations" This reverts commit dba195bdfa331ab3e3533031f96c1632517d24eb. --- core/types.rs | 34 +++++++++++----------------------- core/vdbe/execute.rs | 23 ++++++++++++++++------- core/vdbe/mod.rs | 13 +++++++------ 3 files changed, 34 insertions(+), 36 deletions(-) diff --git a/core/types.rs b/core/types.rs index 8183450fa..cb9c99082 100644 --- a/core/types.rs +++ b/core/types.rs @@ -219,12 +219,6 @@ pub enum ValueRef<'a> { Blob(&'a [u8]), } -impl<'a, 'b> From<&'b ValueRef<'a>> for ValueRef<'a> { - fn from(value: &'b ValueRef<'a>) -> Self { - *value - } -} - impl Debug for ValueRef<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -1820,15 +1814,12 @@ fn compare_records_int( /// 4. **Length comparison**: If strings are equal, compares lengths /// 5. **Remaining fields**: If first field is equal and more fields exist, /// delegates to `compare_records_generic()` with `skip=1` -fn compare_records_string<'a, T>( +fn compare_records_string( serialized: &ImmutableRecord, - unpacked: &'a [T], + unpacked: &[ValueRef], index_info: &IndexInfo, tie_breaker: std::cmp::Ordering, -) -> Result -where - ValueRef<'a>: From<&'a T>, -{ +) -> Result { turso_assert!( index_info.key_info.len() >= unpacked.len(), "index_info.key_info.len() < unpacked.len()" @@ -1856,7 +1847,7 @@ where return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker); } - let ValueRef::Text(rhs_text, _) = (&unpacked[0]).into() else { + let ValueRef::Text(rhs_text, _) = &unpacked[0] else { return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker); }; @@ -1935,16 +1926,13 @@ where /// The serialized and unpacked records do not have to contain the same number /// of fields. If all fields that appear in both records are equal, then /// `tie_breaker` is returned. -pub fn compare_records_generic<'a, T>( +pub fn compare_records_generic( serialized: &ImmutableRecord, - unpacked: &'a [T], + unpacked: &[ValueRef], index_info: &IndexInfo, skip: usize, tie_breaker: std::cmp::Ordering, -) -> Result -where - ValueRef<'a>: From<&'a T>, -{ +) -> Result { turso_assert!( index_info.key_info.len() >= unpacked.len(), "index_info.key_info.len() < unpacked.len()" @@ -1984,7 +1972,7 @@ where header_pos += bytes_read; let serial_type = SerialType::try_from(serial_type_raw)?; - let rhs_value = (&unpacked[field_idx]).into(); + let rhs_value = &unpacked[field_idx]; let lhs_value = match serial_type.kind() { SerialTypeKind::ConstInt0 => ValueRef::Integer(0), @@ -2006,14 +1994,14 @@ where } (ValueRef::Integer(lhs_int), ValueRef::Float(rhs_float)) => { - sqlite_int_float_compare(*lhs_int, rhs_float) + sqlite_int_float_compare(*lhs_int, *rhs_float) } (ValueRef::Float(lhs_float), ValueRef::Integer(rhs_int)) => { - sqlite_int_float_compare(rhs_int, *lhs_float).reverse() + sqlite_int_float_compare(*rhs_int, *lhs_float).reverse() } - _ => lhs_value.partial_cmp(&rhs_value).unwrap(), + _ => lhs_value.partial_cmp(rhs_value).unwrap(), }; let final_comparison = match index_info.key_info[field_idx].sort_order { diff --git a/core/vdbe/execute.rs b/core/vdbe/execute.rs index c671a81a7..247960d16 100644 --- a/core/vdbe/execute.rs +++ b/core/vdbe/execute.rs @@ -21,7 +21,7 @@ use crate::util::{ normalize_ident, rewrite_column_references_if_needed, rewrite_fk_parent_cols_if_self_ref, }; use crate::vdbe::insn::InsertFlags; -use crate::vdbe::TxnCleanup; +use crate::vdbe::{registers_to_ref_values, TxnCleanup}; use crate::vector::{vector32_sparse, vector_concat, vector_distance_jaccard, vector_slice}; use crate::{ error::{ @@ -3384,11 +3384,13 @@ pub fn op_idx_ge( let pc = if let Some(idx_record) = return_if_io!(cursor.record()) { // Create the comparison record from registers + let values = + registers_to_ref_values(&state.registers[*start_reg..*start_reg + *num_regs]); let tie_breaker = get_tie_breaker_from_idx_comp_op(insn); let ord = compare_records_generic( - &idx_record, // The serialized record from the index - &state.registers[*start_reg..*start_reg + *num_regs], // The record built from registers - cursor.get_index_info(), // Sort order flags + &idx_record, // The serialized record from the index + &values, // The record built from registers + cursor.get_index_info(), // Sort order flags 0, tie_breaker, )?; @@ -3450,10 +3452,12 @@ pub fn op_idx_le( let cursor = cursor.as_btree_mut(); let pc = if let Some(idx_record) = return_if_io!(cursor.record()) { + let values = + registers_to_ref_values(&state.registers[*start_reg..*start_reg + *num_regs]); let tie_breaker = get_tie_breaker_from_idx_comp_op(insn); let ord = compare_records_generic( &idx_record, - &state.registers[*start_reg..*start_reg + *num_regs], + &values, cursor.get_index_info(), 0, tie_breaker, @@ -3499,10 +3503,12 @@ pub fn op_idx_gt( let cursor = cursor.as_btree_mut(); let pc = if let Some(idx_record) = return_if_io!(cursor.record()) { + let values = + registers_to_ref_values(&state.registers[*start_reg..*start_reg + *num_regs]); let tie_breaker = get_tie_breaker_from_idx_comp_op(insn); let ord = compare_records_generic( &idx_record, - &state.registers[*start_reg..*start_reg + *num_regs], + &values, cursor.get_index_info(), 0, tie_breaker, @@ -3548,10 +3554,13 @@ pub fn op_idx_lt( let cursor = cursor.as_btree_mut(); let pc = if let Some(idx_record) = return_if_io!(cursor.record()) { + let values = + registers_to_ref_values(&state.registers[*start_reg..*start_reg + *num_regs]); + let tie_breaker = get_tie_breaker_from_idx_comp_op(insn); let ord = compare_records_generic( &idx_record, - &state.registers[*start_reg..*start_reg + *num_regs], + &values, cursor.get_index_info(), 0, tie_breaker, diff --git a/core/vdbe/mod.rs b/core/vdbe/mod.rs index 3aec5900c..b462b8f82 100644 --- a/core/vdbe/mod.rs +++ b/core/vdbe/mod.rs @@ -251,12 +251,6 @@ pub enum Register { Record(ImmutableRecord), } -impl<'a> From<&'a Register> for ValueRef<'a> { - fn from(value: &'a Register) -> Self { - value.get_value().as_ref() - } -} - impl Register { #[inline] pub fn is_null(&self) -> bool { @@ -1028,6 +1022,13 @@ fn make_record(registers: &[Register], start_reg: &usize, count: &usize) -> Immu ImmutableRecord::from_registers(regs, regs.len()) } +pub fn registers_to_ref_values<'a>(registers: &'a [Register]) -> Vec> { + registers + .iter() + .map(|reg| reg.get_value().as_ref()) + .collect() +} + #[instrument(skip(program), level = Level::DEBUG)] fn trace_insn(program: &Program, addr: InsnReference, insn: &Insn) { tracing::trace!( From a071d40d5f757e693bbac4e2fc036f99be3731c0 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 20:21:47 +0400 Subject: [PATCH 24/25] Revert "faster extend_from_slice" This reverts commit ae8adc044958145dfc3f1224c55f1b1b3be45422. --- core/types.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/core/types.rs b/core/types.rs index cb9c99082..a7259f8b1 100644 --- a/core/types.rs +++ b/core/types.rs @@ -1061,15 +1061,7 @@ impl ImmutableRecord { } pub fn start_serialization(&mut self, payload: &[u8]) { - let blob = self.as_blob_mut(); - blob.reserve(payload.len()); - - let len = blob.len(); - unsafe { - let dst = blob.as_mut_ptr().add(len); - std::ptr::copy_nonoverlapping(payload.as_ptr(), dst, payload.len()); - blob.set_len(len + payload.len()); - } + self.as_blob_mut().extend_from_slice(payload); } pub fn invalidate(&mut self) { From 6aa67c6ea0d24f414df3950d4fd40778187c6860 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 22 Oct 2025 20:21:52 +0400 Subject: [PATCH 25/25] Revert "slight reorder of operations" This reverts commit 8e107ab18e5197e2d4d4ffcc7e0774d81e59c6e9. --- core/storage/btree.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 3124050cd..440c0de79 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -5001,26 +5001,30 @@ impl CursorTrait for BTreeCursor { first_overflow_page, .. }) => (payload, payload_size, first_overflow_page), - BTreeCell::IndexLeafCell(IndexLeafCell { - payload, - payload_size, - first_overflow_page, - }) => (payload, payload_size, first_overflow_page), BTreeCell::IndexInteriorCell(IndexInteriorCell { payload, payload_size, first_overflow_page, .. }) => (payload, payload_size, first_overflow_page), + BTreeCell::IndexLeafCell(IndexLeafCell { + payload, + first_overflow_page, + payload_size, + }) => (payload, payload_size, first_overflow_page), _ => unreachable!("unexpected page_type"), }; if let Some(next_page) = first_overflow_page { return_if_io!(self.process_overflow_read(payload, next_page, payload_size)) } else { - let mut record = self.get_immutable_record_or_create(); - let record = record.as_mut().unwrap(); - record.invalidate(); - record.start_serialization(payload); + self.get_immutable_record_or_create() + .as_mut() + .unwrap() + .invalidate(); + self.get_immutable_record_or_create() + .as_mut() + .unwrap() + .start_serialization(payload); self.record_cursor.borrow_mut().invalidate(); };