aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-01-27 11:54:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'Refactor affinity conversions for reusability' from Pedro Muniz

Browse Source 
Depends on #3920
Moves some code around so it is easier to reuse and less cluttered in
`execute.rs`, and changes how `compare` works. Instead of mutating some
register, we now just return the possible `ValueRef` representation of
that affinity. This allows other parts of the codebase to reuse this
logic without needing to have an owned `Value` or a `&mut Register`

Reviewed-by: Jussi Saurio <jussi.saurio@gmail.com>

Closes #3923
This commit is contained in:
Pekka Enberg
2025-11-10 09:02:22 +02:00
committed by GitHub
parent 2be515247f 9007340e99
commit 7891be96fd
13 changed files with 864 additions and 770 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
core/Cargo.toml
View File

@@ -85,6 +85,7 @@ roaring = "0.11.2"
simsimd = "6.5.3"
arc-swap = "1.7"
rustc-hash = "2.0"
either = { workspace = true }
[build-dependencies]
chrono = { workspace = true, default-features = false }

171
core/schema.rs
View File

@@ -4,6 +4,7 @@ use crate::index_method::{IndexMethodAttachment, IndexMethodConfiguration};
use crate::translate::expr::{bind_and_rewrite_expr, walk_expr, BindingBehavior, WalkControl};
use crate::translate::index::{resolve_index_method_parameters, resolve_sorted_columns};
use crate::translate::planner::ROWID_STRS;
use crate::vdbe::affinity::Affinity;
use parking_lot::RwLock;
use turso_macros::AtomicEnum;
@@ -2081,7 +2082,7 @@ const COLL_MASK: u16 = 0b11 << COLL_SHIFT;
impl Column {
pub fn affinity(&self) -> Affinity {
affinity(&self.ty_str)
Affinity::affinity(&self.ty_str)
}
pub const fn new_default_text(
name: Option<String>,
@@ -2309,47 +2310,6 @@ impl From<&ColumnDefinition> for Column {
}
}
/// 3.1. Determination Of Column Affinity
/// For tables not declared as STRICT, the affinity of a column is determined by the declared type of the column, according to the following rules in the order shown:
///
/// If the declared type contains the string "INT" then it is assigned INTEGER affinity.
///
/// If the declared type of the column contains any of the strings "CHAR", "CLOB", or "TEXT" then that column has TEXT affinity. Notice that the type VARCHAR contains the string "CHAR" and is thus assigned TEXT affinity.
///
/// If the declared type for a column contains the string "BLOB" or if no type is specified then the column has affinity BLOB.
///
/// If the declared type for a column contains any of the strings "REAL", "FLOA", or "DOUB" then the column has REAL affinity.
///
/// Otherwise, the affinity is NUMERIC.
///
/// Note that the order of the rules for determining column affinity is important. A column whose declared type is "CHARINT" will match both rules 1 and 2 but the first rule takes precedence and so the column affinity will be INTEGER.
pub fn affinity(datatype: &str) -> Affinity {
let datatype = datatype.to_ascii_uppercase();
// Rule 1: INT -> INTEGER affinity
if datatype.contains("INT") {
return Affinity::Integer;
}
// Rule 2: CHAR/CLOB/TEXT -> TEXT affinity
if datatype.contains("CHAR") || datatype.contains("CLOB") || datatype.contains("TEXT") {
return Affinity::Text;
}
// Rule 3: BLOB or empty -> BLOB affinity (historically called NONE)
if datatype.contains("BLOB") || datatype.is_empty() || datatype.contains("ANY") {
return Affinity::Blob;
}
// Rule 4: REAL/FLOA/DOUB -> REAL affinity
if datatype.contains("REAL") || datatype.contains("FLOA") || datatype.contains("DOUB") {
return Affinity::Real;
}
// Rule 5: Otherwise -> NUMERIC affinity
Affinity::Numeric
}
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Type {
@@ -2376,133 +2336,6 @@ impl Type {
}
}
/// # SQLite Column Type Affinities
///
/// Each column in an SQLite 3 database is assigned one of the following type affinities:
///
/// - **TEXT**
/// - **NUMERIC**
/// - **INTEGER**
/// - **REAL**
/// - **BLOB**
///
/// > **Note:** Historically, the "BLOB" type affinity was called "NONE". However, this term was renamed to avoid confusion with "no affinity".
///
/// ## Affinity Descriptions
///
/// ### **TEXT**
/// - Stores data using the NULL, TEXT, or BLOB storage classes.
/// - Numerical data inserted into a column with TEXT affinity is converted into text form before being stored.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col TEXT);
/// INSERT INTO example (col) VALUES (123); -- Stored as '123' (text)
/// SELECT typeof(col) FROM example; -- Returns 'text'
/// ```
///
/// ### **NUMERIC**
/// - Can store values using all five storage classes.
/// - Text data is converted to INTEGER or REAL (in that order of preference) if it is a well-formed integer or real literal.
/// - If the text represents an integer too large for a 64-bit signed integer, it is converted to REAL.
/// - If the text is not a well-formed literal, it is stored as TEXT.
/// - Hexadecimal integer literals are stored as TEXT for historical compatibility.
/// - Floating-point values that can be exactly represented as integers are converted to integers.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col NUMERIC);
/// INSERT INTO example (col) VALUES ('3.0e+5'); -- Stored as 300000 (integer)
/// SELECT typeof(col) FROM example; -- Returns 'integer'
/// ```
///
/// ### **INTEGER**
/// - Behaves like NUMERIC affinity but differs in `CAST` expressions.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col INTEGER);
/// INSERT INTO example (col) VALUES (4.0); -- Stored as 4 (integer)
/// SELECT typeof(col) FROM example; -- Returns 'integer'
/// ```
///
/// ### **REAL**
/// - Similar to NUMERIC affinity but forces integer values into floating-point representation.
/// - **Optimization:** Small floating-point values with no fractional component may be stored as integers on disk to save space. This is invisible at the SQL level.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col REAL);
/// INSERT INTO example (col) VALUES (4); -- Stored as 4.0 (real)
/// SELECT typeof(col) FROM example; -- Returns 'real'
/// ```
///
/// ### **BLOB**
/// - Does not prefer any storage class.
/// - No coercion is performed between storage classes.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col BLOB);
/// INSERT INTO example (col) VALUES (x'1234'); -- Stored as a binary blob
/// SELECT typeof(col) FROM example; -- Returns 'blob'
/// ```
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Affinity {
Integer,
Text,
Blob,
Real,
Numeric,
}
pub const SQLITE_AFF_NONE: char = 'A'; // Historically called NONE, but it's the same as BLOB
pub const SQLITE_AFF_TEXT: char = 'B';
pub const SQLITE_AFF_NUMERIC: char = 'C';
pub const SQLITE_AFF_INTEGER: char = 'D';
pub const SQLITE_AFF_REAL: char = 'E';
impl Affinity {
/// This is meant to be used in opcodes like Eq, which state:
///
/// "The SQLITE_AFF_MASK portion of P5 must be an affinity character - SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth.
/// An attempt is made to coerce both inputs according to this affinity before the comparison is made.
/// If the SQLITE_AFF_MASK is 0x00, then numeric affinity is used.
/// Note that the affinity conversions are stored back into the input registers P1 and P3.
/// So this opcode can cause persistent changes to registers P1 and P3.""
pub fn aff_mask(&self) -> char {
match self {
Affinity::Integer => SQLITE_AFF_INTEGER,
Affinity::Text => SQLITE_AFF_TEXT,
Affinity::Blob => SQLITE_AFF_NONE,
Affinity::Real => SQLITE_AFF_REAL,
Affinity::Numeric => SQLITE_AFF_NUMERIC,
}
}
pub fn from_char(char: char) -> Self {
match char {
SQLITE_AFF_INTEGER => Affinity::Integer,
SQLITE_AFF_TEXT => Affinity::Text,
SQLITE_AFF_NONE => Affinity::Blob,
SQLITE_AFF_REAL => Affinity::Real,
SQLITE_AFF_NUMERIC => Affinity::Numeric,
_ => Affinity::Blob,
}
}
pub fn as_char_code(&self) -> u8 {
self.aff_mask() as u8
}
pub fn from_char_code(code: u8) -> Self {
Self::from_char(code as char)
}
pub fn is_numeric(&self) -> bool {
matches!(self, Affinity::Integer | Affinity::Real | Affinity::Numeric)
}
pub fn has_affinity(&self) -> bool {
!matches!(self, Affinity::Blob)
}
}
impl fmt::Display for Type {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {

7
core/translate/expr.rs
View File

@@ -10,11 +10,12 @@ use super::plan::TableReferences;
use crate::function::JsonFunc;
use crate::function::{Func, FuncCtx, MathFuncArity, ScalarFunc, VectorFunc};
use crate::functions::datetime;
use crate::schema::{affinity, Affinity, Table, Type};
use crate::schema::{Table, Type};
use crate::translate::optimizer::TakeOwnership;
use crate::translate::plan::{Operation, ResultSetColumn};
use crate::translate::planner::parse_row_id;
use crate::util::{exprs_are_equivalent, normalize_ident, parse_numeric_literal};
use crate::vdbe::affinity::Affinity;
use crate::vdbe::builder::CursorKey;
use crate::vdbe::{
builder::ProgramBuilder,
@@ -868,7 +869,7 @@ pub fn translate_expr(
ast::Expr::Cast { expr, type_name } => {
let type_name = type_name.as_ref().unwrap(); // TODO: why is this optional?
translate_expr(program, referenced_tables, expr, target_register, resolver)?;
let type_affinity = affinity(&type_name.name);
let type_affinity = Affinity::affinity(&type_name.name);
program.emit_insn(Insn::Cast {
reg: target_register,
affinity: type_affinity,
@@ -4084,7 +4085,7 @@ pub fn get_expr_affinity(
ast::Expr::RowId { .. } => Affinity::Integer,
ast::Expr::Cast { type_name, .. } => {
if let Some(type_name) = type_name {
crate::schema::affinity(&type_name.name)
Affinity::affinity(&type_name.name)
} else {
Affinity::Blob
}

3
core/translate/insert.rs
View File

@@ -7,7 +7,7 @@ use turso_parser::ast::{
use crate::error::{
SQLITE_CONSTRAINT_NOTNULL, SQLITE_CONSTRAINT_PRIMARYKEY, SQLITE_CONSTRAINT_UNIQUE,
};
use crate::schema::{self, Affinity, BTreeTable, Index, ResolvedFkRef, Table};
use crate::schema::{self, BTreeTable, Index, ResolvedFkRef, Table};
use crate::translate::emitter::{
emit_cdc_insns, emit_cdc_patch_record, prepare_cdc_if_necessary, OperationMode,
};
@@ -25,6 +25,7 @@ use crate::translate::upsert::{
collect_set_clauses_for_upsert, emit_upsert, resolve_upsert_target, ResolvedUpsertTarget,
};
use crate::util::normalize_ident;
use crate::vdbe::affinity::Affinity;
use crate::vdbe::builder::ProgramBuilderOpts;
use crate::vdbe::insn::{CmpInsFlags, IdxInsertFlags, InsertFlags, RegisterOrLiteral};
use crate::vdbe::BranchOffset;

19
core/translate/main_loop.rs
View File

@@ -18,15 +18,8 @@ use super::{
Search, SeekDef, SelectPlan, TableReferences, WhereTerm,
},
};
use crate::translate::{
collate::get_collseq_from_expr,
emitter::UpdateRowSource,
plan::{EvalAt, NonFromClauseSubquery},
subquery::emit_non_from_clause_subquery,
window::emit_window_loop_source,
};
use crate::{
schema::{Affinity, Index, IndexColumn, Table},
schema::{Index, IndexColumn, Table},
translate::{
emitter::prepare_cdc_if_necessary,
plan::{DistinctCtx, Distinctness, Scan, SeekKeyComponent},
@@ -40,6 +33,16 @@ use crate::{
},
Result,
};
use crate::{
translate::{
collate::get_collseq_from_expr,
emitter::UpdateRowSource,
plan::{EvalAt, NonFromClauseSubquery},
subquery::emit_non_from_clause_subquery,
window::emit_window_loop_source,
},
vdbe::affinity::Affinity,
};
// Metadata for handling LEFT JOIN operations
#[derive(Debug)]

114
core/types.rs
View File

@@ -1,3 +1,4 @@
use either::Either;
#[cfg(feature = "serde")]
use serde::Deserialize;
use turso_ext::{AggCtx, FinalizeFunction, StepFunction};
@@ -323,6 +324,34 @@ impl AsValueRef for &mut Value {
}
}
impl<V1, V2> AsValueRef for Either<V1, V2>
where
V1: AsValueRef,
V2: AsValueRef,
{
#[inline]
fn as_value_ref<'a>(&'a self) -> ValueRef<'a> {
match self {
Either::Left(left) => left.as_value_ref(),
Either::Right(right) => right.as_value_ref(),
}
}
}
impl<V1, V2> AsValueRef for &Either<V1, V2>
where
V1: AsValueRef,
V2: AsValueRef,
{
#[inline]
fn as_value_ref<'a>(&'a self) -> ValueRef<'a> {
match self {
Either::Left(left) => left.as_value_ref(),
Either::Right(right) => right.as_value_ref(),
}
}
}
impl Value {
pub fn as_ref<'a>(&'a self) -> ValueRef<'a> {
match self {
@@ -1723,14 +1752,20 @@ pub enum RecordCompare {
}
impl RecordCompare {
pub fn compare(
pub fn compare<V, E, I>(
&self,
serialized: &ImmutableRecord,
unpacked: &[ValueRef],
unpacked: I,
index_info: &IndexInfo,
skip: usize,
tie_breaker: std::cmp::Ordering,
) -> Result<std::cmp::Ordering> {
) -> Result<std::cmp::Ordering>
where
V: AsValueRef,
E: ExactSizeIterator<Item = V>,
I: IntoIterator<IntoIter = E, Item = E::Item>,
{
let unpacked = unpacked.into_iter();
match self {
RecordCompare::Int => {
compare_records_int(serialized, unpacked, index_info, tie_breaker)
@@ -1813,12 +1848,16 @@ pub fn get_tie_breaker_from_seek_op(seek_op: SeekOp) -> std::cmp::Ordering {
/// 4. **Sort order**: Applies ascending/descending order to comparison result
/// 5. **Remaining fields**: If first field is equal and more fields exist,
/// delegates to `compare_records_generic()` with `skip=1`
fn compare_records_int(
fn compare_records_int<V, I>(
serialized: &ImmutableRecord,
unpacked: &[ValueRef],
unpacked: I,
index_info: &IndexInfo,
tie_breaker: std::cmp::Ordering,
) -> Result<std::cmp::Ordering> {
) -> Result<std::cmp::Ordering>
where
V: AsValueRef,
I: ExactSizeIterator<Item = V>,
{
let payload = serialized.get_payload();
if payload.len() < 2 {
return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker);
@@ -1845,7 +1884,9 @@ fn compare_records_int(
let data_start = header_size;
let lhs_int = read_integer(&payload[data_start..], first_serial_type as u8)?;
let ValueRef::Integer(rhs_int) = unpacked[0] else {
let mut unpacked = unpacked.peekable();
// Do not consume iterator here
let ValueRef::Integer(rhs_int) = unpacked.peek().unwrap().as_value_ref() else {
return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker);
};
let comparison = match index_info.key_info[0].sort_order {
@@ -1902,12 +1943,16 @@ fn compare_records_int(
/// 4. **Length comparison**: If strings are equal, compares lengths
/// 5. **Remaining fields**: If first field is equal and more fields exist,
/// delegates to `compare_records_generic()` with `skip=1`
fn compare_records_string(
fn compare_records_string<V, I>(
serialized: &ImmutableRecord,
unpacked: &[ValueRef],
unpacked: I,
index_info: &IndexInfo,
tie_breaker: std::cmp::Ordering,
) -> Result<std::cmp::Ordering> {
) -> Result<std::cmp::Ordering>
where
V: AsValueRef,
I: ExactSizeIterator<Item = V>,
{
let payload = serialized.get_payload();
if payload.len() < 2 {
return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker);
@@ -1931,7 +1976,9 @@ fn compare_records_string(
return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker);
}
let ValueRef::Text(rhs_text) = &unpacked[0] else {
let mut unpacked = unpacked.peekable();
let ValueRef::Text(rhs_text) = unpacked.peek().unwrap().as_value_ref() else {
return compare_records_generic(serialized, unpacked, index_info, 0, tie_breaker);
};
@@ -1948,7 +1995,7 @@ fn compare_records_string(
};
let collation = index_info.key_info[0].collation;
let comparison = collation.compare_strings(&lhs_text, rhs_text);
let comparison = collation.compare_strings(&lhs_text, &rhs_text);
let final_comparison = match index_info.key_info[0].sort_order {
SortOrder::Asc => comparison,
@@ -2007,13 +2054,17 @@ fn compare_records_string(
/// The serialized and unpacked records do not have to contain the same number
/// of fields. If all fields that appear in both records are equal, then
/// `tie_breaker` is returned.
pub fn compare_records_generic(
pub fn compare_records_generic<V, I>(
serialized: &ImmutableRecord,
unpacked: &[ValueRef],
unpacked: I,
index_info: &IndexInfo,
skip: usize,
tie_breaker: std::cmp::Ordering,
) -> Result<std::cmp::Ordering> {
) -> Result<std::cmp::Ordering>
where
V: AsValueRef,
I: ExactSizeIterator<Item = V>,
{
let payload = serialized.get_payload();
if payload.is_empty() {
return Ok(std::cmp::Ordering::Less);
@@ -2045,12 +2096,17 @@ pub fn compare_records_generic(
let mut field_idx = skip;
let field_limit = unpacked.len().min(index_info.key_info.len());
while field_idx < field_limit && header_pos < header_end {
// assumes that that the `unpacked' iterator was not skipped outside this function call`
for rhs_value in unpacked.skip(skip) {
let rhs_value = &rhs_value.as_value_ref();
if field_idx >= field_limit || header_pos >= header_end {
break;
}
let (serial_type_raw, bytes_read) = read_varint(&payload[header_pos..])?;
header_pos += bytes_read;
let serial_type = SerialType::try_from(serial_type_raw)?;
let rhs_value = &unpacked[field_idx];
let lhs_value = match serial_type.kind() {
SerialTypeKind::ConstInt0 => ValueRef::Integer(0),
@@ -2729,12 +2785,17 @@ mod tests {
"Test '{test_name}' failed: Full Comparison: {gold_result:?}, Optimized: {optimized_result:?}, Strategy: {comparer:?}"
);
let generic_result =
compare_records_generic(&serialized, &unpacked_values, index_info, 0, tie_breaker)
.unwrap();
let generic_result = compare_records_generic(
&serialized,
unpacked_values.iter(),
index_info,
0,
tie_breaker,
)
.unwrap();
assert_eq!(
gold_result, generic_result,
"Test '{test_name}' failed with generic: Full Comparison: {gold_result:?}, Generic: {generic_result:?}"
"Test '{test_name}' failed with generic: Full Comparison: {gold_result:?}, Generic: {generic_result:?}\n LHS: {serialized_values:?}\n RHS: {unpacked_values:?}"
);
}
@@ -2859,6 +2920,9 @@ mod tests {
];
for (serialized_values, unpacked_values, test_name) in test_cases {
println!(
"Testing integer fast path `{test_name}`\nLHS: {serialized_values:?}\nRHS: {unpacked_values:?}"
);
assert_compare_matches_full_comparison(
serialized_values,
unpacked_values,
@@ -3123,7 +3187,7 @@ mod tests {
Value::Integer(2),
Value::Integer(3),
]);
let unpacked = vec![
let unpacked = [
ValueRef::Integer(1),
ValueRef::Integer(99),
ValueRef::Integer(3),
@@ -3131,9 +3195,11 @@ mod tests {
let tie_breaker = std::cmp::Ordering::Equal;
let result_skip_0 =
compare_records_generic(&serialized, &unpacked, &index_info, 0, tie_breaker).unwrap();
compare_records_generic(&serialized, unpacked.iter(), &index_info, 0, tie_breaker)
.unwrap();
let result_skip_1 =
compare_records_generic(&serialized, &unpacked, &index_info, 1, tie_breaker).unwrap();
compare_records_generic(&serialized, unpacked.iter(), &index_info, 1, tie_breaker)
.unwrap();
assert_eq!(result_skip_0, std::cmp::Ordering::Less);

618
core/vdbe/affinity.rs Normal file
View File

@@ -0,0 +1,618 @@
use either::Either;
use turso_parser::ast::{Expr, Literal};
use crate::{types::AsValueRef, Value, ValueRef};
/// # SQLite Column Type Affinities
///
/// Each column in an SQLite 3 database is assigned one of the following type affinities:
///
/// - **TEXT**
/// - **NUMERIC**
/// - **INTEGER**
/// - **REAL**
/// - **BLOB**
///
/// > **Note:** Historically, the "BLOB" type affinity was called "NONE". However, this term was renamed to avoid confusion with "no affinity".
///
/// ## Affinity Descriptions
///
/// ### **TEXT**
/// - Stores data using the NULL, TEXT, or BLOB storage classes.
/// - Numerical data inserted into a column with TEXT affinity is converted into text form before being stored.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col TEXT);
/// INSERT INTO example (col) VALUES (123); -- Stored as '123' (text)
/// SELECT typeof(col) FROM example; -- Returns 'text'
/// ```
///
/// ### **NUMERIC**
/// - Can store values using all five storage classes.
/// - Text data is converted to INTEGER or REAL (in that order of preference) if it is a well-formed integer or real literal.
/// - If the text represents an integer too large for a 64-bit signed integer, it is converted to REAL.
/// - If the text is not a well-formed literal, it is stored as TEXT.
/// - Hexadecimal integer literals are stored as TEXT for historical compatibility.
/// - Floating-point values that can be exactly represented as integers are converted to integers.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col NUMERIC);
/// INSERT INTO example (col) VALUES ('3.0e+5'); -- Stored as 300000 (integer)
/// SELECT typeof(col) FROM example; -- Returns 'integer'
/// ```
///
/// ### **INTEGER**
/// - Behaves like NUMERIC affinity but differs in `CAST` expressions.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col INTEGER);
/// INSERT INTO example (col) VALUES (4.0); -- Stored as 4 (integer)
/// SELECT typeof(col) FROM example; -- Returns 'integer'
/// ```
///
/// ### **REAL**
/// - Similar to NUMERIC affinity but forces integer values into floating-point representation.
/// - **Optimization:** Small floating-point values with no fractional component may be stored as integers on disk to save space. This is invisible at the SQL level.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col REAL);
/// INSERT INTO example (col) VALUES (4); -- Stored as 4.0 (real)
/// SELECT typeof(col) FROM example; -- Returns 'real'
/// ```
///
/// ### **BLOB**
/// - Does not prefer any storage class.
/// - No coercion is performed between storage classes.
/// - **Example:**
/// ```sql
/// CREATE TABLE example (col BLOB);
/// INSERT INTO example (col) VALUES (x'1234'); -- Stored as a binary blob
/// SELECT typeof(col) FROM example; -- Returns 'blob'
/// ```
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Affinity {
Blob = 0,
Text = 1,
Numeric = 2,
Integer = 3,
Real = 4,
}
pub const SQLITE_AFF_NONE: char = 'A'; // Historically called NONE, but it's the same as BLOB
pub const SQLITE_AFF_TEXT: char = 'B';
pub const SQLITE_AFF_NUMERIC: char = 'C';
pub const SQLITE_AFF_INTEGER: char = 'D';
pub const SQLITE_AFF_REAL: char = 'E';
impl Affinity {
/// This is meant to be used in opcodes like Eq, which state:
///
/// "The SQLITE_AFF_MASK portion of P5 must be an affinity character - SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth.
/// An attempt is made to coerce both inputs according to this affinity before the comparison is made.
/// If the SQLITE_AFF_MASK is 0x00, then numeric affinity is used.
/// Note that the affinity conversions are stored back into the input registers P1 and P3.
/// So this opcode can cause persistent changes to registers P1 and P3.""
pub fn aff_mask(&self) -> char {
match self {
Affinity::Integer => SQLITE_AFF_INTEGER,
Affinity::Text => SQLITE_AFF_TEXT,
Affinity::Blob => SQLITE_AFF_NONE,
Affinity::Real => SQLITE_AFF_REAL,
Affinity::Numeric => SQLITE_AFF_NUMERIC,
}
}
pub fn from_char(char: char) -> Self {
match char {
SQLITE_AFF_INTEGER => Affinity::Integer,
SQLITE_AFF_TEXT => Affinity::Text,
SQLITE_AFF_NONE => Affinity::Blob,
SQLITE_AFF_REAL => Affinity::Real,
SQLITE_AFF_NUMERIC => Affinity::Numeric,
_ => Affinity::Blob,
}
}
pub fn as_char_code(&self) -> u8 {
self.aff_mask() as u8
}
pub fn from_char_code(code: u8) -> Self {
Self::from_char(code as char)
}
pub fn is_numeric(&self) -> bool {
matches!(self, Affinity::Integer | Affinity::Real | Affinity::Numeric)
}
pub fn has_affinity(&self) -> bool {
!matches!(self, Affinity::Blob)
}
/// 3.1. Determination Of Column Affinity
/// For tables not declared as STRICT, the affinity of a column is determined by the declared type of the column, according to the following rules in the order shown:
///
/// If the declared type contains the string "INT" then it is assigned INTEGER affinity.
///
/// If the declared type of the column contains any of the strings "CHAR", "CLOB", or "TEXT" then that column has TEXT affinity. Notice that the type VARCHAR contains the string "CHAR" and is thus assigned TEXT affinity.
///
/// If the declared type for a column contains the string "BLOB" or if no type is specified then the column has affinity BLOB.
///
/// If the declared type for a column contains any of the strings "REAL", "FLOA", or "DOUB" then the column has REAL affinity.
///
/// Otherwise, the affinity is NUMERIC.
///
/// Note that the order of the rules for determining column affinity is important. A column whose declared type is "CHARINT" will match both rules 1 and 2 but the first rule takes precedence and so the column affinity will be INTEGER.
#[expect(clippy::self_named_constructors)]
pub fn affinity(datatype: &str) -> Self {
let datatype = datatype.to_ascii_uppercase();
// Rule 1: INT -> INTEGER affinity
if datatype.contains("INT") {
return Affinity::Integer;
}
// Rule 2: CHAR/CLOB/TEXT -> TEXT affinity
if datatype.contains("CHAR") || datatype.contains("CLOB") || datatype.contains("TEXT") {
return Affinity::Text;
}
// Rule 3: BLOB or empty -> BLOB affinity (historically called NONE)
if datatype.contains("BLOB") || datatype.is_empty() || datatype.contains("ANY") {
return Affinity::Blob;
}
// Rule 4: REAL/FLOA/DOUB -> REAL affinity
if datatype.contains("REAL") || datatype.contains("FLOA") || datatype.contains("DOUB") {
return Affinity::Real;
}
// Rule 5: Otherwise -> NUMERIC affinity
Affinity::Numeric
}
pub fn convert<'a>(&self, val: &'a impl AsValueRef) -> Option<Either<ValueRef<'a>, Value>> {
let val = val.as_value_ref();
let is_text = matches!(val, ValueRef::Text(_));
// Apply affinity conversions
match self {
Affinity::Numeric | Affinity::Integer => is_text
.then(|| apply_numeric_affinity(val, false))
.flatten()
.map(Either::Left),
Affinity::Text => {
if is_text {
is_numeric_value(val)
.then(|| stringify_register(val))
.flatten()
.map(Either::Right)
} else {
None
}
}
Affinity::Real => {
let mut left = is_text
.then(|| apply_numeric_affinity(val, false))
.flatten();
if let ValueRef::Integer(i) = left.unwrap_or(val) {
left = Some(ValueRef::Float(i as f64));
}
left.map(Either::Left)
}
Affinity::Blob => None, // Do nothing for blob affinity.
}
}
/// Return TRUE if the given expression is a constant which would be
/// unchanged by OP_Affinity with the affinity given in the second
/// argument.
///
/// This routine is used to determine if the OP_Affinity operation
/// can be omitted. When in doubt return FALSE. A false negative
/// is harmless. A false positive, however, can result in the wrong
/// answer.
///
/// reference https://github.com/sqlite/sqlite/blob/master/src/expr.c#L3000
pub fn expr_needs_no_affinity_change(&self, expr: &Expr) -> bool {
if !self.has_affinity() {
return true;
}
// TODO: check for unary minus in the expr, as it may be an additional optimization.
// This involves mostly likely walking the expression
match expr {
Expr::Literal(literal) => match literal {
Literal::Numeric(_) => self.is_numeric(),
Literal::String(_) => matches!(self, Affinity::Text),
Literal::Blob(_) => true,
_ => false,
},
Expr::Column {
is_rowid_alias: true,
..
} => self.is_numeric(),
_ => false,
}
}
}
#[derive(Debug, PartialEq)]
pub enum NumericParseResult {
NotNumeric, // not a valid number
PureInteger, // pure integer (entire string)
HasDecimalOrExp, // has decimal point or exponent (entire string)
ValidPrefixOnly, // valid prefix but not entire string
}
#[derive(Debug)]
pub enum ParsedNumber {
None,
Integer(i64),
Float(f64),
}
impl ParsedNumber {
fn as_integer(&self) -> Option<i64> {
match self {
ParsedNumber::Integer(i) => Some(*i),
_ => None,
}
}
fn as_float(&self) -> Option<f64> {
match self {
ParsedNumber::Float(f) => Some(*f),
_ => None,
}
}
}
pub fn try_for_float(text: &str) -> (NumericParseResult, ParsedNumber) {
let bytes = text.as_bytes();
if bytes.is_empty() {
return (NumericParseResult::NotNumeric, ParsedNumber::None);
}
let mut pos = 0;
let len = bytes.len();
while pos < len && is_space(bytes[pos]) {
pos += 1;
}
if pos >= len {
return (NumericParseResult::NotNumeric, ParsedNumber::None);
}
let mut sign = 1i64;
if bytes[pos] == b'-' {
sign = -1;
pos += 1;
} else if bytes[pos] == b'+' {
pos += 1;
}
if pos >= len {
return (NumericParseResult::NotNumeric, ParsedNumber::None);
}
let mut significand = 0u64;
let mut decimal_adjust = 0i32;
let mut has_digits = false;
// Parse digits before decimal point
while pos < len && bytes[pos].is_ascii_digit() {
has_digits = true;
let digit = (bytes[pos] - b'0') as u64;
if significand <= (u64::MAX - 9) / 10 {
significand = significand * 10 + digit;
} else {
// Skip overflow digits but adjust exponent
decimal_adjust += 1;
}
pos += 1;
}
let mut has_decimal = false;
let mut has_exponent = false;
// Check for decimal point
if pos < len && bytes[pos] == b'.' {
has_decimal = true;
pos += 1;
// Parse fractional digits
while pos < len && bytes[pos].is_ascii_digit() {
has_digits = true;
let digit = (bytes[pos] - b'0') as u64;
if significand <= (u64::MAX - 9) / 10 {
significand = significand * 10 + digit;
decimal_adjust -= 1;
}
pos += 1;
}
}
if !has_digits {
return (NumericParseResult::NotNumeric, ParsedNumber::None);
}
// Check for exponent
let mut exponent = 0i32;
if pos < len && (bytes[pos] == b'e' || bytes[pos] == b'E') {
has_exponent = true;
pos += 1;
if pos >= len {
// Incomplete exponent, but we have valid digits before
return create_result_from_significand(
significand,
sign,
decimal_adjust,
has_decimal,
has_exponent,
NumericParseResult::ValidPrefixOnly,
);
}
let mut exp_sign = 1i32;
if bytes[pos] == b'-' {
exp_sign = -1;
pos += 1;
} else if bytes[pos] == b'+' {
pos += 1;
}
if pos >= len || !bytes[pos].is_ascii_digit() {
// Incomplete exponent
return create_result_from_significand(
significand,
sign,
decimal_adjust,
has_decimal,
false,
NumericParseResult::ValidPrefixOnly,
);
}
// Parse exponent digits
while pos < len && bytes[pos].is_ascii_digit() {
let digit = (bytes[pos] - b'0') as i32;
if exponent < 10000 {
exponent = exponent * 10 + digit;
} else {
exponent = 10000; // Cap at large value
}
pos += 1;
}
exponent *= exp_sign;
}
// Skip trailing whitespace
while pos < len && is_space(bytes[pos]) {
pos += 1;
}
// Determine if we consumed the entire string
let consumed_all = pos >= len;
let final_exponent = decimal_adjust + exponent;
let parse_result = if !consumed_all {
NumericParseResult::ValidPrefixOnly
} else if has_decimal || has_exponent {
NumericParseResult::HasDecimalOrExp
} else {
NumericParseResult::PureInteger
};
create_result_from_significand(
significand,
sign,
final_exponent,
has_decimal,
has_exponent,
parse_result,
)
}
fn create_result_from_significand(
significand: u64,
sign: i64,
exponent: i32,
has_decimal: bool,
has_exponent: bool,
parse_result: NumericParseResult,
) -> (NumericParseResult, ParsedNumber) {
if significand == 0 {
match parse_result {
NumericParseResult::PureInteger => {
return (parse_result, ParsedNumber::Integer(0));
}
_ => {
return (parse_result, ParsedNumber::Float(0.0));
}
}
}
// For pure integers without exponent, try to return as integer
if !has_decimal && !has_exponent && exponent == 0 && significand <= i64::MAX as u64 {
let signed_val = (significand as i64).wrapping_mul(sign);
return (parse_result, ParsedNumber::Integer(signed_val));
}
// Convert to float
let mut result = significand as f64;
let mut exp = exponent;
match exp.cmp(&0) {
std::cmp::Ordering::Greater => {
while exp >= 100 {
result *= 1e100;
exp -= 100;
}
while exp >= 10 {
result *= 1e10;
exp -= 10;
}
while exp >= 1 {
result *= 10.0;
exp -= 1;
}
}
std::cmp::Ordering::Less => {
while exp <= -100 {
result *= 1e-100;
exp += 100;
}
while exp <= -10 {
result *= 1e-10;
exp += 10;
}
while exp <= -1 {
result *= 0.1;
exp += 1;
}
}
std::cmp::Ordering::Equal => {}
}
if sign < 0 {
result = -result;
}
(parse_result, ParsedNumber::Float(result))
}
pub fn is_space(byte: u8) -> bool {
matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | b'\x0c')
}
fn real_to_i64(r: f64) -> i64 {
if r < -9223372036854774784.0 {
i64::MIN
} else if r > 9223372036854774784.0 {
i64::MAX
} else {
r as i64
}
}
fn apply_integer_affinity(val: ValueRef) -> Option<ValueRef> {
let ValueRef::Float(f) = val else {
return None;
};
let ix = real_to_i64(f);
// Only convert if round-trip is exact and not at extreme values
if f == (ix as f64) && ix > i64::MIN && ix < i64::MAX {
Some(ValueRef::Integer(ix))
} else {
None
}
}
/// Try to convert a value into a numeric representation if we can
/// do so without loss of information. In other words, if the string
/// looks like a number, convert it into a number. If it does not
/// look like a number, leave it alone.
pub fn apply_numeric_affinity(val: ValueRef, try_for_int: bool) -> Option<ValueRef> {
let ValueRef::Text(text) = val else {
return None; // Only apply to text values
};
let text_str = text.as_str();
let (parse_result, parsed_value) = try_for_float(text_str);
// Only convert if we have a complete valid number (not just a prefix)
match parse_result {
NumericParseResult::NotNumeric | NumericParseResult::ValidPrefixOnly => {
None // Leave as text
}
NumericParseResult::PureInteger => {
if let Some(int_val) = parsed_value.as_integer() {
Some(ValueRef::Integer(int_val))
} else if let Some(float_val) = parsed_value.as_float() {
let res = ValueRef::Float(float_val);
if try_for_int {
apply_integer_affinity(res)
} else {
Some(res)
}
} else {
None
}
}
NumericParseResult::HasDecimalOrExp => {
if let Some(float_val) = parsed_value.as_float() {
let res = ValueRef::Float(float_val);
// If try_for_int is true, try to convert float to int if exact
if try_for_int {
apply_integer_affinity(res)
} else {
Some(res)
}
} else {
None
}
}
}
}
fn is_numeric_value(val: ValueRef) -> bool {
matches!(val, ValueRef::Integer(_) | ValueRef::Float(_))
}
fn stringify_register(val: ValueRef) -> Option<Value> {
match val {
ValueRef::Integer(i) => Some(Value::build_text(i.to_string())),
ValueRef::Float(f) => Some(Value::build_text(f.to_string())),
ValueRef::Text(_) | ValueRef::Null | ValueRef::Blob(_) => None,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_apply_numeric_affinity_partial_numbers() {
let val = Value::Text("123abc".into());
let res = apply_numeric_affinity(val.as_value_ref(), false);
assert!(res.is_none());
let val = Value::Text("-53093015420544-15062897".into());
let res = apply_numeric_affinity(val.as_value_ref(), false);
assert!(res.is_none());
let val = Value::Text("123.45xyz".into());
let res = apply_numeric_affinity(val.as_value_ref(), false);
assert!(res.is_none());
}
#[test]
fn test_apply_numeric_affinity_complete_numbers() {
let val = Value::Text("123".into());
let res = apply_numeric_affinity(val.as_value_ref(), false);
assert_eq!(res, Some(ValueRef::Integer(123)));
let val = Value::Text("123.45".into());
let res = apply_numeric_affinity(val.as_value_ref(), false);
assert_eq!(res, Some(ValueRef::Float(123.45)));
let val = Value::Text(" -456 ".into());
let res = apply_numeric_affinity(val.as_value_ref(), false);
assert_eq!(res, Some(ValueRef::Integer(-456)));
let val = Value::Text("0".into());
let res = apply_numeric_affinity(val.as_value_ref(), false);
assert_eq!(res, Some(ValueRef::Integer(0)));
}
}

615
core/vdbe/execute.rs
View File

@@ -13,14 +13,16 @@ use crate::storage::pager::{AtomicDbState, CreateBTreeFlags, DbState};
use crate::storage::sqlite3_ondisk::{read_varint_fast, DatabaseHeader, PageSize};
use crate::translate::collate::CollationSeq;
use crate::types::{
compare_immutable, compare_records_generic, Extendable, IOCompletions, ImmutableRecord,
SeekResult, Text,
compare_immutable, compare_records_generic, AsValueRef, Extendable, IOCompletions,
ImmutableRecord, SeekResult, Text,
};
use crate::util::{
normalize_ident, rewrite_column_references_if_needed, rewrite_fk_parent_cols_if_self_ref,
rewrite_fk_parent_table_if_needed, rewrite_inline_col_fk_target_if_needed,
};
use crate::vdbe::affinity::{apply_numeric_affinity, try_for_float, Affinity, ParsedNumber};
use crate::vdbe::insn::InsertFlags;
use crate::vdbe::value::ComparisonOp;
use crate::vdbe::{registers_to_ref_values, EndStatement, TxnCleanup};
use crate::vector::{vector32_sparse, vector_concat, vector_distance_jaccard, vector_slice};
use crate::{
@@ -38,6 +40,7 @@ use crate::{
translate::emitter::TransactionMode,
};
use crate::{get_cursor, CheckpointMode, Connection, DatabaseStorage, MvCursor};
use either::Either;
use std::any::Any;
use std::env::temp_dir;
use std::ops::DerefMut;
@@ -49,10 +52,7 @@ use turso_macros::match_ignore_ascii_case;
use crate::pseudo::PseudoCursor;
use crate::{
schema::Affinity,
storage::btree::{BTreeCursor, BTreeKey},
};
use crate::storage::btree::{BTreeCursor, BTreeKey};
use crate::{
storage::wal::CheckpointResult,
@@ -668,69 +668,6 @@ pub fn op_not_null(
Ok(InsnFunctionStepResult::Step)
}
#[derive(Debug, Clone, Copy, PartialEq)]
enum ComparisonOp {
Eq,
Ne,
Lt,
Le,
Gt,
Ge,
}
impl ComparisonOp {
fn compare(&self, lhs: &Value, rhs: &Value, collation: &CollationSeq) -> bool {
match (lhs, rhs) {
(Value::Text(lhs_text), Value::Text(rhs_text)) => {
let order = collation.compare_strings(lhs_text.as_str(), rhs_text.as_str());
match self {
ComparisonOp::Eq => order.is_eq(),
ComparisonOp::Ne => order.is_ne(),
ComparisonOp::Lt => order.is_lt(),
ComparisonOp::Le => order.is_le(),
ComparisonOp::Gt => order.is_gt(),
ComparisonOp::Ge => order.is_ge(),
}
}
(_, _) => match self {
ComparisonOp::Eq => *lhs == *rhs,
ComparisonOp::Ne => *lhs != *rhs,
ComparisonOp::Lt => *lhs < *rhs,
ComparisonOp::Le => *lhs <= *rhs,
ComparisonOp::Gt => *lhs > *rhs,
ComparisonOp::Ge => *lhs >= *rhs,
},
}
}
fn compare_integers(&self, lhs: &Value, rhs: &Value) -> bool {
match self {
ComparisonOp::Eq => lhs == rhs,
ComparisonOp::Ne => lhs != rhs,
ComparisonOp::Lt => lhs < rhs,
ComparisonOp::Le => lhs <= rhs,
ComparisonOp::Gt => lhs > rhs,
ComparisonOp::Ge => lhs >= rhs,
}
}
fn handle_nulls(&self, lhs: &Value, rhs: &Value, null_eq: bool, jump_if_null: bool) -> bool {
match self {
ComparisonOp::Eq => {
let both_null = lhs == rhs;
(null_eq && both_null) || (!null_eq && jump_if_null)
}
ComparisonOp::Ne => {
let at_least_one_null = lhs != rhs;
(null_eq && at_least_one_null) || (!null_eq && jump_if_null)
}
ComparisonOp::Lt | ComparisonOp::Le | ComparisonOp::Gt | ComparisonOp::Ge => {
jump_if_null
}
}
}
}
pub fn op_comparison(
program: &Program,
state: &mut ProgramState,
@@ -828,7 +765,7 @@ pub fn op_comparison(
assert!(target_pc.is_offset());
let nulleq = flags.has_nulleq();
let null_eq = flags.has_nulleq();
let jump_if_null = flags.has_jump_if_null();
let affinity = flags.get_affinity();
@@ -837,7 +774,7 @@ pub fn op_comparison(
// Fast path for integers
if matches!(lhs_value, Value::Integer(_)) && matches!(rhs_value, Value::Integer(_)) {
if op.compare_integers(lhs_value, rhs_value) {
if op.compare(lhs_value, rhs_value, collation) {
state.pc = target_pc.as_offset_int();
} else {
state.pc += 1;
@@ -847,7 +784,15 @@ pub fn op_comparison(
// Handle NULL values
if matches!(lhs_value, Value::Null) || matches!(rhs_value, Value::Null) {
if op.handle_nulls(lhs_value, rhs_value, nulleq, jump_if_null) {
let cmp_res = op.compare_nulls(lhs_value, rhs_value, null_eq);
let jump = match op {
ComparisonOp::Eq => cmp_res || (!null_eq && jump_if_null),
ComparisonOp::Ne => cmp_res || (!null_eq && jump_if_null),
ComparisonOp::Lt | ComparisonOp::Le | ComparisonOp::Gt | ComparisonOp::Ge => {
jump_if_null
}
};
if jump {
state.pc = target_pc.as_offset_int();
} else {
state.pc += 1;
@@ -855,96 +800,34 @@ pub fn op_comparison(
return Ok(InsnFunctionStepResult::Step);
}
let mut lhs_temp_reg = None;
let mut rhs_temp_reg = None;
let mut lhs_converted = false;
let mut rhs_converted = false;
// Apply affinity conversions
match affinity {
Affinity::Numeric | Affinity::Integer => {
let lhs_is_text = matches!(state.registers[lhs].get_value(), Value::Text(_));
let rhs_is_text = matches!(state.registers[rhs].get_value(), Value::Text(_));
if lhs_is_text || rhs_is_text {
if lhs_is_text {
lhs_temp_reg = Some(state.registers[lhs].clone());
lhs_converted = apply_numeric_affinity(lhs_temp_reg.as_mut().unwrap(), false);
}
if rhs_is_text {
rhs_temp_reg = Some(state.registers[rhs].clone());
rhs_converted = apply_numeric_affinity(rhs_temp_reg.as_mut().unwrap(), false);
}
}
}
Affinity::Text => {
let lhs_is_text = matches!(state.registers[lhs].get_value(), Value::Text(_));
let rhs_is_text = matches!(state.registers[rhs].get_value(), Value::Text(_));
if lhs_is_text || rhs_is_text {
if is_numeric_value(&state.registers[lhs]) {
lhs_temp_reg = Some(state.registers[lhs].clone());
lhs_converted = stringify_register(lhs_temp_reg.as_mut().unwrap());
}
if is_numeric_value(&state.registers[rhs]) {
rhs_temp_reg = Some(state.registers[rhs].clone());
rhs_converted = stringify_register(rhs_temp_reg.as_mut().unwrap());
}
}
}
Affinity::Real => {
if matches!(state.registers[lhs].get_value(), Value::Text(_)) {
lhs_temp_reg = Some(state.registers[lhs].clone());
lhs_converted = apply_numeric_affinity(lhs_temp_reg.as_mut().unwrap(), false);
}
if matches!(state.registers[rhs].get_value(), Value::Text(_)) {
rhs_temp_reg = Some(state.registers[rhs].clone());
rhs_converted = apply_numeric_affinity(rhs_temp_reg.as_mut().unwrap(), false);
}
if let Value::Integer(i) =
(lhs_temp_reg.as_ref().unwrap_or(&state.registers[lhs])).get_value()
{
lhs_temp_reg = Some(Register::Value(Value::Float(*i as f64)));
lhs_converted = true;
}
if let Value::Integer(i) = rhs_temp_reg
.as_ref()
.unwrap_or(&state.registers[rhs])
.get_value()
{
rhs_temp_reg = Some(Register::Value(Value::Float(*i as f64)));
rhs_converted = true;
}
}
Affinity::Blob => {} // Do nothing for blob affinity.
}
let (new_lhs, new_rhs) = (affinity.convert(lhs_value), affinity.convert(rhs_value));
let should_jump = op.compare(
lhs_temp_reg
new_lhs
.as_ref()
.unwrap_or(&state.registers[lhs])
.get_value(),
rhs_temp_reg
.map_or(Either::Left(lhs_value), Either::Right),
new_rhs
.as_ref()
.unwrap_or(&state.registers[rhs])
.get_value(),
&collation,
.map_or(Either::Left(rhs_value), Either::Right),
collation,
);
if lhs_converted {
state.registers[lhs] = lhs_temp_reg.unwrap();
}
if rhs_converted {
state.registers[rhs] = rhs_temp_reg.unwrap();
match (new_lhs, new_rhs) {
(Some(new_lhs), None) => {
state.registers[lhs] = Register::Value(new_lhs.as_value_ref().to_owned());
}
(None, Some(new_rhs)) => {
state.registers[rhs] = Register::Value(new_rhs.as_value_ref().to_owned());
}
(Some(new_lhs), Some(new_rhs)) => {
let (new_lhs, new_rhs) = (
new_lhs.as_value_ref().to_owned(),
new_rhs.as_value_ref().to_owned(),
);
state.registers[lhs] = Register::Value(new_lhs);
state.registers[rhs] = Register::Value(new_rhs);
}
(None, None) => {}
}
if should_jump {
@@ -3192,13 +3075,15 @@ pub fn seek_internal(
unreachable!("op_seek: record_source should be Unpacked for table-btree");
};
assert_eq!(num_regs, 1, "op_seek: num_regs should be 1 for table-btree");
let original_value = state.registers[start_reg].get_value().clone();
let original_value = state.registers[start_reg].get_value();
let mut temp_value = original_value.clone();
let conversion_successful = if matches!(temp_value, Value::Text(_)) {
let mut temp_reg = Register::Value(temp_value);
let converted = apply_numeric_affinity(&mut temp_reg, false);
temp_value = temp_reg.get_value().clone();
let new_val = apply_numeric_affinity(temp_value.as_value_ref(), false);
let converted = new_val.is_some();
if let Some(new_val) = new_val {
temp_value = new_val.to_owned();
}
converted
} else {
true // Non-text values don't need conversion
@@ -3452,7 +3337,7 @@ pub fn op_idx_ge(
let tie_breaker = get_tie_breaker_from_idx_comp_op(insn);
let ord = compare_records_generic(
&idx_record, // The serialized record from the index
&values, // The record built from registers
values, // The record built from registers
cursor.get_index_info(), // Sort order flags
0,
tie_breaker,
@@ -3520,7 +3405,7 @@ pub fn op_idx_le(
let tie_breaker = get_tie_breaker_from_idx_comp_op(insn);
let ord = compare_records_generic(
&idx_record,
&values,
values,
cursor.get_index_info(),
0,
tie_breaker,
@@ -3571,7 +3456,7 @@ pub fn op_idx_gt(
let tie_breaker = get_tie_breaker_from_idx_comp_op(insn);
let ord = compare_records_generic(
&idx_record,
&values,
values,
cursor.get_index_info(),
0,
tie_breaker,
@@ -3623,7 +3508,7 @@ pub fn op_idx_lt(
let tie_breaker = get_tie_breaker_from_idx_comp_op(insn);
let ord = compare_records_generic(
&idx_record,
&values,
values,
cursor.get_index_info(),
0,
tie_breaker,
@@ -9067,24 +8952,25 @@ fn execute_turso_version(version_integer: i64) -> String {
format!("{major}.{minor}.{release}")
}
pub fn extract_int_value(value: &Value) -> i64 {
pub fn extract_int_value<V: AsValueRef>(value: V) -> i64 {
let value = value.as_value_ref();
match value {
Value::Integer(i) => *i,
Value::Float(f) => {
ValueRef::Integer(i) => i,
ValueRef::Float(f) => {
// Use sqlite3RealToI64 equivalent
if *f < -9223372036854774784.0 {
if f < -9223372036854774784.0 {
i64::MIN
} else if *f > 9223372036854774784.0 {
} else if f > 9223372036854774784.0 {
i64::MAX
} else {
*f as i64
f as i64
}
}
Value::Text(t) => {
ValueRef::Text(t) => {
// Try to parse as integer, return 0 if failed
t.as_str().parse::<i64>().unwrap_or(0)
}
Value::Blob(b) => {
ValueRef::Blob(b) => {
// Try to parse blob as string then as integer
if let Ok(s) = std::str::from_utf8(b) {
s.parse::<i64>().unwrap_or(0)
@@ -9092,356 +8978,7 @@ pub fn extract_int_value(value: &Value) -> i64 {
0
}
}
Value::Null => 0,
}
}
#[derive(Debug, PartialEq)]
enum NumericParseResult {
NotNumeric, // not a valid number
PureInteger, // pure integer (entire string)
HasDecimalOrExp, // has decimal point or exponent (entire string)
ValidPrefixOnly, // valid prefix but not entire string
}
#[derive(Debug)]
enum ParsedNumber {
None,
Integer(i64),
Float(f64),
}
impl ParsedNumber {
fn as_integer(&self) -> Option<i64> {
match self {
ParsedNumber::Integer(i) => Some(*i),
_ => None,
}
}
fn as_float(&self) -> Option<f64> {
match self {
ParsedNumber::Float(f) => Some(*f),
_ => None,
}
}
}
fn try_for_float(text: &str) -> (NumericParseResult, ParsedNumber) {
let bytes = text.as_bytes();
if bytes.is_empty() {
return (NumericParseResult::NotNumeric, ParsedNumber::None);
}
let mut pos = 0;
let len = bytes.len();
while pos < len && is_space(bytes[pos]) {
pos += 1;
}
if pos >= len {
return (NumericParseResult::NotNumeric, ParsedNumber::None);
}
let start_pos = pos;
let mut sign = 1i64;
if bytes[pos] == b'-' {
sign = -1;
pos += 1;
} else if bytes[pos] == b'+' {
pos += 1;
}
if pos >= len {
return (NumericParseResult::NotNumeric, ParsedNumber::None);
}
let mut significand = 0u64;
let mut digit_count = 0;
let mut decimal_adjust = 0i32;
let mut has_digits = false;
// Parse digits before decimal point
while pos < len && bytes[pos].is_ascii_digit() {
has_digits = true;
let digit = (bytes[pos] - b'0') as u64;
if significand <= (u64::MAX - 9) / 10 {
significand = significand * 10 + digit;
digit_count += 1;
} else {
// Skip overflow digits but adjust exponent
decimal_adjust += 1;
}
pos += 1;
}
let mut has_decimal = false;
let mut has_exponent = false;
// Check for decimal point
if pos < len && bytes[pos] == b'.' {
has_decimal = true;
pos += 1;
// Parse fractional digits
while pos < len && bytes[pos].is_ascii_digit() {
has_digits = true;
let digit = (bytes[pos] - b'0') as u64;
if significand <= (u64::MAX - 9) / 10 {
significand = significand * 10 + digit;
digit_count += 1;
decimal_adjust -= 1;
}
pos += 1;
}
}
if !has_digits {
return (NumericParseResult::NotNumeric, ParsedNumber::None);
}
// Check for exponent
let mut exponent = 0i32;
if pos < len && (bytes[pos] == b'e' || bytes[pos] == b'E') {
has_exponent = true;
pos += 1;
if pos >= len {
// Incomplete exponent, but we have valid digits before
return create_result_from_significand(
significand,
sign,
decimal_adjust,
has_decimal,
has_exponent,
NumericParseResult::ValidPrefixOnly,
);
}
let mut exp_sign = 1i32;
if bytes[pos] == b'-' {
exp_sign = -1;
pos += 1;
} else if bytes[pos] == b'+' {
pos += 1;
}
if pos >= len || !bytes[pos].is_ascii_digit() {
// Incomplete exponent
return create_result_from_significand(
significand,
sign,
decimal_adjust,
has_decimal,
false,
NumericParseResult::ValidPrefixOnly,
);
}
// Parse exponent digits
while pos < len && bytes[pos].is_ascii_digit() {
let digit = (bytes[pos] - b'0') as i32;
if exponent < 10000 {
exponent = exponent * 10 + digit;
} else {
exponent = 10000; // Cap at large value
}
pos += 1;
}
exponent *= exp_sign;
}
// Skip trailing whitespace
while pos < len && is_space(bytes[pos]) {
pos += 1;
}
// Determine if we consumed the entire string
let consumed_all = pos >= len;
let final_exponent = decimal_adjust + exponent;
let parse_result = if !consumed_all {
NumericParseResult::ValidPrefixOnly
} else if has_decimal || has_exponent {
NumericParseResult::HasDecimalOrExp
} else {
NumericParseResult::PureInteger
};
create_result_from_significand(
significand,
sign,
final_exponent,
has_decimal,
has_exponent,
parse_result,
)
}
fn create_result_from_significand(
significand: u64,
sign: i64,
exponent: i32,
has_decimal: bool,
has_exponent: bool,
parse_result: NumericParseResult,
) -> (NumericParseResult, ParsedNumber) {
if significand == 0 {
match parse_result {
NumericParseResult::PureInteger => {
return (parse_result, ParsedNumber::Integer(0));
}
_ => {
return (parse_result, ParsedNumber::Float(0.0));
}
}
}
// For pure integers without exponent, try to return as integer
if !has_decimal && !has_exponent && exponent == 0 && significand <= i64::MAX as u64 {
let signed_val = (significand as i64).wrapping_mul(sign);
return (parse_result, ParsedNumber::Integer(signed_val));
}
// Convert to float
let mut result = significand as f64;
let mut exp = exponent;
match exp.cmp(&0) {
std::cmp::Ordering::Greater => {
while exp >= 100 {
result *= 1e100;
exp -= 100;
}
while exp >= 10 {
result *= 1e10;
exp -= 10;
}
while exp >= 1 {
result *= 10.0;
exp -= 1;
}
}
std::cmp::Ordering::Less => {
while exp <= -100 {
result *= 1e-100;
exp += 100;
}
while exp <= -10 {
result *= 1e-10;
exp += 10;
}
while exp <= -1 {
result *= 0.1;
exp += 1;
}
}
std::cmp::Ordering::Equal => {}
}
if sign < 0 {
result = -result;
}
(parse_result, ParsedNumber::Float(result))
}
pub fn is_space(byte: u8) -> bool {
matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | b'\x0c')
}
fn real_to_i64(r: f64) -> i64 {
if r < -9223372036854774784.0 {
i64::MIN
} else if r > 9223372036854774784.0 {
i64::MAX
} else {
r as i64
}
}
fn apply_integer_affinity(register: &mut Register) -> bool {
let Register::Value(Value::Float(f)) = register else {
return false;
};
let ix = real_to_i64(*f);
// Only convert if round-trip is exact and not at extreme values
if *f == (ix as f64) && ix > i64::MIN && ix < i64::MAX {
*register = Register::Value(Value::Integer(ix));
true
} else {
false
}
}
/// Try to convert a value into a numeric representation if we can
/// do so without loss of information. In other words, if the string
/// looks like a number, convert it into a number. If it does not
/// look like a number, leave it alone.
pub fn apply_numeric_affinity(register: &mut Register, try_for_int: bool) -> bool {
let Register::Value(Value::Text(text)) = register else {
return false; // Only apply to text values
};
let text_str = text.as_str();
let (parse_result, parsed_value) = try_for_float(text_str);
// Only convert if we have a complete valid number (not just a prefix)
match parse_result {
NumericParseResult::NotNumeric | NumericParseResult::ValidPrefixOnly => {
false // Leave as text
}
NumericParseResult::PureInteger => {
if let Some(int_val) = parsed_value.as_integer() {
*register = Register::Value(Value::Integer(int_val));
true
} else if let Some(float_val) = parsed_value.as_float() {
*register = Register::Value(Value::Float(float_val));
if try_for_int {
apply_integer_affinity(register);
}
true
} else {
false
}
}
NumericParseResult::HasDecimalOrExp => {
if let Some(float_val) = parsed_value.as_float() {
*register = Register::Value(Value::Float(float_val));
// If try_for_int is true, try to convert float to int if exact
if try_for_int {
apply_integer_affinity(register);
}
true
} else {
false
}
}
}
}
fn is_numeric_value(reg: &Register) -> bool {
matches!(reg.get_value(), Value::Integer(_) | Value::Float(_))
}
fn stringify_register(reg: &mut Register) -> bool {
match reg.get_value() {
Value::Integer(i) => {
*reg = Register::Value(Value::build_text(i.to_string()));
true
}
Value::Float(f) => {
*reg = Register::Value(Value::build_text(f.to_string()));
true
}
Value::Text(_) | Value::Null | Value::Blob(_) => false,
ValueRef::Null => 0,
}
}
@@ -9567,40 +9104,6 @@ fn get_schema_cookie(
mod tests {
use super::*;
#[test]
fn test_apply_numeric_affinity_partial_numbers() {
let mut reg = Register::Value(Value::Text("123abc".into()));
assert!(!apply_numeric_affinity(&mut reg, false));
assert!(matches!(reg, Register::Value(Value::Text(_))));
let mut reg = Register::Value(Value::Text("-53093015420544-15062897".into()));
assert!(!apply_numeric_affinity(&mut reg, false));
assert!(matches!(reg, Register::Value(Value::Text(_))));
let mut reg = Register::Value(Value::Text("123.45xyz".into()));
assert!(!apply_numeric_affinity(&mut reg, false));
assert!(matches!(reg, Register::Value(Value::Text(_))));
}
#[test]
fn test_apply_numeric_affinity_complete_numbers() {
let mut reg = Register::Value(Value::Text("123".into()));
assert!(apply_numeric_affinity(&mut reg, false));
assert_eq!(*reg.get_value(), Value::Integer(123));
let mut reg = Register::Value(Value::Text("123.45".into()));
assert!(apply_numeric_affinity(&mut reg, false));
assert_eq!(*reg.get_value(), Value::Float(123.45));
let mut reg = Register::Value(Value::Text(" -456 ".into()));
assert!(apply_numeric_affinity(&mut reg, false));
assert_eq!(*reg.get_value(), Value::Integer(-456));
let mut reg = Register::Value(Value::Text("0".into()));
assert!(apply_numeric_affinity(&mut reg, false));
assert_eq!(*reg.get_value(), Value::Integer(0));
}
#[test]
fn test_execute_sqlite_version() {
let version_integer = 3046001;

3
core/vdbe/insn.rs
View File

@@ -5,10 +5,11 @@ use std::{
use super::{execute, AggFunc, BranchOffset, CursorID, FuncCtx, InsnFunction, PageIdx};
use crate::{
schema::{Affinity, BTreeTable, Column, Index},
schema::{BTreeTable, Column, Index},
storage::{pager::CreateBTreeFlags, wal::CheckpointMode},
translate::{collate::CollationSeq, emitter::TransactionMode},
types::KeyInfo,
vdbe::affinity::Affinity,
Value,
};
use strum::EnumCount;

10
core/vdbe/mod.rs
View File

@@ -17,6 +17,7 @@
//!
//! https://www.sqlite.org/opcode.html
pub mod affinity;
pub mod builder;
pub mod execute;
pub mod explain;
@@ -1092,11 +1093,10 @@ fn make_record(registers: &[Register], start_reg: &usize, count: &usize) -> Immu
ImmutableRecord::from_registers(regs, regs.len())
}
pub fn registers_to_ref_values<'a>(registers: &'a [Register]) -> Vec<ValueRef<'a>> {
registers
.iter()
.map(|reg| reg.get_value().as_ref())
.collect()
pub fn registers_to_ref_values<'a>(
registers: &'a [Register],
) -> impl ExactSizeIterator<Item = ValueRef<'a>> {
registers.iter().map(|reg| reg.get_value().as_ref())
}
#[instrument(skip(program), level = Level::DEBUG)]

71
core/vdbe/value.rs
View File

@@ -5,8 +5,10 @@ use regex::{Regex, RegexBuilder};
use crate::{
function::MathFunc,
numeric::{NullableInteger, Numeric},
schema::{affinity, Affinity},
LimboError, Result, Value,
translate::collate::CollationSeq,
types::{compare_immutable_single, AsValueRef, SeekOp},
vdbe::affinity::Affinity,
LimboError, Result, Value, ValueRef,
};
mod cmath {
@@ -35,6 +37,69 @@ mod cmath {
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub(super) enum ComparisonOp {
Eq,
Ne,
Lt,
Le,
Gt,
Ge,
}
impl ComparisonOp {
pub(super) fn compare<V1: AsValueRef, V2: AsValueRef>(
&self,
lhs: V1,
rhs: V2,
collation: CollationSeq,
) -> bool {
let order = compare_immutable_single(lhs, rhs, collation);
match self {
ComparisonOp::Eq => order.is_eq(),
ComparisonOp::Ne => order.is_ne(),
ComparisonOp::Lt => order.is_lt(),
ComparisonOp::Le => order.is_le(),
ComparisonOp::Gt => order.is_gt(),
ComparisonOp::Ge => order.is_ge(),
}
}
pub(super) fn compare_nulls<V1: AsValueRef, V2: AsValueRef>(
&self,
lhs: V1,
rhs: V2,
null_eq: bool,
) -> bool {
let (lhs, rhs) = (lhs.as_value_ref(), rhs.as_value_ref());
assert!(matches!(lhs, ValueRef::Null) || matches!(rhs, ValueRef::Null));
match self {
ComparisonOp::Eq => {
let both_null = lhs == rhs;
null_eq && both_null
}
ComparisonOp::Ne => {
let at_least_one_null = lhs != rhs;
null_eq && at_least_one_null
}
ComparisonOp::Lt | ComparisonOp::Le | ComparisonOp::Gt | ComparisonOp::Ge => false,
}
}
}
impl From<SeekOp> for ComparisonOp {
fn from(value: SeekOp) -> Self {
match value {
SeekOp::GE { eq_only: true } | SeekOp::LE { eq_only: true } => ComparisonOp::Eq,
SeekOp::GE { eq_only: false } => ComparisonOp::Ge,
SeekOp::GT => ComparisonOp::Gt,
SeekOp::LE { eq_only: false } => ComparisonOp::Le,
SeekOp::LT => ComparisonOp::Lt,
}
}
}
enum TrimType {
All,
Left,
@@ -548,7 +613,7 @@ impl Value {
if matches!(self, Value::Null) {
return Value::Null;
}
match affinity(datatype) {
match Affinity::affinity(datatype) {
// NONE Casting a value to a type-name with no affinity causes the value to be converted into a BLOB. Casting to a BLOB consists of first casting the value to TEXT in the encoding of the database connection, then interpreting the resulting byte sequence as a BLOB instead of as TEXT.
// Historically called NONE, but it's the same as BLOB
Affinity::Blob => {