aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-02-03 15:24:22 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'tursodatabase:main' into fix-and-predicate

Browse Source
This commit is contained in:
Nikita Sivukhin
2025-02-15 22:57:56 +04:00
committed by GitHub
parent becc58565d aea8b416bc
commit db7544fe7a
8 changed files with 247 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.cargo/config.toml Normal file
View File

@@ -0,0 +1,2 @@
[env]
LIBSQLITE3_FLAGS = "-DSQLITE_ENABLE_MATH_FUNCTIONS" # necessary for rusqlite dependency in order to bundle SQLite with math functions included

2
.github/workflows/rust.yml vendored
View File

@@ -27,6 +27,8 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: Swatinem/rust-cache@v2
with:
prefix-key: "v1-rust" # can be updated if we need to reset caches due to non-trivial change in the dependencies (for example, custom env var were set for single workspace project)
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:

34
core/mvcc/database/mod.rs
View File

@@ -318,8 +318,9 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
.ok_or(DatabaseError::NoSuchTransactionID(tx_id))?;
let tx = tx.value().read().unwrap();
assert_eq!(tx.state, TransactionState::Active);
let version_is_visible_to_current_tx = is_version_visible(&self.txs, &tx, rv);
if !version_is_visible_to_current_tx {
// A transaction cannot delete a version that it cannot see,
// nor can it conflict with it.
if !rv.is_visible_to(&tx, &self.txs) {
continue;
}
if is_write_write_conflict(&self.txs, &tx, rv) {
@@ -366,11 +367,14 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
assert_eq!(tx.state, TransactionState::Active);
if let Some(row_versions) = self.rows.get(&id) {
let row_versions = row_versions.value().read().unwrap();
for rv in row_versions.iter().rev() {
if is_version_visible(&self.txs, &tx, rv) {
tx.insert_to_read_set(id);
return Ok(Some(rv.row.clone()));
}
if let Some(rv) = row_versions
.iter()
.rev()
.filter(|rv| rv.is_visible_to(&tx, &self.txs))
.next()
{
tx.insert_to_read_set(id);
return Ok(Some(rv.row.clone()));
}
}
Ok(None)
@@ -534,7 +538,7 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
}
if let Some(TxTimestampOrID::TxID(id)) = row_version.end {
if id == tx_id {
// New version is valid UNTIL committing transaction's end timestamp
// Old version is valid UNTIL committing transaction's end timestamp
// See diagram on page 299: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf
row_version.end = Some(TxTimestampOrID::Timestamp(end_ts));
self.insert_version_raw(
@@ -763,12 +767,14 @@ pub(crate) fn is_write_write_conflict<T>(
}
}
pub(crate) fn is_version_visible<T>(
txs: &SkipMap<TxID, RwLock<Transaction>>,
tx: &Transaction,
rv: &RowVersion<T>,
) -> bool {
is_begin_visible(txs, tx, rv) && is_end_visible(txs, tx, rv)
impl<T> RowVersion<T> {
pub fn is_visible_to(
&self,
tx: &Transaction,
txs: &SkipMap<TxID, RwLock<Transaction>>,
) -> bool {
is_begin_visible(txs, tx, self) && is_end_visible(txs, tx, self)
}
}
fn is_begin_visible<T>(

2
core/mvcc/database/tests.rs
View File

@@ -699,7 +699,7 @@ fn test_snapshot_isolation_tx_visible1() {
},
};
tracing::debug!("Testing visibility of {row_version:?}");
is_version_visible(&txs, &current_tx, &row_version)
row_version.is_visible_to(&current_tx, &txs)
};
// begin visible: transaction committed with ts < current_tx.begin_ts

23
core/translate/expr.rs
View File

@@ -1849,15 +1849,24 @@ pub fn translate_expr(
MathFuncArity::Binary => {
let args = expect_arguments_exact!(args, 2, math_func);
let reg1 = program.alloc_register();
let _ =
translate_expr(program, referenced_tables, &args[0], reg1, resolver)?;
let reg2 = program.alloc_register();
let _ =
translate_expr(program, referenced_tables, &args[1], reg2, resolver)?;
let start_reg = program.alloc_registers(2);
let _ = translate_expr(
program,
referenced_tables,
&args[0],
start_reg,
resolver,
)?;
let _ = translate_expr(
program,
referenced_tables,
&args[1],
start_reg + 1,
resolver,
)?;
program.emit_insn(Insn::Function {
constant_mask: 0,
start_reg: target_register + 1,
start_reg,
dest: target_register,
func: func_ctx,
});

82
core/vdbe/mod.rs
View File

@@ -1399,8 +1399,8 @@ impl Program {
let record_from_regs: Record =
make_owned_record(&state.registers, start_reg, num_regs);
if let Some(ref idx_record) = *cursor.record()? {
// omit the rowid from the idx_record, which is the last value
if idx_record.get_values()[..idx_record.len() - 1]
// Compare against the same number of values
if idx_record.get_values()[..record_from_regs.len()]
>= record_from_regs.get_values()[..]
{
state.pc = target_pc.to_offset_int();
@@ -1423,8 +1423,8 @@ impl Program {
let record_from_regs: Record =
make_owned_record(&state.registers, start_reg, num_regs);
if let Some(ref idx_record) = *cursor.record()? {
// omit the rowid from the idx_record, which is the last value
if idx_record.get_values()[..idx_record.len() - 1]
// Compare against the same number of values
if idx_record.get_values()[..record_from_regs.len()]
<= record_from_regs.get_values()[..]
{
state.pc = target_pc.to_offset_int();
@@ -1447,8 +1447,8 @@ impl Program {
let record_from_regs: Record =
make_owned_record(&state.registers, start_reg, num_regs);
if let Some(ref idx_record) = *cursor.record()? {
// omit the rowid from the idx_record, which is the last value
if idx_record.get_values()[..idx_record.len() - 1]
// Compare against the same number of values
if idx_record.get_values()[..record_from_regs.len()]
> record_from_regs.get_values()[..]
{
state.pc = target_pc.to_offset_int();
@@ -1471,8 +1471,8 @@ impl Program {
let record_from_regs: Record =
make_owned_record(&state.registers, start_reg, num_regs);
if let Some(ref idx_record) = *cursor.record()? {
// omit the rowid from the idx_record, which is the last value
if idx_record.get_values()[..idx_record.len() - 1]
// Compare against the same number of values
if idx_record.get_values()[..record_from_regs.len()]
< record_from_regs.get_values()[..]
{
state.pc = target_pc.to_offset_int();
@@ -2250,7 +2250,11 @@ impl Program {
ScalarFunc::Substr | ScalarFunc::Substring => {
let str_value = &state.registers[*start_reg];
let start_value = &state.registers[*start_reg + 1];
let length_value = &state.registers[*start_reg + 2];
let length_value = if func.arg_count == 3 {
Some(&state.registers[*start_reg + 2])
} else {
None
};
let result = exec_substring(str_value, start_value, length_value);
state.registers[*dest] = result;
}
@@ -3286,39 +3290,31 @@ fn exec_nullif(first_value: &OwnedValue, second_value: &OwnedValue) -> OwnedValu
fn exec_substring(
str_value: &OwnedValue,
start_value: &OwnedValue,
length_value: &OwnedValue,
length_value: Option<&OwnedValue>,
) -> OwnedValue {
if let (OwnedValue::Text(str), OwnedValue::Integer(start), OwnedValue::Integer(length)) =
(str_value, start_value, length_value)
{
let start = *start as usize;
let str_len = str.as_str().len();
if let (OwnedValue::Text(str), OwnedValue::Integer(start)) = (str_value, start_value) {
let str_len = str.as_str().len() as i64;
if start > str_len {
return OwnedValue::build_text("");
}
let start_idx = start - 1;
let end = if *length != -1 {
start_idx + *length as usize
// The left-most character of X is number 1.
// If Y is negative then the first character of the substring is found by counting from the right rather than the left.
let first_position = if *start < 0 {
str_len.saturating_sub((*start).abs())
} else {
str_len
*start - 1
};
let substring = &str.as_str()[start_idx..end.min(str_len)];
OwnedValue::build_text(substring)
} else if let (OwnedValue::Text(str), OwnedValue::Integer(start)) = (str_value, start_value) {
let start = *start as usize;
let str_len = str.as_str().len();
if start > str_len {
return OwnedValue::build_text("");
}
let start_idx = start - 1;
let substring = &str.as_str()[start_idx..str_len];
OwnedValue::build_text(substring)
// If Z is negative then the abs(Z) characters preceding the Y-th character are returned.
let last_position = match length_value {
Some(OwnedValue::Integer(length)) => first_position + *length,
_ => str_len,
};
let (start, end) = if first_position <= last_position {
(first_position, last_position)
} else {
(last_position, first_position)
};
OwnedValue::build_text(
&str.as_str()[start.clamp(-0, str_len) as usize..end.clamp(0, str_len) as usize],
)
} else {
OwnedValue::Null
}
@@ -4393,7 +4389,7 @@ mod tests {
let length_value = OwnedValue::Integer(3);
let expected_val = OwnedValue::build_text("lim");
assert_eq!(
exec_substring(&str_value, &start_value, &length_value),
exec_substring(&str_value, &start_value, Some(&length_value)),
expected_val
);
@@ -4402,7 +4398,7 @@ mod tests {
let length_value = OwnedValue::Integer(10);
let expected_val = OwnedValue::build_text("limbo");
assert_eq!(
exec_substring(&str_value, &start_value, &length_value),
exec_substring(&str_value, &start_value, Some(&length_value)),
expected_val
);
@@ -4411,7 +4407,7 @@ mod tests {
let length_value = OwnedValue::Integer(3);
let expected_val = OwnedValue::build_text("");
assert_eq!(
exec_substring(&str_value, &start_value, &length_value),
exec_substring(&str_value, &start_value, Some(&length_value)),
expected_val
);
@@ -4420,7 +4416,7 @@ mod tests {
let length_value = OwnedValue::Null;
let expected_val = OwnedValue::build_text("mbo");
assert_eq!(
exec_substring(&str_value, &start_value, &length_value),
exec_substring(&str_value, &start_value, Some(&length_value)),
expected_val
);
@@ -4429,7 +4425,7 @@ mod tests {
let length_value = OwnedValue::Null;
let expected_val = OwnedValue::build_text("");
assert_eq!(
exec_substring(&str_value, &start_value, &length_value),
exec_substring(&str_value, &start_value, Some(&length_value)),
expected_val
);
}

15
testing/scalar-functions.test
View File

@@ -535,6 +535,21 @@ do_execsql_test substr-2-args {
SELECT substr('limbo', 3);
} {mbo}
do_execsql_test substr-cases {
SELECT substr('limbo', 0);
SELECT substr('limbo', 0, 3);
SELECT substr('limbo', -2);
SELECT substr('limbo', -2, 1);
SELECT substr('limbo', -10, 7);
SELECT substr('limbo', 10, -7);
} {limbo
li
bo
b
li
mbo
}
do_execsql_test substring-3-args {
SELECT substring('limbo', 1, 3);
} {lim}

153
tests/integration/fuzz/mod.rs
View File

@@ -167,7 +167,7 @@ mod tests {
}
#[test]
pub fn logical_expression_fuzz_ex1() {
pub fn fuzz_ex() {
let _ = env_logger::try_init();
let db = TempDatabase::new_empty();
let limbo_conn = db.connect_limbo();
@@ -182,6 +182,7 @@ mod tests {
"SELECT CASE ( NULL < NULL ) WHEN ( 0 ) THEN ( NULL ) ELSE ( 2.0 ) END;",
"SELECT (COALESCE(0, COALESCE(0, 0)));",
"SELECT CAST((1 > 0) AS INTEGER);",
"SELECT substr('ABC', -1)",
] {
let limbo = limbo_exec_rows(&db, &limbo_conn, query);
let sqlite = sqlite_exec_rows(&sqlite_conn, query);
@@ -193,6 +194,111 @@ mod tests {
}
}
#[test]
pub fn math_expression_fuzz_run() {
let _ = env_logger::try_init();
let g = GrammarGenerator::new();
let (expr, expr_builder) = g.create_handle();
let (bin_op, bin_op_builder) = g.create_handle();
let (scalar, scalar_builder) = g.create_handle();
let (paren, paren_builder) = g.create_handle();
paren_builder
.concat("")
.push_str("(")
.push(expr)
.push_str(")")
.build();
bin_op_builder
.concat(" ")
.push(expr)
.push(
g.create()
.choice()
.options_str(["+", "-", "/", "*"])
.build(),
)
.push(expr)
.build();
scalar_builder
.choice()
.option(
g.create()
.concat("")
.push(
g.create()
.choice()
.options_str([
"acos", "acosh", "asin", "asinh", "atan", "atanh", "ceil",
"ceiling", "cos", "cosh", "degrees", "exp", "floor", "ln", "log",
"log10", "log2", "radians", "sin", "sinh", "sqrt", "tan", "tanh",
"trunc",
])
.build(),
)
.push_str("(")
.push(expr)
.push_str(")")
.build(),
)
.option(
g.create()
.concat("")
.push(
g.create()
.choice()
.options_str(["atan2", "log", "mod", "pow", "power"])
.build(),
)
.push_str("(")
.push(g.create().concat("").push(expr).repeat(2..3, ", ").build())
.push_str(")")
.build(),
)
.build();
expr_builder
.choice()
.options_str(["-2.0", "-1.0", "0.0", "0.5", "1.0", "2.0"])
.option_w(bin_op, 10.0)
.option_w(paren, 10.0)
.option_w(scalar, 10.0)
.build();
let sql = g.create().concat(" ").push_str("SELECT").push(expr).build();
let db = TempDatabase::new_empty();
let limbo_conn = db.connect_limbo();
let sqlite_conn = rusqlite::Connection::open_in_memory().unwrap();
let (mut rng, seed) = rng_from_time();
log::info!("seed: {}", seed);
for _ in 0..1024 {
let query = g.generate(&mut rng, sql, 50);
log::info!("query: {}", query);
let limbo = limbo_exec_row(&limbo_conn, &query);
let sqlite = sqlite_exec_row(&sqlite_conn, &query);
match (&limbo[0], &sqlite[0]) {
// compare only finite results because some evaluations are not so stable around infinity
(rusqlite::types::Value::Real(limbo), rusqlite::types::Value::Real(sqlite))
if limbo.is_finite() && sqlite.is_finite() =>
{
assert!(
(limbo - sqlite).abs() < 1e-9
|| (limbo - sqlite) / (limbo.abs().max(sqlite.abs())) < 1e-9,
"query: {}, limbo: {:?}, sqlite: {:?}",
query,
limbo,
sqlite
)
}
_ => {}
}
}
}
#[test]
pub fn string_expression_fuzz_run() {
let _ = env_logger::try_init();
@@ -201,6 +307,20 @@ mod tests {
let (bin_op, bin_op_builder) = g.create_handle();
let (scalar, scalar_builder) = g.create_handle();
let (paren, paren_builder) = g.create_handle();
let (number, number_builder) = g.create_handle();
number_builder
.choice()
.option_symbol(rand_int(-5..10))
.option(
g.create()
.concat(" ")
.push(number)
.push(g.create().choice().options_str(["+", "-", "*"]).build())
.push(number)
.build(),
)
.build();
paren_builder
.concat("")
@@ -262,6 +382,37 @@ mod tests {
.push_str(")")
.build(),
)
.option(
g.create()
.concat("")
.push(g.create().choice().options_str(["replace"]).build())
.push_str("(")
.push(g.create().concat("").push(expr).repeat(3..4, ", ").build())
.push_str(")")
.build(),
)
.option(
g.create()
.concat("")
.push(
g.create()
.choice()
.options_str(["substr", "substring"])
.build(),
)
.push_str("(")
.push(expr)
.push_str(", ")
.push(
g.create()
.concat("")
.push(number)
.repeat(1..3, ", ")
.build(),
)
.push_str(")")
.build(),
)
.build();
expr_builder