aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-02-23 00:45:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'core/io: Make random generation deterministically simulated' from Pedro Muniz

Browse Source 
Depends on #3584 to use the most up-to-date implementation of
`ThreadRng`
- Add `fill_bytes` method to `IO`
- use `thread_rng` instead of `getrandom`, as `getrandom` is much slower
and `thread_rng` offers enough security
- modify `exec_randomblob`, `exec_random` and random_rowid generation to
use methods from IO for determinism
- modified simulator IO to implement `fill_bytes`
This the PRNG for sqlite if someone is curious. It is similar to
`thread_rng`:
```c
/* Initialize the state of the random number generator once,
  ** the first time this routine is called.
  */
  if( wsdPrng.s[0]==0 ){
    sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
    static const u32 chacha20_init[] = {
      0x61707865, 0x3320646e, 0x79622d32, 0x6b206574
    };
    memcpy(&wsdPrng.s[0], chacha20_init, 16);
    if( NEVER(pVfs==0) ){
      memset(&wsdPrng.s[4], 0, 44);
    }else{
      sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]);
    }
    wsdPrng.s[15] = wsdPrng.s[12];
    wsdPrng.s[12] = 0;
    wsdPrng.n = 0;
  }

  assert( N>0 );
  while( 1 /* exit by break */ ){
    if( N<=wsdPrng.n ){
      memcpy(zBuf, &wsdPrng.out[wsdPrng.n-N], N);
      wsdPrng.n -= N;
      break;
    }
    if( wsdPrng.n>0 ){
      memcpy(zBuf, wsdPrng.out, wsdPrng.n);
      N -= wsdPrng.n;
      zBuf += wsdPrng.n;
    }
    wsdPrng.s[12]++;
    chacha_block((u32*)wsdPrng.out, wsdPrng.s);
    wsdPrng.n = 64;
  }
  sqlite3_mutex_leave(mutex);
```

Reviewed-by: Pere Diaz Bou <pere-altea@homail.com>

Closes #3799
This commit is contained in:
Pekka Enberg
2025-10-22 09:10:36 +03:00
committed by GitHub
parent 9005458aa2 72baf48863
commit 1aad1b224a
11 changed files with 80 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@@ -4897,7 +4897,6 @@ dependencies = [
"crossbeam-skiplist",
"env_logger 0.11.7",
"fallible-iterator",
"getrandom 0.2.15",
"hex",
"intrusive-collections",
"io-uring",
@@ -4915,7 +4914,7 @@ dependencies = [
"pprof",
"quickcheck",
"quickcheck_macros",
"rand 0.8.5",
"rand 0.9.2",
"rand_chacha 0.9.0",
"regex",
"regex-syntax",

4
core/Cargo.toml
View File

@@ -52,14 +52,13 @@ cfg_block = "0.1.1"
fallible-iterator = { workspace = true }
hex = { workspace = true }
thiserror = { workspace = true }
getrandom = { version = "0.2.15" }
regex = { workspace = true }
regex-syntax = { workspace = true, default-features = false, features = [
"unicode",
] }
chrono = { workspace = true, default-features = false, features = ["clock"] }
julian_day_converter = "0.4.5"
rand = "0.8.5"
rand = { workspace = true }
libm = "0.2"
turso_macros = { workspace = true }
miette = { workspace = true }
@@ -103,7 +102,6 @@ rstest = "0.18.2"
rusqlite = { workspace = true, features = ["series"] }
quickcheck = { version = "1.0", default-features = false }
quickcheck_macros = { version = "1.0", default-features = false }
rand = "0.8.5" # Required for quickcheck
rand_chacha = { workspace = true }
env_logger = { workspace = true }
test-log = { version = "0.2.17", features = ["trace"] }

10
core/io/mod.rs
View File

@@ -3,6 +3,7 @@ use crate::storage::sqlite3_ondisk::WAL_FRAME_HEADER_SIZE;
use crate::{BufferPool, Result};
use bitflags::bitflags;
use cfg_block::cfg_block;
use rand::{Rng, RngCore};
use std::cell::RefCell;
use std::fmt;
use std::ptr::NonNull;
@@ -147,9 +148,12 @@ pub trait IO: Clock + Send + Sync {
}
fn generate_random_number(&self) -> i64 {
let mut buf = [0u8; 8];
getrandom::getrandom(&mut buf).unwrap();
i64::from_ne_bytes(buf)
rand::rng().random()
}
/// Fill `dest` with random data.
fn fill_bytes(&self, dest: &mut [u8]) {
rand::rng().fill_bytes(dest);
}
fn get_memory_io(&self) -> Arc<MemoryIO> {

4
core/mvcc/persistent_storage/logical_log.rs
View File

@@ -485,7 +485,7 @@ impl StreamingLogicalLogReader {
mod tests {
use std::{collections::HashSet, sync::Arc};
use rand::{thread_rng, Rng};
use rand::{rng, Rng};
use rand_chacha::{
rand_core::{RngCore, SeedableRng},
ChaCha8Rng,
@@ -646,7 +646,7 @@ mod tests {
#[test]
fn test_logical_log_read_fuzz() {
let seed = thread_rng().gen();
let seed = rng().random();
let mut rng = ChaCha8Rng::seed_from_u64(seed);
let num_transactions = rng.next_u64() % 128;
let mut txns = vec![];

4
core/storage/btree.rs
View File

@@ -7765,7 +7765,7 @@ fn shift_pointers_left(page: &mut PageContent, cell_idx: usize) {
#[cfg(test)]
mod tests {
use rand::{thread_rng, Rng};
use rand::{rng, Rng};
use rand_chacha::{
rand_core::{RngCore, SeedableRng},
ChaCha8Rng,
@@ -9603,7 +9603,7 @@ mod tests {
let mut cells = Vec::new();
let usable_space = 4096;
let mut i = 100000;
let seed = thread_rng().gen();
let seed = rng().random();
tracing::info!("seed {}", seed);
let mut rng = ChaCha8Rng::seed_from_u64(seed);
while i > 0 {

38
core/storage/encryption.rs
View File

@@ -979,14 +979,14 @@ mod tests {
}
fn generate_random_hex_key() -> String {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let mut bytes = [0u8; 32];
rng.fill(&mut bytes);
hex::encode(bytes)
}
fn generate_random_hex_key_128() -> String {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let mut bytes = [0u8; 16];
rng.fill(&mut bytes);
hex::encode(bytes)
@@ -995,7 +995,7 @@ mod tests {
fn create_test_page_1() -> Vec<u8> {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page[..SQLITE_HEADER.len()].copy_from_slice(SQLITE_HEADER);
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
// 48 is the max reserved bytes we might need for metadata with any cipher
rng.fill(&mut page[SQLITE_HEADER.len()..DEFAULT_ENCRYPTED_PAGE_SIZE - 48]);
page
@@ -1135,7 +1135,7 @@ mod tests {
#[test]
fn test_aes128gcm_encrypt_decrypt_round_trip() {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let cipher_mode = CipherMode::Aes128Gcm;
let metadata_size = cipher_mode.metadata_size();
let data_size = DEFAULT_ENCRYPTED_PAGE_SIZE - metadata_size;
@@ -1144,7 +1144,7 @@ mod tests {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page.iter_mut()
.take(data_size)
.for_each(|byte| *byte = rng.gen());
.for_each(|byte| *byte = rng.random());
page
};
@@ -1165,7 +1165,7 @@ mod tests {
#[test]
fn test_aes_encrypt_decrypt_round_trip() {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let cipher_mode = CipherMode::Aes256Gcm;
let metadata_size = cipher_mode.metadata_size();
let data_size = DEFAULT_ENCRYPTED_PAGE_SIZE - metadata_size;
@@ -1174,7 +1174,7 @@ mod tests {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page.iter_mut()
.take(data_size)
.for_each(|byte| *byte = rng.gen());
.for_each(|byte| *byte = rng.random());
page
};
@@ -1211,7 +1211,7 @@ mod tests {
#[test]
fn test_aegis256_encrypt_decrypt_round_trip() {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let cipher_mode = CipherMode::Aegis256;
let metadata_size = cipher_mode.metadata_size();
let data_size = DEFAULT_ENCRYPTED_PAGE_SIZE - metadata_size;
@@ -1220,7 +1220,7 @@ mod tests {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page.iter_mut()
.take(data_size)
.for_each(|byte| *byte = rng.gen());
.for_each(|byte| *byte = rng.random());
page
};
@@ -1256,7 +1256,7 @@ mod tests {
#[test]
fn test_aegis128x2_encrypt_decrypt_round_trip() {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let cipher_mode = CipherMode::Aegis128X2;
let metadata_size = cipher_mode.metadata_size();
let data_size = DEFAULT_ENCRYPTED_PAGE_SIZE - metadata_size;
@@ -1265,7 +1265,7 @@ mod tests {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page.iter_mut()
.take(data_size)
.for_each(|byte| *byte = rng.gen());
.for_each(|byte| *byte = rng.random());
page
};
@@ -1301,7 +1301,7 @@ mod tests {
#[test]
fn test_aegis128l_encrypt_decrypt_round_trip() {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let cipher_mode = CipherMode::Aegis128L;
let metadata_size = cipher_mode.metadata_size();
let data_size = DEFAULT_ENCRYPTED_PAGE_SIZE - metadata_size;
@@ -1310,7 +1310,7 @@ mod tests {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page.iter_mut()
.take(data_size)
.for_each(|byte| *byte = rng.gen());
.for_each(|byte| *byte = rng.random());
page
};
@@ -1346,7 +1346,7 @@ mod tests {
#[test]
fn test_aegis128x4_encrypt_decrypt_round_trip() {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let cipher_mode = CipherMode::Aegis128X4;
let metadata_size = cipher_mode.metadata_size();
let data_size = DEFAULT_ENCRYPTED_PAGE_SIZE - metadata_size;
@@ -1355,7 +1355,7 @@ mod tests {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page.iter_mut()
.take(data_size)
.for_each(|byte| *byte = rng.gen());
.for_each(|byte| *byte = rng.random());
page
};
@@ -1391,7 +1391,7 @@ mod tests {
#[test]
fn test_aegis256x2_encrypt_decrypt_round_trip() {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let cipher_mode = CipherMode::Aegis256X2;
let metadata_size = cipher_mode.metadata_size();
let data_size = DEFAULT_ENCRYPTED_PAGE_SIZE - metadata_size;
@@ -1400,7 +1400,7 @@ mod tests {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page.iter_mut()
.take(data_size)
.for_each(|byte| *byte = rng.gen());
.for_each(|byte| *byte = rng.random());
page
};
@@ -1436,7 +1436,7 @@ mod tests {
#[test]
fn test_aegis256x4_encrypt_decrypt_round_trip() {
let mut rng = rand::thread_rng();
let mut rng = rand::rng();
let cipher_mode = CipherMode::Aegis256X4;
let metadata_size = cipher_mode.metadata_size();
let data_size = DEFAULT_ENCRYPTED_PAGE_SIZE - metadata_size;
@@ -1445,7 +1445,7 @@ mod tests {
let mut page = vec![0u8; DEFAULT_ENCRYPTED_PAGE_SIZE];
page.iter_mut()
.take(data_size)
.for_each(|byte| *byte = rng.gen());
.for_each(|byte| *byte = rng.random());
page
};

16
core/storage/slot_bitmap.rs
View File

@@ -516,14 +516,14 @@ pub mod tests {
];
for &seed in seeds {
let mut rng = StdRng::seed_from_u64(seed);
let n_slots = rng.gen_range(1..10) * 64;
let n_slots = rng.random_range(1..10) * 64;
let mut pb = SlotBitmap::new(n_slots);
let mut model = vec![true; n_slots as usize];
let iters = 2000usize;
for _ in 0..iters {
let op = rng.gen_range(0..100);
let op = rng.random_range(0..100);
match op {
0..=49 => {
// alloc_one
@@ -540,8 +540,9 @@ pub mod tests {
}
50..=79 => {
// alloc_run with random length
let need =
rng.gen_range(1..=std::cmp::max(1, (n_slots as usize).min(128))) as u32;
let need = rng
.random_range(1..=std::cmp::max(1, (n_slots as usize).min(128)))
as u32;
let got = pb.alloc_run(need);
if let Some(start) = got {
assert!(start + need <= n_slots, "within bounds");
@@ -560,13 +561,14 @@ pub mod tests {
}
_ => {
// free_run on a random valid range
let len =
rng.gen_range(1..=std::cmp::max(1, (n_slots as usize).min(128))) as u32;
let len = rng
.random_range(1..=std::cmp::max(1, (n_slots as usize).min(128)))
as u32;
let max_start = n_slots.saturating_sub(len);
let start = if max_start == 0 {
0
} else {
rng.gen_range(0..=max_start)
rng.random_range(0..=max_start)
};
pb.free_run(start, len);
ref_mark_run(&mut model, start, len, true);

36
core/vdbe/execute.rs
View File

@@ -75,7 +75,6 @@ use super::{
CommitState,
};
use parking_lot::RwLock;
use rand::{thread_rng, Rng, RngCore};
use turso_parser::ast::{self, ForeignKeyClause, Name, SortOrder};
use turso_parser::parser::Parser;
@@ -4821,7 +4820,9 @@ pub fn op_function(
ScalarFunc::Typeof => Some(reg_value.exec_typeof()),
ScalarFunc::Unicode => Some(reg_value.exec_unicode()),
ScalarFunc::Quote => Some(reg_value.exec_quote()),
ScalarFunc::RandomBlob => Some(reg_value.exec_randomblob()),
ScalarFunc::RandomBlob => {
Some(reg_value.exec_randomblob(|dest| pager.io.fill_bytes(dest)))
}
ScalarFunc::ZeroBlob => Some(reg_value.exec_zeroblob()),
ScalarFunc::Soundex => Some(reg_value.exec_soundex()),
_ => unreachable!(),
@@ -4846,7 +4847,8 @@ pub fn op_function(
state.registers[*dest] = Register::Value(result);
}
ScalarFunc::Random => {
state.registers[*dest] = Register::Value(Value::exec_random());
state.registers[*dest] =
Register::Value(Value::exec_random(|| pager.io.generate_random_number()));
}
ScalarFunc::Trim => {
let reg_value = &state.registers[*start_reg];
@@ -6638,8 +6640,7 @@ pub fn op_new_rowid(
// Generate a random i64 and constrain it to the lower half of the rowid range.
// We use the lower half (1 to MAX_ROWID/2) because we're in random mode only
// when sequential allocation reached MAX_ROWID, meaning the upper range is full.
let mut rng = thread_rng();
let mut random_rowid: i64 = rng.gen();
let mut random_rowid: i64 = pager.io.generate_random_number();
random_rowid &= MAX_ROWID >> 1; // Mask to keep value in range [0, MAX_ROWID/2]
random_rowid += 1; // Ensure positive
@@ -8848,14 +8849,17 @@ impl Value {
})
}
pub fn exec_random() -> Self {
let mut buf = [0u8; 8];
getrandom::getrandom(&mut buf).unwrap();
let random_number = i64::from_ne_bytes(buf);
Value::Integer(random_number)
pub fn exec_random<F>(generate_random_number: F) -> Self
where
F: Fn() -> i64,
{
Value::Integer(generate_random_number())
}
pub fn exec_randomblob(&self) -> Value {
pub fn exec_randomblob<F>(&self, fill_bytes: F) -> Value
where
F: Fn(&mut [u8]),
{
let length = match self {
Value::Integer(i) => *i,
Value::Float(f) => *f as i64,
@@ -8865,7 +8869,7 @@ impl Value {
.max(1) as usize;
let mut blob: Vec<u8> = vec![0; length];
rand::thread_rng().fill_bytes(&mut blob);
fill_bytes(&mut blob);
Value::Blob(blob)
}
@@ -10222,6 +10226,8 @@ where
#[cfg(test)]
mod tests {
use rand::{Rng, RngCore};
use super::*;
use crate::types::Value;
@@ -10982,7 +10988,7 @@ mod tests {
#[test]
fn test_random() {
match Value::exec_random() {
match Value::exec_random(|| rand::rng().random()) {
Value::Integer(value) => {
// Check that the value is within the range of i64
assert!(
@@ -11045,7 +11051,9 @@ mod tests {
];
for test_case in &test_cases {
let result = test_case.input.exec_randomblob();
let result = test_case.input.exec_randomblob(|dest| {
rand::rng().fill_bytes(dest);
});
match result {
Value::Blob(blob) => {
assert_eq!(blob.len(), test_case.expected_len);

8
simulator/runner/io.rs
View File

@@ -3,7 +3,7 @@ use std::{
sync::Arc,
};
use rand::{RngCore, SeedableRng};
use rand::{Rng, RngCore, SeedableRng};
use rand_chacha::ChaCha8Rng;
use turso_core::{Clock, IO, Instant, OpenFlags, PlatformIO, Result};
@@ -136,6 +136,10 @@ impl IO for SimulatorIO {
}
fn generate_random_number(&self) -> i64 {
self.rng.borrow_mut().next_u64() as i64
self.rng.borrow_mut().random()
}
fn fill_bytes(&self, dest: &mut [u8]) {
self.rng.borrow_mut().fill_bytes(dest);
}
}

8
simulator/runner/memory/io.rs
View File

@@ -3,7 +3,7 @@ use std::sync::Arc;
use indexmap::IndexMap;
use parking_lot::Mutex;
use rand::{RngCore, SeedableRng};
use rand::{Rng, RngCore, SeedableRng};
use rand_chacha::ChaCha8Rng;
use turso_core::{Clock, Completion, IO, Instant, OpenFlags, Result};
@@ -269,7 +269,11 @@ impl IO for MemorySimIO {
}
fn generate_random_number(&self) -> i64 {
self.rng.borrow_mut().next_u64() as i64
self.rng.borrow_mut().random()
}
fn fill_bytes(&self, dest: &mut [u8]) {
self.rng.borrow_mut().fill_bytes(dest);
}
fn remove_file(&self, path: &str) -> Result<()> {

5
whopper/io.rs
View File

@@ -142,6 +142,11 @@ impl IO for SimulatorIO {
let mut rng = self.rng.lock().unwrap();
rng.next_u64() as i64
}
fn fill_bytes(&self, dest: &mut [u8]) {
let mut rng = self.rng.lock().unwrap();
rng.fill_bytes(dest);
}
}
const MAX_FILE_SIZE: usize = 1 << 33; // 8 GiB