aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-01-02 16:04:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

adjust remaining calculation to use the profile

Browse Source
This commit is contained in:
pedrocarlo
2025-08-28 17:26:36 -03:00
parent 962666831b
commit 2f237fdcfd
6 changed files with 159 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
simulator/generation/plan.rs
View File

@@ -254,16 +254,27 @@ impl Display for InteractionPlan {
#[derive(Debug, Clone, Copy)]
pub(crate) struct InteractionStats {
pub(crate) read_count: usize,
pub(crate) write_count: usize,
pub(crate) delete_count: usize,
pub(crate) update_count: usize,
pub(crate) create_count: usize,
pub(crate) create_index_count: usize,
pub(crate) drop_count: usize,
pub(crate) begin_count: usize,
pub(crate) commit_count: usize,
pub(crate) rollback_count: usize,
pub(crate) select_count: u32,
pub(crate) insert_count: u32,
pub(crate) delete_count: u32,
pub(crate) update_count: u32,
pub(crate) create_count: u32,
pub(crate) create_index_count: u32,
pub(crate) drop_count: u32,
pub(crate) begin_count: u32,
pub(crate) commit_count: u32,
pub(crate) rollback_count: u32,
}
impl InteractionStats {
pub fn total_writes(&self) -> u32 {
self.insert_count
+ self.delete_count
+ self.update_count
+ self.create_count
+ self.create_index_count
+ self.drop_count
}
}
impl Display for InteractionStats {
@@ -271,8 +282,8 @@ impl Display for InteractionStats {
write!(
f,
"Read: {}, Write: {}, Delete: {}, Update: {}, Create: {}, CreateIndex: {}, Drop: {}, Begin: {}, Commit: {}, Rollback: {}",
self.read_count,
self.write_count,
self.select_count,
self.insert_count,
self.delete_count,
self.update_count,
self.create_count,
@@ -351,8 +362,8 @@ impl InteractionPlan {
pub(crate) fn stats(&self) -> InteractionStats {
let mut stats = InteractionStats {
read_count: 0,
write_count: 0,
select_count: 0,
insert_count: 0,
delete_count: 0,
update_count: 0,
create_count: 0,
@@ -365,8 +376,8 @@ impl InteractionPlan {
fn query_stat(q: &Query, stats: &mut InteractionStats) {
match q {
Query::Select(_) => stats.read_count += 1,
Query::Insert(_) => stats.write_count += 1,
Query::Select(_) => stats.select_count += 1,
Query::Insert(_) => stats.insert_count += 1,
Query::Delete(_) => stats.delete_count += 1,
Query::Create(_) => stats.create_count += 1,
Query::Drop(_) => stats.drop_count += 1,
@@ -399,7 +410,7 @@ impl InteractionPlan {
pub fn generate_plan<R: rand::Rng>(rng: &mut R, env: &mut SimulatorEnv) -> Self {
let mut plan = InteractionPlan::new();
let num_interactions = env.opts.max_interactions;
let num_interactions = env.opts.max_interactions as usize;
// First create at least one table
let create_query = Create::arbitrary(rng, env);
@@ -821,25 +832,25 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions {
_context: &C,
(env, stats): (&SimulatorEnv, InteractionStats),
) -> Self {
let remaining_ = remaining(&env.opts, &stats);
let remaining_ = remaining(env.opts.max_interactions, &env.profile.query, &stats);
frequency(
vec![
(
f64::min(remaining_.read, remaining_.write) + remaining_.create,
u32::min(remaining_.select, remaining_.insert) + remaining_.create,
Box::new(|rng: &mut R| {
Interactions::Property(Property::arbitrary_from(rng, env, (env, &stats)))
}),
),
(
remaining_.read,
remaining_.select,
Box::new(|rng: &mut R| random_read(rng, env)),
),
(
remaining_.read / 3.0,
remaining_.select / 3,
Box::new(|rng: &mut R| random_expr(rng, env)),
),
(
remaining_.write,
remaining_.insert,
Box::new(|rng: &mut R| random_write(rng, env)),
),
(
@@ -867,15 +878,15 @@ impl ArbitraryFrom<(&SimulatorEnv, InteractionStats)> for Interactions {
),
(
// remaining_.drop,
0.0,
0,
Box::new(|rng: &mut R| random_drop(rng, env)),
),
(
remaining_
.read
.min(remaining_.write)
.select
.min(remaining_.insert)
.min(remaining_.create)
.max(1.0),
.max(1),
Box::new(|rng: &mut R| random_fault(rng, env)),
),
],

136
simulator/generation/property.rs
View File

@@ -16,9 +16,7 @@ use turso_core::{LimboError, types};
use turso_parser::ast::{self, Distinctness};
use crate::{
generation::Shadow as _,
model::Query,
runner::env::{SimulatorEnv, SimulatorOpts},
generation::Shadow as _, model::Query, profiles::query::QueryProfile, runner::env::SimulatorEnv,
};
use super::plan::{Assertion, Interaction, InteractionStats, ResultSet};
@@ -1034,44 +1032,66 @@ fn assert_all_table_values(tables: &[String]) -> impl Iterator<Item = Interactio
#[derive(Debug)]
pub(crate) struct Remaining {
pub(crate) read: f64,
pub(crate) write: f64,
pub(crate) create: f64,
pub(crate) create_index: f64,
pub(crate) delete: f64,
pub(crate) update: f64,
pub(crate) drop: f64,
pub(crate) select: u32,
pub(crate) insert: u32,
pub(crate) create: u32,
pub(crate) create_index: u32,
pub(crate) delete: u32,
pub(crate) update: u32,
pub(crate) drop: u32,
}
pub(crate) fn remaining(opts: &SimulatorOpts, stats: &InteractionStats) -> Remaining {
let remaining_read = ((opts.max_interactions as f64 * opts.read_percent / 100.0)
- (stats.read_count as f64))
.max(0.0);
let remaining_write = ((opts.max_interactions as f64 * opts.write_percent / 100.0)
- (stats.write_count as f64))
.max(0.0);
let remaining_create = ((opts.max_interactions as f64 * opts.create_percent / 100.0)
- (stats.create_count as f64))
.max(0.0);
pub(crate) fn remaining(
max_interactions: u32,
opts: &QueryProfile,
stats: &InteractionStats,
) -> Remaining {
let total_weight = opts.read_weight + opts.write_weight;
let remaining_create_index = ((opts.max_interactions as f64 * opts.create_index_percent
/ 100.0)
- (stats.create_index_count as f64))
.max(0.0);
// Total amount of reads. Only considers select operations
let total_reads = (max_interactions * opts.read_weight) / total_weight;
// Total amount of writes.
let total_writes = (max_interactions * opts.write_weight) / total_weight;
let remaining_delete = ((opts.max_interactions as f64 * opts.delete_percent / 100.0)
- (stats.delete_count as f64))
.max(0.0);
let remaining_update = ((opts.max_interactions as f64 * opts.update_percent / 100.0)
- (stats.update_count as f64))
.max(0.0);
let remaining_drop = ((opts.max_interactions as f64 * opts.drop_percent / 100.0)
- (stats.drop_count as f64))
.max(0.0);
let remaining_select = total_reads
.checked_sub(stats.select_count)
.unwrap_or_default();
// This total is the sum of all the query weights that are write operations
let sum_write_weight = opts.create_table_weight
+ opts.create_index_weight
+ opts.insert_weight
+ opts.update_weight
+ opts.delete_weight
+ opts.drop_table_weight;
let total_insert = (total_writes * opts.insert_weight) / sum_write_weight;
let total_create = (total_writes * opts.create_table_weight) / sum_write_weight;
let total_create_index = (total_writes * opts.create_index_weight) / sum_write_weight;
let total_delete = (total_writes * opts.delete_weight) / sum_write_weight;
let total_update = (total_writes * opts.update_weight) / sum_write_weight;
let total_drop = (total_writes * opts.drop_table_weight) / sum_write_weight;
let remaining_insert = total_insert
.checked_sub(stats.insert_count)
.unwrap_or_default();
let remaining_create = total_create
.checked_sub(stats.create_count)
.unwrap_or_default();
let remaining_create_index = total_create_index
.checked_sub(stats.create_index_count)
.unwrap_or_default();
let remaining_delete = total_delete
.checked_sub(stats.delete_count)
.unwrap_or_default();
let remaining_update = total_update
.checked_sub(stats.update_count)
.unwrap_or_default();
let remaining_drop = total_drop.checked_sub(stats.drop_count).unwrap_or_default();
Remaining {
read: remaining_read,
write: remaining_write,
select: remaining_select,
insert: remaining_insert,
create: remaining_create,
create_index: remaining_create_index,
delete: remaining_delete,
@@ -1434,72 +1454,72 @@ impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property {
_context: &C,
(env, stats): (&SimulatorEnv, &InteractionStats),
) -> Self {
let remaining_ = remaining(&env.opts, stats);
let remaining_ = remaining(env.opts.max_interactions, &env.profile.query, stats);
frequency(
vec![
(
if !env.opts.disable_insert_values_select {
f64::min(remaining_.read, remaining_.write)
u32::min(remaining_.select, remaining_.insert)
} else {
0.0
0
},
Box::new(|rng: &mut R| property_insert_values_select(rng, env, &remaining_)),
),
(
remaining_.read,
remaining_.select,
Box::new(|rng: &mut R| property_table_has_expected_content(rng, env)),
),
(
f64::min(remaining_.read, remaining_.write),
u32::min(remaining_.select, remaining_.insert),
Box::new(|rng: &mut R| property_read_your_updates_back(rng, env)),
),
(
if !env.opts.disable_double_create_failure {
remaining_.create / 2.0
remaining_.create / 2
} else {
0.0
0
},
Box::new(|rng: &mut R| property_double_create_failure(rng, env, &remaining_)),
),
(
if !env.opts.disable_select_limit {
remaining_.read
remaining_.select
} else {
0.0
0
},
Box::new(|rng: &mut R| property_select_limit(rng, env)),
),
(
if !env.opts.disable_delete_select {
f64::min(remaining_.read, remaining_.write).min(remaining_.delete)
u32::min(remaining_.select, remaining_.insert).min(remaining_.delete)
} else {
0.0
0
},
Box::new(|rng: &mut R| property_delete_select(rng, env, &remaining_)),
),
(
if !env.opts.disable_drop_select {
// remaining_.drop
0.0
0
} else {
0.0
0
},
Box::new(|rng: &mut R| property_drop_select(rng, env, &remaining_)),
),
(
if !env.opts.disable_select_optimizer {
remaining_.read / 2.0
remaining_.select / 2
} else {
0.0
0
},
Box::new(|rng: &mut R| property_select_select_optimizer(rng, env)),
),
(
if env.opts.experimental_indexes && !env.opts.disable_where_true_false_null {
remaining_.read / 2.0
remaining_.select / 2
} else {
0.0
0
},
Box::new(|rng: &mut R| property_where_true_false_null(rng, env)),
),
@@ -1507,25 +1527,25 @@ impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property {
if env.opts.experimental_indexes
&& !env.opts.disable_union_all_preserves_cardinality
{
remaining_.read / 3.0
remaining_.select / 3
} else {
0.0
0
},
Box::new(|rng: &mut R| property_union_all_preserves_cardinality(rng, env)),
),
(
if !env.opts.disable_fsync_no_wait {
50.0 // Freestyle number
50 // Freestyle number
} else {
0.0
0
},
Box::new(|rng: &mut R| property_fsync_no_wait(rng, env, &remaining_)),
),
(
if !env.opts.disable_faulty_query {
20.0
20
} else {
0.0
0
},
Box::new(|rng: &mut R| property_faulty_query(rng, env, &remaining_)),
),

6
simulator/generation/query.rs
View File

@@ -20,11 +20,11 @@ impl ArbitraryFrom<&Remaining> for Query {
Box::new(|rng| Self::Create(Create::arbitrary(rng, context))),
),
(
remaining.read,
remaining.select,
Box::new(|rng| Self::Select(Select::arbitrary(rng, context))),
),
(
remaining.write,
remaining.insert,
Box::new(|rng| Self::Insert(Insert::arbitrary(rng, context))),
),
(
@@ -32,7 +32,7 @@ impl ArbitraryFrom<&Remaining> for Query {
Box::new(|rng| Self::Update(Update::arbitrary(rng, context))),
),
(
f64::min(remaining.write, remaining.delete),
remaining.insert.min(remaining.delete),
Box::new(|rng| Self::Delete(Delete::arbitrary(rng, context))),
),
],

12
simulator/profiles/mod.rs
View File

@@ -14,8 +14,8 @@ use strum::EnumString;
use crate::profiles::{io::IOProfile, query::QueryProfile};
mod io;
mod query;
pub mod io;
pub mod query;
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, Validate)]
#[serde(deny_unknown_fields, default)]
@@ -45,11 +45,13 @@ impl Profile {
query: QueryProfile {
gen_opts: Opts {
// TODO: in the future tweak blob size for bigger inserts
// TODO: increase number of rows increased as well
// TODO: increase number of rows as well
..Default::default()
},
delete: false,
update: false,
read_weight: 30,
write_weight: 70,
delete_weight: 0,
update_weight: 0,
..Default::default()
},
..Default::default()

43
simulator/profiles/query.rs
View File

@@ -9,29 +9,48 @@ pub struct QueryProfile {
#[garde(dive)]
pub gen_opts: Opts,
#[garde(skip)]
pub create_table: bool,
/// Effectively the weight of how many select operations we want
pub read_weight: u32,
#[garde(skip)]
pub create_index: bool,
pub write_weight: u32,
// All weights below are only going to be sampled when we determine we are doing a write operation,
// meaning we first sample between `read_weight` and `write_weight`, and if we a write_weight we will then sample the weights below
#[garde(skip)]
pub insert: bool,
pub create_table_weight: u32,
#[garde(skip)]
pub update: bool,
pub create_index_weight: u32,
#[garde(skip)]
pub delete: bool,
pub insert_weight: u32,
#[garde(skip)]
pub drop_table: bool,
pub update_weight: u32,
#[garde(skip)]
pub delete_weight: u32,
#[garde(skip)]
pub drop_table_weight: u32,
}
impl Default for QueryProfile {
fn default() -> Self {
Self {
gen_opts: Opts::default(),
create_table: true,
create_index: true,
insert: true,
update: true,
delete: true,
drop_table: true,
read_weight: 60,
write_weight: 50,
create_table_weight: 15,
create_index_weight: 5,
insert_weight: 30,
update_weight: 20,
delete_weight: 20,
drop_table_weight: 2,
}
}
}
#[derive(Debug, Clone, strum::VariantArray)]
pub enum QueryTypes {
CreateTable,
CreateIndex,
Insert,
Update,
Delete,
DropTable,
}

21
simulator/runner/env.rs
View File

@@ -227,13 +227,6 @@ impl SimulatorEnv {
max_connections: 1, // TODO: for now let's use one connection as we didn't implement
// correct transactions processing
max_tables: rng.random_range(0..128),
create_percent,
create_index_percent,
read_percent,
write_percent,
delete_percent,
drop_percent,
update_percent,
disable_select_optimizer: cli_opts.disable_select_optimizer,
disable_insert_values_select: cli_opts.disable_insert_values_select,
disable_double_create_failure: cli_opts.disable_double_create_failure,
@@ -246,7 +239,8 @@ impl SimulatorEnv {
disable_fsync_no_wait: cli_opts.disable_fsync_no_wait,
disable_faulty_query: cli_opts.disable_faulty_query,
page_size: 4096, // TODO: randomize this too
max_interactions: rng.random_range(cli_opts.minimum_tests..=cli_opts.maximum_tests),
max_interactions: rng.random_range(cli_opts.minimum_tests..=cli_opts.maximum_tests)
as u32,
max_time_simulation: cli_opts.maximum_time,
disable_reopen_database: cli_opts.disable_reopen_database,
latency_probability: cli_opts.latency_probability,
@@ -399,15 +393,6 @@ pub(crate) struct SimulatorOpts {
pub(crate) ticks: usize,
pub(crate) max_connections: usize,
pub(crate) max_tables: usize,
// this next options are the distribution of workload where read_percent + write_percent +
// delete_percent == 100%
pub(crate) create_percent: f64,
pub(crate) create_index_percent: f64,
pub(crate) read_percent: f64,
pub(crate) write_percent: f64,
pub(crate) delete_percent: f64,
pub(crate) update_percent: f64,
pub(crate) drop_percent: f64,
pub(crate) disable_select_optimizer: bool,
pub(crate) disable_insert_values_select: bool,
@@ -421,7 +406,7 @@ pub(crate) struct SimulatorOpts {
pub(crate) disable_faulty_query: bool,
pub(crate) disable_reopen_database: bool,
pub(crate) max_interactions: usize,
pub(crate) max_interactions: u32,
pub(crate) page_size: usize,
pub(crate) max_time_simulation: usize,
pub(crate) latency_probability: usize,