Files
turso/core/benches/benchmark.rs
2025-08-11 08:02:35 +02:00

607 lines
22 KiB
Rust

use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use pprof::criterion::{Output, PProfProfiler};
use std::{sync::Arc, time::Instant};
use turso_core::{Database, PlatformIO};
#[cfg(not(target_family = "wasm"))]
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
fn rusqlite_open() -> rusqlite::Connection {
let sqlite_conn = rusqlite::Connection::open("../testing/testing.db").unwrap();
sqlite_conn
.pragma_update(None, "locking_mode", "EXCLUSIVE")
.unwrap();
sqlite_conn
}
fn bench_open(criterion: &mut Criterion) {
// https://github.com/tursodatabase/turso/issues/174
// The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err();
if !std::fs::exists("../testing/schema_5k.db").unwrap() {
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/schema_5k.db", false, false).unwrap();
let conn = db.connect().unwrap();
for i in 0..5000 {
conn.execute(
format!("CREATE TABLE table_{i} ( id INTEGER PRIMARY KEY, name TEXT, value INTEGER, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )")
).unwrap();
}
}
let mut group = criterion.benchmark_group("Open/Connect");
group.bench_function(BenchmarkId::new("limbo_schema", ""), |b| {
b.iter(|| {
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db =
Database::open_file(io.clone(), "../testing/schema_5k.db", false, false).unwrap();
let conn = db.connect().unwrap();
conn.execute("SELECT * FROM table_0").unwrap();
});
});
if enable_rusqlite {
group.bench_function(BenchmarkId::new("sqlite_schema", ""), |b| {
b.iter(|| {
let conn = rusqlite::Connection::open("../testing/schema_5k.db").unwrap();
conn.execute("SELECT * FROM table_0", ()).unwrap();
});
});
}
group.finish();
}
fn bench_alter(criterion: &mut Criterion) {
// https://github.com/tursodatabase/turso/issues/174
// The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err();
if !std::fs::exists("../testing/schema_5k.db").unwrap() {
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/schema_5k.db", false, false).unwrap();
let conn = db.connect().unwrap();
for i in 0..5000 {
conn.execute(
format!("CREATE TABLE table_{i} ( id INTEGER PRIMARY KEY, name TEXT, value INTEGER, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP )")
).unwrap();
}
}
let mut group = criterion.benchmark_group("`ALTER TABLE _ RENAME TO _`");
group.bench_function(BenchmarkId::new("limbo_rename_table", ""), |b| {
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/schema_5k.db", false, false).unwrap();
let conn = db.connect().unwrap();
b.iter_custom(|iters| {
(0..iters)
.map(|_| {
conn.execute("CREATE TABLE x(a)").unwrap();
let elapsed = {
let start = Instant::now();
conn.execute("ALTER TABLE x RENAME TO y").unwrap();
start.elapsed()
};
conn.execute("DROP TABLE y").unwrap();
elapsed
})
.sum()
});
});
if enable_rusqlite {
group.bench_function(BenchmarkId::new("sqlite_rename_table", ""), |b| {
let conn = rusqlite::Connection::open("../testing/schema_5k.db").unwrap();
b.iter_custom(|iters| {
(0..iters)
.map(|_| {
conn.execute("CREATE TABLE x(a)", ()).unwrap();
let elapsed = {
let start = Instant::now();
conn.execute("ALTER TABLE x RENAME TO y", ()).unwrap();
start.elapsed()
};
conn.execute("DROP TABLE y", ()).unwrap();
elapsed
})
.sum()
});
});
}
group.finish();
let mut group = criterion.benchmark_group("`ALTER TABLE _ RENAME COLUMN _ TO _`");
group.bench_function(BenchmarkId::new("limbo_rename_column", ""), |b| {
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/schema_5k.db", false, false).unwrap();
let conn = db.connect().unwrap();
b.iter_custom(|iters| {
(0..iters)
.map(|_| {
conn.execute("CREATE TABLE x(a)").unwrap();
let elapsed = {
let start = Instant::now();
conn.execute("ALTER TABLE x RENAME COLUMN a TO b").unwrap();
start.elapsed()
};
conn.execute("DROP TABLE x").unwrap();
elapsed
})
.sum()
});
});
if enable_rusqlite {
group.bench_function(BenchmarkId::new("sqlite_rename_column", ""), |b| {
let conn = rusqlite::Connection::open("../testing/schema_5k.db").unwrap();
b.iter_custom(|iters| {
(0..iters)
.map(|_| {
conn.execute("CREATE TABLE x(a)", ()).unwrap();
let elapsed = {
let start = Instant::now();
conn.execute("ALTER TABLE x RENAME COLUMN a TO b", ())
.unwrap();
start.elapsed()
};
conn.execute("DROP TABLE x", ()).unwrap();
elapsed
})
.sum()
});
});
}
group.finish();
let mut group = criterion.benchmark_group("`ALTER TABLE _ ADD COLUMN _`");
group.bench_function(BenchmarkId::new("limbo_add_column", ""), |b| {
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/schema_5k.db", false, false).unwrap();
let conn = db.connect().unwrap();
b.iter_custom(|iters| {
(0..iters)
.map(|_| {
conn.execute("CREATE TABLE x(a)").unwrap();
let elapsed = {
let start = Instant::now();
conn.execute("ALTER TABLE x ADD COLUMN b").unwrap();
start.elapsed()
};
conn.execute("DROP TABLE x").unwrap();
elapsed
})
.sum()
});
});
if enable_rusqlite {
group.bench_function(BenchmarkId::new("sqlite_add_column", ""), |b| {
let conn = rusqlite::Connection::open("../testing/schema_5k.db").unwrap();
b.iter_custom(|iters| {
(0..iters)
.map(|_| {
conn.execute("CREATE TABLE x(a)", ()).unwrap();
let elapsed = {
let start = Instant::now();
conn.execute("ALTER TABLE x ADD COLUMN b", ()).unwrap();
start.elapsed()
};
conn.execute("DROP TABLE x", ()).unwrap();
elapsed
})
.sum()
});
});
}
group.finish();
let mut group = criterion.benchmark_group("`ALTER TABLE _ DROP COLUMN _`");
group.bench_function(BenchmarkId::new("limbo_drop_column", ""), |b| {
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/schema_5k.db", false, false).unwrap();
let conn = db.connect().unwrap();
b.iter_custom(|iters| {
(0..iters)
.map(|_| {
conn.execute("CREATE TABLE x(a, b)").unwrap();
let elapsed = {
let start = Instant::now();
conn.execute("ALTER TABLE x DROP COLUMN b").unwrap();
start.elapsed()
};
conn.execute("DROP TABLE x").unwrap();
elapsed
})
.sum()
});
});
if enable_rusqlite {
group.bench_function(BenchmarkId::new("sqlite_drop_column", ""), |b| {
let conn = rusqlite::Connection::open("../testing/schema_5k.db").unwrap();
b.iter_custom(|iters| {
(0..iters)
.map(|_| {
conn.execute("CREATE TABLE x(a, b)", ()).unwrap();
let elapsed = {
let start = Instant::now();
conn.execute("ALTER TABLE x DROP COLUMN b", ()).unwrap();
start.elapsed()
};
conn.execute("DROP TABLE x", ()).unwrap();
elapsed
})
.sum()
});
});
}
group.finish();
}
fn bench_prepare_query(criterion: &mut Criterion) {
// https://github.com/tursodatabase/turso/issues/174
// The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err();
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/testing.db", false, false).unwrap();
let limbo_conn = db.connect().unwrap();
let queries = [
"SELECT 1",
"SELECT * FROM users LIMIT 1",
"SELECT first_name, count(1) FROM users GROUP BY first_name HAVING count(1) > 1 ORDER BY count(1) LIMIT 1",
];
for query in queries.iter() {
let mut group = criterion.benchmark_group(format!("Prepare `{query}`"));
group.bench_with_input(
BenchmarkId::new("limbo_parse_query", query),
query,
|b, query| {
b.iter(|| {
limbo_conn.prepare(query).unwrap();
});
},
);
if enable_rusqlite {
let sqlite_conn = rusqlite_open();
group.bench_with_input(
BenchmarkId::new("sqlite_parse_query", query),
query,
|b, query| {
b.iter(|| {
sqlite_conn.prepare(query).unwrap();
});
},
);
}
group.finish();
}
}
fn bench_execute_select_rows(criterion: &mut Criterion) {
// https://github.com/tursodatabase/turso/issues/174
// The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err();
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/testing.db", false, false).unwrap();
let limbo_conn = db.connect().unwrap();
let mut group = criterion.benchmark_group("Execute `SELECT * FROM users LIMIT ?`");
for i in [1, 10, 50, 100] {
group.bench_with_input(
BenchmarkId::new("limbo_execute_select_rows", i),
&i,
|b, i| {
// TODO: LIMIT doesn't support query parameters.
let mut stmt = limbo_conn
.prepare(format!("SELECT * FROM users LIMIT {}", *i))
.unwrap();
b.iter(|| {
loop {
match stmt.step().unwrap() {
turso_core::StepResult::Row => {
black_box(stmt.row());
}
turso_core::StepResult::IO => {
stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
}
turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => {
unreachable!();
}
}
}
stmt.reset();
});
},
);
if enable_rusqlite {
let sqlite_conn = rusqlite_open();
group.bench_with_input(
BenchmarkId::new("sqlite_execute_select_rows", i),
&i,
|b, i| {
// TODO: Use parameters once we fix the above.
let mut stmt = sqlite_conn
.prepare(&format!("SELECT * FROM users LIMIT {}", *i))
.unwrap();
b.iter(|| {
let mut rows = stmt.raw_query();
while let Some(row) = rows.next().unwrap() {
black_box(row);
}
});
},
);
}
}
group.finish();
}
fn bench_execute_select_1(criterion: &mut Criterion) {
// https://github.com/tursodatabase/turso/issues/174
// The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err();
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/testing.db", false, false).unwrap();
let limbo_conn = db.connect().unwrap();
let mut group = criterion.benchmark_group("Execute `SELECT 1`");
group.bench_function("limbo_execute_select_1", |b| {
let mut stmt = limbo_conn.prepare("SELECT 1").unwrap();
b.iter(|| {
loop {
match stmt.step().unwrap() {
turso_core::StepResult::Row => {
black_box(stmt.row());
}
turso_core::StepResult::IO => {
stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
}
turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => {
unreachable!();
}
}
}
stmt.reset();
});
});
if enable_rusqlite {
let sqlite_conn = rusqlite_open();
group.bench_function("sqlite_execute_select_1", |b| {
let mut stmt = sqlite_conn.prepare("SELECT 1").unwrap();
b.iter(|| {
let mut rows = stmt.raw_query();
while let Some(row) = rows.next().unwrap() {
black_box(row);
}
});
});
}
group.finish();
}
fn bench_execute_select_count(criterion: &mut Criterion) {
// https://github.com/tursodatabase/turso/issues/174
// The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err();
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/testing.db", false, false).unwrap();
let limbo_conn = db.connect().unwrap();
let mut group = criterion.benchmark_group("Execute `SELECT count() FROM users`");
group.bench_function("limbo_execute_select_count", |b| {
let mut stmt = limbo_conn.prepare("SELECT count() FROM users").unwrap();
b.iter(|| {
loop {
match stmt.step().unwrap() {
turso_core::StepResult::Row => {
black_box(stmt.row());
}
turso_core::StepResult::IO => {
stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
}
turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => {
unreachable!();
}
}
}
stmt.reset();
});
});
if enable_rusqlite {
let sqlite_conn = rusqlite_open();
group.bench_function("sqlite_execute_select_count", |b| {
let mut stmt = sqlite_conn.prepare("SELECT count() FROM users").unwrap();
b.iter(|| {
let mut rows = stmt.raw_query();
while let Some(row) = rows.next().unwrap() {
black_box(row);
}
});
});
}
group.finish();
}
fn bench_insert_rows(criterion: &mut Criterion) {
// The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err();
let mut group = criterion.benchmark_group("Insert rows in batches");
// Test different batch sizes
for batch_size in [1, 10, 100] {
let temp_dir = tempfile::tempdir().unwrap();
let db_path = temp_dir.path().join("bench.db");
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), db_path.to_str().unwrap(), false, false).unwrap();
let limbo_conn = db.connect().unwrap();
let mut stmt = limbo_conn
.query("CREATE TABLE test (id INTEGER, value TEXT)")
.unwrap()
.unwrap();
loop {
match stmt.step().unwrap() {
turso_core::StepResult::IO => {
stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
}
turso_core::StepResult::Row => {
unreachable!();
}
turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => {
unreachable!();
}
}
}
group.bench_function(format!("limbo_insert_{batch_size}_rows"), |b| {
let mut values = String::from("INSERT INTO test VALUES ");
for i in 0..batch_size {
if i > 0 {
values.push(',');
}
values.push_str(&format!("({}, '{}')", i, format_args!("value_{i}")));
}
let mut stmt = limbo_conn.prepare(&values).unwrap();
b.iter(|| {
loop {
match stmt.step().unwrap() {
turso_core::StepResult::IO => {
stmt.run_once().unwrap();
}
turso_core::StepResult::Done => {
break;
}
turso_core::StepResult::Row => {
unreachable!();
}
turso_core::StepResult::Interrupt | turso_core::StepResult::Busy => {
unreachable!();
}
}
}
stmt.reset();
});
});
if enable_rusqlite {
let temp_dir = tempfile::tempdir().unwrap();
let db_path = temp_dir.path().join("bench.db");
let sqlite_conn = rusqlite::Connection::open(db_path).unwrap();
sqlite_conn
.pragma_update(None, "synchronous", "FULL")
.unwrap();
sqlite_conn
.pragma_update(None, "journal_mode", "WAL")
.unwrap();
sqlite_conn
.pragma_update(None, "locking_mode", "EXCLUSIVE")
.unwrap();
let journal_mode = sqlite_conn
.pragma_query_value(None, "journal_mode", |row| row.get::<_, String>(0))
.unwrap();
assert_eq!(journal_mode.to_lowercase(), "wal");
let synchronous = sqlite_conn
.pragma_query_value(None, "synchronous", |row| row.get::<_, usize>(0))
.unwrap();
const FULL: usize = 2;
assert_eq!(synchronous, FULL);
// Create test table
sqlite_conn
.execute("CREATE TABLE test (id INTEGER, value TEXT)", [])
.unwrap();
sqlite_conn
.pragma_update(None, "locking_mode", "EXCLUSIVE")
.unwrap();
group.bench_function(format!("sqlite_insert_{batch_size}_rows"), |b| {
let mut values = String::from("INSERT INTO test VALUES ");
for i in 0..batch_size {
if i > 0 {
values.push(',');
}
values.push_str(&format!("({}, '{}')", i, format_args!("value_{i}")));
}
let mut stmt = sqlite_conn.prepare(&values).unwrap();
b.iter(|| {
let mut rows = stmt.raw_query();
while let Some(row) = rows.next().unwrap() {
black_box(row);
}
});
});
}
}
group.finish();
}
criterion_group! {
name = benches;
config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
targets = bench_open, bench_alter, bench_prepare_query, bench_execute_select_1, bench_execute_select_rows, bench_execute_select_count, bench_insert_rows
}
criterion_main!(benches);