aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-01-06 09:44:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

improve sync engine

Browse Source
This commit is contained in:
Nikita Sivukhin
2025-08-27 15:29:30 +04:00
parent 30c5473151
commit 009aa479bf
21 changed files with 2482 additions and 1389 deletions
Download Patch File Download Diff File Expand all files Collapse all files

363
sync/engine/src/database_replay_generator.rs
View File

@@ -1,29 +1,110 @@
use std::sync::Arc;
use std::{collections::HashMap, sync::Arc};
use crate::{
database_tape::{run_stmt_once, DatabaseReplaySessionOpts},
errors::Error,
types::{Coro, DatabaseChangeType, DatabaseTapeRowChange, DatabaseTapeRowChangeType},
types::{
Coro, DatabaseChangeType, DatabaseRowMutation, DatabaseTapeRowChange,
DatabaseTapeRowChangeType,
},
Result,
};
pub struct DatabaseReplayGenerator {
pub struct DatabaseReplayGenerator<Ctx = ()> {
pub conn: Arc<turso_core::Connection>,
pub opts: DatabaseReplaySessionOpts,
pub opts: DatabaseReplaySessionOpts<Ctx>,
}
pub struct ReplayInfo {
pub change_type: DatabaseChangeType,
pub query: String,
pub pk_column_indices: Option<Vec<usize>>,
pub column_names: Vec<String>,
pub is_ddl_replay: bool,
}
const SQLITE_SCHEMA_TABLE: &str = "sqlite_schema";
impl DatabaseReplayGenerator {
pub fn new(conn: Arc<turso_core::Connection>, opts: DatabaseReplaySessionOpts) -> Self {
impl<Ctx> DatabaseReplayGenerator<Ctx> {
pub fn new(conn: Arc<turso_core::Connection>, opts: DatabaseReplaySessionOpts<Ctx>) -> Self {
Self { conn, opts }
}
pub fn create_mutation(
&self,
info: &ReplayInfo,
change: &DatabaseTapeRowChange,
) -> Result<DatabaseRowMutation> {
match &change.change {
DatabaseTapeRowChangeType::Delete { before } => Ok(DatabaseRowMutation {
change_time: change.change_time,
table_name: change.table_name.to_string(),
id: change.id,
change_type: info.change_type,
before: Some(self.create_row_full(info, before)),
after: None,
updates: None,
}),
DatabaseTapeRowChangeType::Insert { after } => Ok(DatabaseRowMutation {
change_time: change.change_time,
table_name: change.table_name.to_string(),
id: change.id,
change_type: info.change_type,
before: None,
after: Some(self.create_row_full(info, after)),
updates: None,
}),
DatabaseTapeRowChangeType::Update {
before,
after,
updates,
} => Ok(DatabaseRowMutation {
change_time: change.change_time,
table_name: change.table_name.to_string(),
id: change.id,
change_type: info.change_type,
before: Some(self.create_row_full(info, before)),
after: Some(self.create_row_full(info, after)),
updates: updates
.as_ref()
.map(|updates| self.create_row_update(info, &updates)),
}),
}
}
fn create_row_full(
&self,
info: &ReplayInfo,
values: &Vec<turso_core::Value>,
) -> HashMap<String, turso_core::Value> {
let mut row = HashMap::with_capacity(info.column_names.len());
for (i, value) in values.iter().enumerate() {
row.insert(info.column_names[i].clone(), value.clone());
}
row
}
fn create_row_update(
&self,
info: &ReplayInfo,
updates: &Vec<turso_core::Value>,
) -> HashMap<String, turso_core::Value> {
let mut row = HashMap::with_capacity(info.column_names.len());
assert!(updates.len() % 2 == 0);
let columns_cnt = updates.len() / 2;
for (i, value) in updates.iter().take(columns_cnt).enumerate() {
let updated = match value {
turso_core::Value::Integer(x @ (1 | 0)) => *x > 0,
_ => {
panic!("unexpected 'changes' binary record first-half component: {value:?}")
}
};
if !updated {
continue;
}
row.insert(
info.column_names[i].clone(),
updates[columns_cnt + i].clone(),
);
}
row
}
pub fn replay_values(
&self,
info: &ReplayInfo,
@@ -89,9 +170,9 @@ impl DatabaseReplayGenerator {
}
pub async fn replay_info(
&self,
coro: &Coro,
coro: &Coro<Ctx>,
change: &DatabaseTapeRowChange,
) -> Result<Vec<ReplayInfo>> {
) -> Result<ReplayInfo> {
tracing::trace!("replay: change={:?}", change);
let table_name = &change.table_name;
@@ -117,9 +198,10 @@ impl DatabaseReplayGenerator {
change_type: DatabaseChangeType::Delete,
query,
pk_column_indices: None,
column_names: Vec::new(),
is_ddl_replay: true,
};
Ok(vec![delete])
Ok(delete)
}
DatabaseTapeRowChangeType::Insert { after } => {
assert!(after.len() == 5);
@@ -133,9 +215,10 @@ impl DatabaseReplayGenerator {
change_type: DatabaseChangeType::Insert,
query: sql.as_str().to_string(),
pk_column_indices: None,
column_names: Vec::new(),
is_ddl_replay: true,
};
Ok(vec![insert])
Ok(insert)
}
DatabaseTapeRowChangeType::Update { updates, .. } => {
let Some(updates) = updates else {
@@ -155,16 +238,17 @@ impl DatabaseReplayGenerator {
change_type: DatabaseChangeType::Update,
query: ddl_stmt.as_str().to_string(),
pk_column_indices: None,
column_names: Vec::new(),
is_ddl_replay: true,
};
Ok(vec![update])
Ok(update)
}
}
} else {
match &change.change {
DatabaseTapeRowChangeType::Delete { .. } => {
let delete = self.delete_query(coro, table_name).await?;
Ok(vec![delete])
Ok(delete)
}
DatabaseTapeRowChangeType::Update { updates, after, .. } => {
if let Some(updates) = updates {
@@ -178,32 +262,159 @@ impl DatabaseReplayGenerator {
});
}
let update = self.update_query(coro, table_name, &columns).await?;
Ok(vec![update])
Ok(update)
} else {
let delete = self.delete_query(coro, table_name).await?;
let insert = self.insert_query(coro, table_name, after.len()).await?;
Ok(vec![delete, insert])
let columns = [true].repeat(after.len());
let update = self.update_query(coro, table_name, &columns).await?;
Ok(update)
}
}
DatabaseTapeRowChangeType::Insert { after } => {
let insert = self.insert_query(coro, table_name, after.len()).await?;
Ok(vec![insert])
Ok(insert)
}
}
}
}
pub(crate) async fn update_query(
&self,
coro: &Coro,
coro: &Coro<Ctx>,
table_name: &str,
columns: &[bool],
) -> Result<ReplayInfo> {
let (column_names, pk_column_indices) = self.table_columns_info(coro, table_name).await?;
let mut pk_predicates = Vec::with_capacity(1);
let mut column_updates = Vec::with_capacity(1);
for &idx in &pk_column_indices {
pk_predicates.push(format!("{} = ?", column_names[idx]));
}
for (idx, name) in column_names.iter().enumerate() {
if columns[idx as usize] {
column_updates.push(format!("{name} = ?"));
}
}
let (query, pk_column_indices) =
if self.opts.use_implicit_rowid || pk_column_indices.is_empty() {
(
format!(
"UPDATE {table_name} SET {} WHERE rowid = ?",
column_updates.join(", ")
),
None,
)
} else {
(
format!(
"UPDATE {table_name} SET {} WHERE {}",
column_updates.join(", "),
pk_predicates.join(" AND ")
),
Some(pk_column_indices),
)
};
Ok(ReplayInfo {
change_type: DatabaseChangeType::Update,
query,
column_names,
pk_column_indices,
is_ddl_replay: false,
})
}
pub(crate) async fn insert_query(
&self,
coro: &Coro<Ctx>,
table_name: &str,
columns: usize,
) -> Result<ReplayInfo> {
let (mut column_names, pk_column_indices) = self.table_columns_info(coro, table_name).await?;
let conflict_clause = if !pk_column_indices.is_empty() {
let mut pk_column_names = Vec::new();
for &idx in &pk_column_indices {
pk_column_names.push(column_names[idx].clone());
}
let mut update_clauses = Vec::new();
for name in &column_names {
update_clauses.push(format!("{name} = excluded.{name}"));
}
format!(
"ON CONFLICT({}) DO UPDATE SET {}",
pk_column_names.join(","),
update_clauses.join(",")
)
} else {
String::new()
};
if !self.opts.use_implicit_rowid {
let placeholders = ["?"].repeat(columns).join(",");
let query =
format!("INSERT INTO {table_name} VALUES ({placeholders}){conflict_clause}");
return Ok(ReplayInfo {
change_type: DatabaseChangeType::Insert,
query,
pk_column_indices: None,
column_names,
is_ddl_replay: false,
});
};
let original_column_names = column_names.clone();
column_names.push("rowid".to_string());
let placeholders = ["?"].repeat(columns + 1).join(",");
let column_names = column_names.join(", ");
let query = format!("INSERT INTO {table_name}({column_names}) VALUES ({placeholders})");
Ok(ReplayInfo {
change_type: DatabaseChangeType::Insert,
query,
column_names: original_column_names,
pk_column_indices: None,
is_ddl_replay: false,
})
}
pub(crate) async fn delete_query(
&self,
coro: &Coro<Ctx>,
table_name: &str,
) -> Result<ReplayInfo> {
let (column_names, pk_column_indices) = self.table_columns_info(coro, table_name).await?;
let mut pk_predicates = Vec::with_capacity(1);
for &idx in &pk_column_indices {
pk_predicates.push(format!("{} = ?", column_names[idx]));
}
let use_implicit_rowid = self.opts.use_implicit_rowid;
if pk_column_indices.is_empty() || use_implicit_rowid {
let query = format!("DELETE FROM {table_name} WHERE rowid = ?");
tracing::trace!("delete_query: table_name={table_name}, query={query}, use_implicit_rowid={use_implicit_rowid}");
return Ok(ReplayInfo {
change_type: DatabaseChangeType::Delete,
query,
column_names,
pk_column_indices: None,
is_ddl_replay: false,
});
}
let pk_predicates = pk_predicates.join(" AND ");
let query = format!("DELETE FROM {table_name} WHERE {pk_predicates}");
tracing::trace!("delete_query: table_name={table_name}, query={query}, use_implicit_rowid={use_implicit_rowid}");
Ok(ReplayInfo {
change_type: DatabaseChangeType::Delete,
query,
column_names,
pk_column_indices: Some(pk_column_indices),
is_ddl_replay: false,
})
}
async fn table_columns_info(
&self,
coro: &Coro<Ctx>,
table_name: &str,
) -> Result<(Vec<String>, Vec<usize>)> {
let mut table_info_stmt = self.conn.prepare(format!(
"SELECT cid, name, pk FROM pragma_table_info('{table_name}')"
))?;
let mut pk_predicates = Vec::with_capacity(1);
let mut pk_column_indices = Vec::with_capacity(1);
let mut column_updates = Vec::with_capacity(1);
let mut column_names = Vec::new();
while let Some(column) = run_stmt_once(coro, &mut table_info_stmt).await? {
let turso_core::Value::Integer(column_id) = column.get_value(0) else {
return Err(Error::DatabaseTapeError(
@@ -221,118 +432,10 @@ impl DatabaseReplayGenerator {
));
};
if *pk == 1 {
pk_predicates.push(format!("{name} = ?"));
pk_column_indices.push(*column_id as usize);
}
if columns[*column_id as usize] {
column_updates.push(format!("{name} = ?"));
}
column_names.push(name.as_str().to_string());
}
let (query, pk_column_indices) = if self.opts.use_implicit_rowid {
(
format!(
"UPDATE {table_name} SET {} WHERE rowid = ?",
column_updates.join(", ")
),
None,
)
} else {
(
format!(
"UPDATE {table_name} SET {} WHERE {}",
column_updates.join(", "),
pk_predicates.join(" AND ")
),
Some(pk_column_indices),
)
};
Ok(ReplayInfo {
change_type: DatabaseChangeType::Update,
query,
pk_column_indices,
is_ddl_replay: false,
})
}
pub(crate) async fn insert_query(
&self,
coro: &Coro,
table_name: &str,
columns: usize,
) -> Result<ReplayInfo> {
if !self.opts.use_implicit_rowid {
let placeholders = ["?"].repeat(columns).join(",");
let query = format!("INSERT INTO {table_name} VALUES ({placeholders})");
return Ok(ReplayInfo {
change_type: DatabaseChangeType::Insert,
query,
pk_column_indices: None,
is_ddl_replay: false,
});
};
let mut table_info_stmt = self.conn.prepare(format!(
"SELECT name FROM pragma_table_info('{table_name}')"
))?;
let mut column_names = Vec::with_capacity(columns + 1);
while let Some(column) = run_stmt_once(coro, &mut table_info_stmt).await? {
let turso_core::Value::Text(text) = column.get_value(0) else {
return Err(Error::DatabaseTapeError(
"unexpected column type for pragma_table_info query".to_string(),
));
};
column_names.push(text.to_string());
}
column_names.push("rowid".to_string());
let placeholders = ["?"].repeat(columns + 1).join(",");
let column_names = column_names.join(", ");
let query = format!("INSERT INTO {table_name}({column_names}) VALUES ({placeholders})");
Ok(ReplayInfo {
change_type: DatabaseChangeType::Insert,
query,
pk_column_indices: None,
is_ddl_replay: false,
})
}
pub(crate) async fn delete_query(&self, coro: &Coro, table_name: &str) -> Result<ReplayInfo> {
let (query, pk_column_indices) = if self.opts.use_implicit_rowid {
(format!("DELETE FROM {table_name} WHERE rowid = ?"), None)
} else {
let mut pk_info_stmt = self.conn.prepare(format!(
"SELECT cid, name FROM pragma_table_info('{table_name}') WHERE pk = 1"
))?;
let mut pk_predicates = Vec::with_capacity(1);
let mut pk_column_indices = Vec::with_capacity(1);
while let Some(column) = run_stmt_once(coro, &mut pk_info_stmt).await? {
let turso_core::Value::Integer(column_id) = column.get_value(0) else {
return Err(Error::DatabaseTapeError(
"unexpected column type for pragma_table_info query".to_string(),
));
};
let turso_core::Value::Text(name) = column.get_value(1) else {
return Err(Error::DatabaseTapeError(
"unexpected column type for pragma_table_info query".to_string(),
));
};
pk_predicates.push(format!("{name} = ?"));
pk_column_indices.push(*column_id as usize);
}
if pk_column_indices.is_empty() {
(format!("DELETE FROM {table_name} WHERE rowid = ?"), None)
} else {
let pk_predicates = pk_predicates.join(" AND ");
let query = format!("DELETE FROM {table_name} WHERE {pk_predicates}");
(query, Some(pk_column_indices))
}
};
let use_implicit_rowid = self.opts.use_implicit_rowid;
tracing::trace!("delete_query: table_name={table_name}, query={query}, use_implicit_rowid={use_implicit_rowid}");
Ok(ReplayInfo {
change_type: DatabaseChangeType::Delete,
query,
pk_column_indices,
is_ddl_replay: false,
})
Ok((column_names, pk_column_indices))
}
}

961
sync/engine/src/database_sync_engine.rs
View File

File diff suppressed because it is too large Load Diff

1442
sync/engine/src/database_sync_operations.rs
View File

File diff suppressed because it is too large Load Diff

191
sync/engine/src/database_tape.rs
View File

@@ -3,15 +3,15 @@ use std::{
sync::Arc,
};
use turso_core::{types::WalFrameInfo, StepResult};
use turso_core::{types::WalFrameInfo, LimboError, StepResult};
use crate::{
database_replay_generator::{DatabaseReplayGenerator, ReplayInfo},
database_sync_operations::WAL_FRAME_HEADER,
errors::Error,
types::{
Coro, DatabaseChange, DatabaseTapeOperation, DatabaseTapeRowChange,
DatabaseTapeRowChangeType, ProtocolCommand,
Coro, DatabaseChange, DatabaseRowMutation, DatabaseRowStatement, DatabaseTapeOperation,
DatabaseTapeRowChange, DatabaseTapeRowChangeType, ProtocolCommand,
},
wal_session::WalSession,
Result,
@@ -28,7 +28,7 @@ pub struct DatabaseTape {
const DEFAULT_CDC_TABLE_NAME: &str = "turso_cdc";
const DEFAULT_CDC_MODE: &str = "full";
const DEFAULT_CHANGES_BATCH_SIZE: usize = 100;
const CDC_PRAGMA_NAME: &str = "unstable_capture_data_changes_conn";
pub const CDC_PRAGMA_NAME: &str = "unstable_capture_data_changes_conn";
#[derive(Debug, Clone)]
pub struct DatabaseTapeOpts {
@@ -36,8 +36,8 @@ pub struct DatabaseTapeOpts {
pub cdc_mode: Option<String>,
}
pub(crate) async fn run_stmt_once<'a>(
coro: &'_ Coro,
pub(crate) async fn run_stmt_once<'a, Ctx>(
coro: &'_ Coro<Ctx>,
stmt: &'a mut turso_core::Statement,
) -> Result<Option<&'a turso_core::Row>> {
loop {
@@ -61,8 +61,8 @@ pub(crate) async fn run_stmt_once<'a>(
}
}
pub(crate) async fn run_stmt_expect_one_row(
coro: &Coro,
pub(crate) async fn run_stmt_expect_one_row<Ctx>(
coro: &Coro<Ctx>,
stmt: &mut turso_core::Statement,
) -> Result<Option<Vec<turso_core::Value>>> {
let Some(row) = run_stmt_once(coro, stmt).await? else {
@@ -75,15 +75,18 @@ pub(crate) async fn run_stmt_expect_one_row(
Ok(Some(values))
}
pub(crate) async fn run_stmt_ignore_rows(
coro: &Coro,
pub(crate) async fn run_stmt_ignore_rows<Ctx>(
coro: &Coro<Ctx>,
stmt: &mut turso_core::Statement,
) -> Result<()> {
while run_stmt_once(coro, stmt).await?.is_some() {}
Ok(())
}
pub(crate) async fn exec_stmt(coro: &Coro, stmt: &mut turso_core::Statement) -> Result<()> {
pub(crate) async fn exec_stmt<Ctx>(
coro: &Coro<Ctx>,
stmt: &mut turso_core::Statement,
) -> Result<()> {
loop {
match stmt.step()? {
StepResult::IO => {
@@ -128,7 +131,7 @@ impl DatabaseTape {
let connection = self.inner.connect()?;
Ok(connection)
}
pub async fn connect(&self, coro: &Coro) -> Result<Arc<turso_core::Connection>> {
pub async fn connect<Ctx>(&self, coro: &Coro<Ctx>) -> Result<Arc<turso_core::Connection>> {
let connection = self.inner.connect()?;
tracing::debug!("set '{CDC_PRAGMA_NAME}' for new connection");
let mut stmt = connection.prepare(&self.pragma_query)?;
@@ -142,19 +145,20 @@ impl DatabaseTape {
) -> Result<DatabaseChangesIterator> {
tracing::debug!("opening changes iterator with options {:?}", opts);
let conn = self.inner.connect()?;
let query = opts.mode.query(&self.cdc_table, opts.batch_size);
let query_stmt = conn.prepare(&query)?;
Ok(DatabaseChangesIterator {
conn,
cdc_table: self.cdc_table.clone(),
first_change_id: opts.first_change_id,
batch: VecDeque::with_capacity(opts.batch_size),
query_stmt,
query_stmt: None,
txn_boundary_returned: false,
mode: opts.mode,
batch_size: opts.batch_size,
ignore_schema_changes: opts.ignore_schema_changes,
})
}
/// Start raw WAL edit session which can append or rollback pages directly in the current WAL
pub async fn start_wal_session(&self, coro: &Coro) -> Result<DatabaseWalSession> {
pub async fn start_wal_session<Ctx>(&self, coro: &Coro<Ctx>) -> Result<DatabaseWalSession> {
let conn = self.connect(coro).await?;
let mut wal_session = WalSession::new(conn);
wal_session.begin()?;
@@ -162,11 +166,11 @@ impl DatabaseTape {
}
/// Start replay session which can apply [DatabaseTapeOperation] from [Self::iterate_changes]
pub async fn start_replay_session(
pub async fn start_replay_session<Ctx>(
&self,
coro: &Coro,
opts: DatabaseReplaySessionOpts,
) -> Result<DatabaseReplaySession> {
coro: &Coro<Ctx>,
opts: DatabaseReplaySessionOpts<Ctx>,
) -> Result<DatabaseReplaySession<Ctx>> {
tracing::debug!("opening replay session");
let conn = self.connect(coro).await?;
conn.execute("BEGIN IMMEDIATE")?;
@@ -184,12 +188,12 @@ impl DatabaseTape {
pub struct DatabaseWalSession {
page_size: usize,
next_wal_frame_no: u64,
wal_session: WalSession,
pub wal_session: WalSession,
prepared_frame: Option<(u32, Vec<u8>)>,
}
impl DatabaseWalSession {
pub async fn new(coro: &Coro, wal_session: WalSession) -> Result<Self> {
pub async fn new<Ctx>(coro: &Coro<Ctx>, wal_session: WalSession) -> Result<Self> {
let conn = wal_session.conn();
let frames_count = conn.wal_state()?.max_frame;
let mut page_size_stmt = conn.prepare("PRAGMA page_size")?;
@@ -259,13 +263,15 @@ impl DatabaseWalSession {
Ok(())
}
pub fn rollback_changes_after(&mut self, frame_watermark: u64) -> Result<()> {
pub fn rollback_changes_after(&mut self, frame_watermark: u64) -> Result<usize> {
let conn = self.wal_session.conn();
let pages = conn.wal_changed_pages_after(frame_watermark)?;
tracing::info!("rolling back {} pages", pages.len());
let pages_cnt = pages.len();
for page_no in pages {
self.rollback_page(page_no, frame_watermark)?;
}
Ok(())
Ok(pages_cnt)
}
pub fn db_size(&self) -> Result<u32> {
@@ -290,7 +296,7 @@ impl DatabaseWalSession {
frame_info.put_to_frame_header(&mut frame);
let frame_no = self.next_wal_frame_no;
tracing::trace!(
tracing::debug!(
"flush prepared frame {:?} as frame_no {}",
frame_info,
frame_no
@@ -352,17 +358,20 @@ impl Default for DatabaseChangesIteratorOpts {
}
pub struct DatabaseChangesIterator {
query_stmt: turso_core::Statement,
conn: Arc<turso_core::Connection>,
cdc_table: Arc<String>,
query_stmt: Option<turso_core::Statement>,
first_change_id: Option<i64>,
batch: VecDeque<DatabaseTapeRowChange>,
txn_boundary_returned: bool,
mode: DatabaseChangesIteratorMode,
batch_size: usize,
ignore_schema_changes: bool,
}
const SQLITE_SCHEMA_TABLE: &str = "sqlite_schema";
impl DatabaseChangesIterator {
pub async fn next(&mut self, coro: &Coro) -> Result<Option<DatabaseTapeOperation>> {
pub async fn next<Ctx>(&mut self, coro: &Coro<Ctx>) -> Result<Option<DatabaseTapeOperation>> {
if self.batch.is_empty() {
self.refill(coro).await?;
}
@@ -386,15 +395,26 @@ impl DatabaseChangesIterator {
return Ok(next);
}
}
async fn refill(&mut self, coro: &Coro) -> Result<()> {
async fn refill<Ctx>(&mut self, coro: &Coro<Ctx>) -> Result<()> {
if self.query_stmt.is_none() {
let query = self.mode.query(&self.cdc_table, self.batch_size);
let stmt = match self.conn.prepare(&query) {
Ok(stmt) => stmt,
Err(LimboError::ParseError(err)) if err.contains("no such table") => return Ok(()),
Err(err) => return Err(err.into()),
};
self.query_stmt = Some(stmt);
}
let query_stmt = self.query_stmt.as_mut().unwrap();
let change_id_filter = self.first_change_id.unwrap_or(self.mode.first_id());
self.query_stmt.reset();
self.query_stmt.bind_at(
query_stmt.reset();
query_stmt.bind_at(
1.try_into().unwrap(),
turso_core::Value::Integer(change_id_filter),
);
while let Some(row) = run_stmt_once(coro, &mut self.query_stmt).await? {
while let Some(row) = run_stmt_once(coro, query_stmt).await? {
let database_change: DatabaseChange = row.try_into()?;
let tape_change = match self.mode {
DatabaseChangesIteratorMode::Apply => database_change.into_apply()?,
@@ -410,43 +430,59 @@ impl DatabaseChangesIterator {
}
}
#[derive(Debug, Clone)]
pub struct DatabaseReplaySessionOpts {
#[derive(Clone)]
pub struct DatabaseReplaySessionOpts<Ctx = ()> {
pub use_implicit_rowid: bool,
pub transform: Option<
Arc<dyn Fn(&Ctx, DatabaseRowMutation) -> Result<Option<DatabaseRowStatement>> + 'static>,
>,
}
struct CachedStmt {
impl<Ctx> std::fmt::Debug for DatabaseReplaySessionOpts<Ctx> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DatabaseReplaySessionOpts")
.field("use_implicit_rowid", &self.use_implicit_rowid)
.field("transform_mutation.is_some()", &self.transform.is_some())
.finish()
}
}
pub(crate) struct CachedStmt {
stmt: turso_core::Statement,
info: ReplayInfo,
}
pub struct DatabaseReplaySession {
conn: Arc<turso_core::Connection>,
cached_delete_stmt: HashMap<String, CachedStmt>,
cached_insert_stmt: HashMap<(String, usize), CachedStmt>,
cached_update_stmt: HashMap<(String, Vec<bool>), CachedStmt>,
in_txn: bool,
generator: DatabaseReplayGenerator,
pub struct DatabaseReplaySession<Ctx = ()> {
pub(crate) conn: Arc<turso_core::Connection>,
pub(crate) cached_delete_stmt: HashMap<String, CachedStmt>,
pub(crate) cached_insert_stmt: HashMap<(String, usize), CachedStmt>,
pub(crate) cached_update_stmt: HashMap<(String, Vec<bool>), CachedStmt>,
pub(crate) in_txn: bool,
pub(crate) generator: DatabaseReplayGenerator<Ctx>,
}
async fn replay_stmt(
coro: &Coro,
cached: &mut CachedStmt,
async fn replay_stmt<Ctx>(
coro: &Coro<Ctx>,
stmt: &mut turso_core::Statement,
values: Vec<turso_core::Value>,
) -> Result<()> {
cached.stmt.reset();
stmt.reset();
for (i, value) in values.into_iter().enumerate() {
cached.stmt.bind_at((i + 1).try_into().unwrap(), value);
stmt.bind_at((i + 1).try_into().unwrap(), value);
}
exec_stmt(coro, &mut cached.stmt).await?;
exec_stmt(coro, stmt).await?;
Ok(())
}
impl DatabaseReplaySession {
impl<Ctx> DatabaseReplaySession<Ctx> {
pub fn conn(&self) -> Arc<turso_core::Connection> {
self.conn.clone()
}
pub async fn replay(&mut self, coro: &Coro, operation: DatabaseTapeOperation) -> Result<()> {
pub async fn replay(
&mut self,
coro: &Coro<Ctx>,
operation: DatabaseTapeOperation,
) -> Result<()> {
match operation {
DatabaseTapeOperation::Commit => {
tracing::debug!("replay: commit replayed changes after transaction boundary");
@@ -466,10 +502,23 @@ impl DatabaseReplaySession {
if table == SQLITE_SCHEMA_TABLE {
let replay_info = self.generator.replay_info(coro, &change).await?;
for replay in &replay_info {
self.conn.execute(replay.query.as_str())?;
}
self.conn.execute(replay_info.query.as_str())?;
} else {
if let Some(transform) = &self.generator.opts.transform {
let replay_info = self.generator.replay_info(coro, &change).await?;
let mutation = self.generator.create_mutation(&replay_info, &change)?;
let statement = transform(&coro.ctx.borrow(), mutation)?;
if let Some(statement) = statement {
tracing::info!(
"replay: use mutation from custom transformer: sql={}, values={:?}",
statement.sql,
statement.values
);
let mut stmt = self.conn.prepare(&statement.sql)?;
replay_stmt(coro, &mut stmt, statement.values).await?;
return Ok(());
}
}
match change.change {
DatabaseTapeRowChangeType::Delete { before } => {
let key = self.populate_delete_stmt(coro, table).await?;
@@ -486,7 +535,7 @@ impl DatabaseReplaySession {
before,
None,
);
replay_stmt(coro, cached, values).await?;
replay_stmt(coro, &mut cached.stmt, values).await?;
}
DatabaseTapeRowChangeType::Insert { after } => {
let key = self.populate_insert_stmt(coro, table, after.len()).await?;
@@ -503,7 +552,7 @@ impl DatabaseReplaySession {
after,
None,
);
replay_stmt(coro, cached, values).await?;
replay_stmt(coro, &mut cached.stmt, values).await?;
}
DatabaseTapeRowChangeType::Update {
after,
@@ -533,7 +582,7 @@ impl DatabaseReplaySession {
after,
Some(updates),
);
replay_stmt(coro, cached, values).await?;
replay_stmt(coro, &mut cached.stmt, values).await?;
}
DatabaseTapeRowChangeType::Update {
before,
@@ -554,7 +603,7 @@ impl DatabaseReplaySession {
before,
None,
);
replay_stmt(coro, cached, values).await?;
replay_stmt(coro, &mut cached.stmt, values).await?;
let key = self.populate_insert_stmt(coro, table, after.len()).await?;
tracing::trace!(
@@ -570,7 +619,7 @@ impl DatabaseReplaySession {
after,
None,
);
replay_stmt(coro, cached, values).await?;
replay_stmt(coro, &mut cached.stmt, values).await?;
}
}
}
@@ -578,7 +627,11 @@ impl DatabaseReplaySession {
}
Ok(())
}
async fn populate_delete_stmt<'a>(&mut self, coro: &Coro, table: &'a str) -> Result<&'a str> {
async fn populate_delete_stmt<'a>(
&mut self,
coro: &Coro<Ctx>,
table: &'a str,
) -> Result<&'a str> {
if self.cached_delete_stmt.contains_key(table) {
return Ok(table);
}
@@ -591,7 +644,7 @@ impl DatabaseReplaySession {
}
async fn populate_insert_stmt(
&mut self,
coro: &Coro,
coro: &Coro<Ctx>,
table: &str,
columns: usize,
) -> Result<(String, usize)> {
@@ -612,7 +665,7 @@ impl DatabaseReplaySession {
}
async fn populate_update_stmt(
&mut self,
coro: &Coro,
coro: &Coro<Ctx>,
table: &str,
columns: &[bool],
) -> Result<(String, Vec<bool>)> {
@@ -639,7 +692,7 @@ mod tests {
database_tape::{
run_stmt_once, DatabaseChangesIteratorOpts, DatabaseReplaySessionOpts, DatabaseTape,
},
types::{DatabaseTapeOperation, DatabaseTapeRowChange, DatabaseTapeRowChangeType},
types::{Coro, DatabaseTapeOperation, DatabaseTapeRowChange, DatabaseTapeRowChangeType},
};
#[test]
@@ -653,6 +706,7 @@ mod tests {
let mut gen = genawaiter::sync::Gen::new({
let db1 = db1.clone();
|coro| async move {
let coro: Coro<()> = coro.into();
let conn = db1.connect(&coro).await.unwrap();
let mut stmt = conn.prepare("SELECT * FROM turso_cdc").unwrap();
let mut rows = Vec::new();
@@ -683,6 +737,7 @@ mod tests {
let mut gen = genawaiter::sync::Gen::new({
let db1 = db1.clone();
|coro| async move {
let coro: Coro<()> = coro.into();
let conn = db1.connect(&coro).await.unwrap();
conn.execute("CREATE TABLE t(x)").unwrap();
conn.execute("INSERT INTO t VALUES (1), (2), (3)").unwrap();
@@ -754,6 +809,7 @@ mod tests {
let db1 = db1.clone();
let db2 = db2.clone();
|coro| async move {
let coro: Coro<()> = coro.into();
let conn1 = db1.connect(&coro).await.unwrap();
conn1.execute("CREATE TABLE t(x)").unwrap();
conn1
@@ -768,6 +824,7 @@ mod tests {
{
let opts = DatabaseReplaySessionOpts {
use_implicit_rowid: true,
transform: None,
};
let mut session = db2.start_replay_session(&coro, opts).await.unwrap();
let opts = Default::default();
@@ -832,6 +889,7 @@ mod tests {
let db1 = db1.clone();
let db2 = db2.clone();
|coro| async move {
let coro: Coro<()> = coro.into();
let conn1 = db1.connect(&coro).await.unwrap();
conn1.execute("CREATE TABLE t(x)").unwrap();
conn1
@@ -846,6 +904,7 @@ mod tests {
{
let opts = DatabaseReplaySessionOpts {
use_implicit_rowid: false,
transform: None,
};
let mut session = db2.start_replay_session(&coro, opts).await.unwrap();
let opts = Default::default();
@@ -904,6 +963,7 @@ mod tests {
let db1 = db1.clone();
let db2 = db2.clone();
|coro| async move {
let coro: Coro<()> = coro.into();
let conn1 = db1.connect(&coro).await.unwrap();
conn1.execute("CREATE TABLE t(x TEXT PRIMARY KEY)").unwrap();
conn1.execute("INSERT INTO t(x) VALUES ('a')").unwrap();
@@ -915,6 +975,7 @@ mod tests {
{
let opts = DatabaseReplaySessionOpts {
use_implicit_rowid: false,
transform: None,
};
let mut session = db2.start_replay_session(&coro, opts).await.unwrap();
let opts = Default::default();
@@ -969,6 +1030,7 @@ mod tests {
let mut gen = genawaiter::sync::Gen::new({
|coro| async move {
let coro: Coro<()> = coro.into();
let conn1 = db1.connect(&coro).await.unwrap();
conn1
.execute("CREATE TABLE t(x TEXT PRIMARY KEY, y)")
@@ -988,6 +1050,7 @@ mod tests {
{
let opts = DatabaseReplaySessionOpts {
use_implicit_rowid: false,
transform: None,
};
let mut session = db3.start_replay_session(&coro, opts).await.unwrap();
@@ -1094,6 +1157,7 @@ mod tests {
let mut gen = genawaiter::sync::Gen::new({
|coro| async move {
let coro: Coro<()> = coro.into();
let conn1 = db1.connect(&coro).await.unwrap();
conn1
.execute("CREATE TABLE t(x TEXT PRIMARY KEY, y)")
@@ -1104,6 +1168,7 @@ mod tests {
{
let opts = DatabaseReplaySessionOpts {
use_implicit_rowid: false,
transform: None,
};
let mut session = db2.start_replay_session(&coro, opts).await.unwrap();
@@ -1177,6 +1242,7 @@ mod tests {
let mut gen = genawaiter::sync::Gen::new({
|coro| async move {
let coro: Coro<()> = coro.into();
let conn1 = db1.connect(&coro).await.unwrap();
conn1
.execute("CREATE TABLE t(x TEXT PRIMARY KEY, y)")
@@ -1188,6 +1254,7 @@ mod tests {
{
let opts = DatabaseReplaySessionOpts {
use_implicit_rowid: false,
transform: None,
};
let mut session = db2.start_replay_session(&coro, opts).await.unwrap();
@@ -1255,6 +1322,7 @@ mod tests {
let mut gen = genawaiter::sync::Gen::new({
|coro| async move {
let coro: Coro<()> = coro.into();
let conn1 = db1.connect(&coro).await.unwrap();
conn1
.execute("CREATE TABLE t(x TEXT PRIMARY KEY, y, z)")
@@ -1283,6 +1351,7 @@ mod tests {
{
let opts = DatabaseReplaySessionOpts {
use_implicit_rowid: false,
transform: None,
};
let mut session = db3.start_replay_session(&coro, opts).await.unwrap();

2
sync/engine/src/errors.rs
View File

@@ -10,6 +10,8 @@ pub enum Error {
DatabaseSyncEngineError(String),
#[error("database sync engine conflict: {0}")]
DatabaseSyncEngineConflict(String),
#[error("database sync engine IO error: {0}")]
IoError(#[from] std::io::Error),
}
#[cfg(test)]

8
sync/engine/src/io_operations.rs
View File

@@ -12,9 +12,9 @@ pub trait IoOperations {
fn open_tape(&self, path: &str, capture: bool) -> Result<DatabaseTape>;
fn try_open(&self, path: &str) -> Result<Option<Arc<dyn turso_core::File>>>;
fn create(&self, path: &str) -> Result<Arc<dyn turso_core::File>>;
fn truncate(
fn truncate<Ctx>(
&self,
coro: &Coro,
coro: &Coro<Ctx>,
file: Arc<dyn turso_core::File>,
len: usize,
) -> impl std::future::Future<Output = Result<()>>;
@@ -47,9 +47,9 @@ impl IoOperations for Arc<dyn turso_core::IO> {
}
}
async fn truncate(
async fn truncate<Ctx>(
&self,
coro: &Coro,
coro: &Coro<Ctx>,
file: Arc<dyn turso_core::File>,
len: usize,
) -> Result<()> {

9
sync/engine/src/protocol_io.rs
View File

@@ -15,6 +15,11 @@ pub trait ProtocolIO {
type DataCompletion: DataCompletion;
fn full_read(&self, path: &str) -> Result<Self::DataCompletion>;
fn full_write(&self, path: &str, content: Vec<u8>) -> Result<Self::DataCompletion>;
fn http(&self, method: &str, path: &str, body: Option<Vec<u8>>)
-> Result<Self::DataCompletion>;
fn http(
&self,
method: &str,
path: &str,
body: Option<Vec<u8>>,
headers: &[(&str, &str)],
) -> Result<Self::DataCompletion>;
}

183
sync/engine/src/server_proto.rs
View File

@@ -3,6 +3,64 @@ use std::collections::VecDeque;
use bytes::Bytes;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
#[serde(rename_all = "snake_case")]
#[derive(prost::Enumeration)]
#[repr(i32)]
pub enum PageUpdatesEncodingReq {
Raw = 0,
Zstd = 1,
}
#[derive(prost::Message)]
pub struct PullUpdatesReqProtoBody {
#[prost(enumeration = "PageUpdatesEncodingReq", tag = "1")]
pub encoding: i32,
#[prost(string, tag = "2")]
pub server_revision: String,
#[prost(string, tag = "3")]
pub client_revision: String,
#[prost(uint32, tag = "4")]
pub long_poll_timeout_ms: u32,
#[prost(bytes, tag = "5")]
pub server_pages: Bytes,
#[prost(bytes, tag = "6")]
pub client_pages: Bytes,
}
#[derive(prost::Message, Serialize, Deserialize, Clone, Eq, PartialEq)]
pub struct PageData {
#[prost(uint64, tag = "1")]
pub page_id: u64,
#[serde(with = "bytes_as_base64_pad")]
#[prost(bytes, tag = "2")]
pub encoded_page: Bytes,
}
#[derive(prost::Message)]
pub struct PageSetRawEncodingProto {}
#[derive(prost::Message)]
pub struct PageSetZstdEncodingProto {
#[prost(int32, tag = "1")]
pub level: i32,
#[prost(uint32, repeated, tag = "2")]
pub pages_dict: Vec<u32>,
}
#[derive(prost::Message)]
pub struct PullUpdatesRespProtoBody {
#[prost(string, tag = "1")]
pub server_revision: String,
#[prost(uint64, tag = "2")]
pub db_size: u64,
#[prost(optional, message, tag = "3")]
pub raw_encoding: Option<PageSetRawEncodingProto>,
#[prost(optional, message, tag = "4")]
pub zstd_encoding: Option<PageSetZstdEncodingProto>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct PipelineReqBody {
pub baton: Option<String>,
@@ -22,8 +80,6 @@ pub enum StreamRequest {
#[serde(skip_deserializing)]
#[default]
None,
/// See [`CloseStreamReq`]
Close(CloseStreamReq),
/// See [`ExecuteStreamReq`]
Execute(ExecuteStreamReq),
}
@@ -33,15 +89,53 @@ pub enum StreamRequest {
pub enum StreamResult {
#[default]
None,
Ok,
Ok {
response: StreamResponse,
},
Error {
error: Error,
},
}
#[derive(Serialize, Deserialize, Debug)]
/// A request to close the current stream.
pub struct CloseStreamReq {}
#[derive(Serialize, Deserialize, Debug, PartialEq)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum StreamResponse {
Execute(ExecuteStreamResp),
}
#[derive(Serialize, Deserialize, Debug, PartialEq)]
/// A response to a [`ExecuteStreamReq`].
pub struct ExecuteStreamResp {
pub result: StmtResult,
}
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Default)]
pub struct StmtResult {
pub cols: Vec<Col>,
pub rows: Vec<Row>,
pub affected_row_count: u64,
#[serde(with = "option_i64_as_str")]
pub last_insert_rowid: Option<i64>,
#[serde(default, with = "option_u64_as_str")]
pub replication_index: Option<u64>,
#[serde(default)]
pub rows_read: u64,
#[serde(default)]
pub rows_written: u64,
#[serde(default)]
pub query_duration_ms: f64,
}
#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)]
pub struct Col {
pub name: Option<String>,
pub decltype: Option<String>,
}
#[derive(Clone, Deserialize, Serialize, Debug, PartialEq)]
#[serde(transparent)]
pub struct Row {
pub values: Vec<Value>,
}
#[derive(Serialize, Deserialize, Debug)]
/// A request to execute a single SQL statement.
@@ -229,3 +323,80 @@ pub(crate) mod bytes_as_base64 {
Ok(Bytes::from(bytes))
}
}
mod option_i64_as_str {
use serde::de::{Error, Visitor};
use serde::{ser, Deserializer, Serialize as _};
pub fn serialize<S: ser::Serializer>(value: &Option<i64>, ser: S) -> Result<S::Ok, S::Error> {
value.map(|v| v.to_string()).serialize(ser)
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Option<i64>, D::Error> {
struct V;
impl<'de> Visitor<'de> for V {
type Value = Option<i64>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a string representing a signed integer, or null")
}
fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_any(V)
}
fn visit_none<E>(self) -> Result<Self::Value, E>
where
E: Error,
{
Ok(None)
}
fn visit_unit<E>(self) -> Result<Self::Value, E>
where
E: Error,
{
Ok(None)
}
fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
where
E: Error,
{
Ok(Some(v))
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: Error,
{
v.parse().map_err(E::custom).map(Some)
}
}
d.deserialize_option(V)
}
}
pub(crate) mod bytes_as_base64_pad {
use base64::{engine::general_purpose::STANDARD, Engine as _};
use bytes::Bytes;
use serde::{de, ser};
use serde::{de::Error as _, Serialize as _};
pub fn serialize<S: ser::Serializer>(value: &Bytes, ser: S) -> Result<S::Ok, S::Error> {
STANDARD.encode(value).serialize(ser)
}
pub fn deserialize<'de, D: de::Deserializer<'de>>(de: D) -> Result<Bytes, D::Error> {
let text = <&'de str as de::Deserialize>::deserialize(de)?;
let bytes = STANDARD.decode(text).map_err(|_| {
D::Error::invalid_value(de::Unexpected::Str(text), &"binary data encoded as base64")
})?;
Ok(Bytes::from(bytes))
}
}

84
sync/engine/src/types.rs
View File

@@ -1,8 +1,36 @@
use std::{cell::RefCell, collections::HashMap};
use serde::{Deserialize, Serialize};
use crate::{errors::Error, Result};
pub type Coro = genawaiter::sync::Co<ProtocolCommand, Result<()>>;
pub struct Coro<Ctx> {
pub ctx: RefCell<Ctx>,
gen: genawaiter::sync::Co<ProtocolCommand, Result<Ctx>>,
}
impl<Ctx> Coro<Ctx> {
pub fn new(ctx: Ctx, gen: genawaiter::sync::Co<ProtocolCommand, Result<Ctx>>) -> Self {
Self {
ctx: RefCell::new(ctx),
gen,
}
}
pub async fn yield_(&self, value: ProtocolCommand) -> Result<()> {
let ctx = self.gen.yield_(value).await?;
self.ctx.replace(ctx);
Ok(())
}
}
impl From<genawaiter::sync::Co<ProtocolCommand, Result<()>>> for Coro<()> {
fn from(value: genawaiter::sync::Co<ProtocolCommand, Result<()>>) -> Self {
Self {
gen: value,
ctx: RefCell::new(()),
}
}
}
#[derive(Debug, Deserialize, Serialize)]
pub struct DbSyncInfo {
@@ -17,6 +45,17 @@ pub struct DbSyncStatus {
pub max_frame_no: u64,
}
#[derive(Debug)]
pub struct DbChangesStatus {
pub revision: DatabasePullRevision,
pub file_path: String,
}
pub struct SyncEngineStats {
pub cdc_operations: i64,
pub wal_size: i64,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum DatabaseChangeType {
Delete,
@@ -29,12 +68,30 @@ pub struct DatabaseMetadata {
/// Unique identifier of the client - generated on sync startup
pub client_unique_id: String,
/// Latest generation from remote which was pulled locally to the Synced DB
pub synced_generation: u64,
/// Latest frame number from remote which was pulled locally to the Synced DB
pub synced_frame_no: Option<u64>,
pub synced_revision: Option<DatabasePullRevision>,
/// pair of frame_no for Draft and Synced DB such that content of the database file up to these frames is identical
pub draft_wal_match_watermark: u64,
pub synced_wal_match_watermark: u64,
pub revert_since_wal_salt: Option<Vec<u32>>,
pub revert_since_wal_watermark: u64,
pub last_pushed_pull_gen_hint: i64,
pub last_pushed_change_id_hint: i64,
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum DatabasePullRevision {
Legacy {
generation: u64,
synced_frame_no: Option<u64>,
},
V1 {
revision: String,
},
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
pub enum DatabaseSyncEngineProtocolVersion {
Legacy,
V1,
}
impl DatabaseMetadata {
@@ -199,6 +256,21 @@ impl TryFrom<&turso_core::Row> for DatabaseChange {
}
}
pub struct DatabaseRowMutation {
pub change_time: u64,
pub table_name: String,
pub id: i64,
pub change_type: DatabaseChangeType,
pub before: Option<HashMap<String, turso_core::Value>>,
pub after: Option<HashMap<String, turso_core::Value>>,
pub updates: Option<HashMap<String, turso_core::Value>>,
}
pub struct DatabaseRowStatement {
pub sql: String,
pub values: Vec<turso_core::Value>,
}
pub enum DatabaseTapeRowChangeType {
Delete {
before: Vec<turso_core::Value>,

6
sync/engine/src/wal_session.rs
View File

@@ -38,9 +38,9 @@ impl WalSession {
let info = self.conn.wal_get_frame(frame_no, frame)?;
Ok(info)
}
pub fn end(&mut self) -> Result<()> {
pub fn end(&mut self, force_commit: bool) -> Result<()> {
assert!(self.in_txn);
self.conn.wal_insert_end(false)?;
self.conn.wal_insert_end(force_commit)?;
self.in_txn = false;
Ok(())
}
@@ -53,7 +53,7 @@ impl Drop for WalSession {
fn drop(&mut self) {
if self.in_txn {
let _ = self
.end()
.end(false)
.inspect_err(|e| tracing::error!("failed to close WAL session: {}", e));
}
}