aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2025-12-25 20:14:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

improve sync engine

Browse Source
This commit is contained in:
Nikita Sivukhin
2025-08-27 15:29:30 +04:00
parent 30c5473151
commit 009aa479bf
21 changed files with 2482 additions and 1389 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
sync/javascript/index.d.ts vendored
View File

@@ -67,6 +67,14 @@ export declare class Database {
* `Ok(())` if the database is closed successfully.
*/
close(): void
/**
* Sets the default safe integers mode for all statements from this database.
*
* # Arguments
*
* * `toggle` - Whether to use safe integers by default.
*/
defaultSafeIntegers(toggle?: boolean | undefined | null): void
/** Runs the I/O loop synchronously. */
ioLoopSync(): void
/** Runs the I/O loop asynchronously, returning a Promise. */
@@ -107,11 +115,22 @@ export declare class Statement {
raw(raw?: boolean | undefined | null): void
/** Sets the presentation mode to pluck. */
pluck(pluck?: boolean | undefined | null): void
/**
* Sets safe integers mode for this statement.
*
* # Arguments
*
* * `toggle` - Whether to use safe integers.
*/
safeIntegers(toggle?: boolean | undefined | null): void
/** Get column information for the statement */
columns(): unknown[]
/** Finalizes the statement. */
finalize(): void
}
export declare class GeneratorHolder {
resume(error?: string | undefined | null): number
take(): GeneratorResponse | null
}
export declare class JsDataCompletion {
@@ -143,16 +162,42 @@ export declare class SyncEngine {
protocolIo(): JsProtocolRequestData | null
sync(): GeneratorHolder
push(): GeneratorHolder
stats(): GeneratorHolder
pull(): GeneratorHolder
checkpoint(): GeneratorHolder
open(): Database
}
export declare const enum DatabaseChangeTypeJs {
Insert = 0,
Update = 1,
Delete = 2
}
export interface DatabaseOpts {
path: string
}
export interface DatabaseRowMutationJs {
changeTime: number
tableName: string
id: number
changeType: DatabaseChangeTypeJs
before?: Record<string, any>
after?: Record<string, any>
updates?: Record<string, any>
}
export interface DatabaseRowStatementJs {
sql: string
values: Array<any>
}
export type GeneratorResponse =
| { type: 'SyncEngineStats', operations: number, wal: number }
export type JsProtocolRequest =
| { type: 'Http', method: string, path: string, body?: Buffer }
| { type: 'Http', method: string, path: string, body?: Array<number>, headers: Array<[string, string]> }
| { type: 'FullRead', path: string }
| { type: 'FullWrite', path: string, content: Array<number> }
@@ -160,5 +205,13 @@ export interface SyncEngineOpts {
path: string
clientName?: string
walPullBatchSize?: number
enableTracing?: boolean
enableTracing?: string
tablesIgnore?: Array<string>
transform?: (arg: DatabaseRowMutationJs) => DatabaseRowStatementJs | null
protocolVersion?: SyncEngineProtocolVersion
}
export declare const enum SyncEngineProtocolVersion {
Legacy = 0,
V1 = 1
}

171
sync/javascript/index.js
View File

@@ -79,12 +79,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-android-arm64')
const bindingPackageVersion = require('@tursodatabase/sync-android-arm64/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-android-arm64')
} catch (e) {
loadErrors.push(e)
}
@@ -95,12 +90,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-android-arm-eabi')
const bindingPackageVersion = require('@tursodatabase/sync-android-arm-eabi/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-android-arm-eabi')
} catch (e) {
loadErrors.push(e)
}
@@ -115,12 +105,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-win32-x64-msvc')
const bindingPackageVersion = require('@tursodatabase/sync-win32-x64-msvc/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-win32-x64-msvc')
} catch (e) {
loadErrors.push(e)
}
@@ -131,12 +116,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-win32-ia32-msvc')
const bindingPackageVersion = require('@tursodatabase/sync-win32-ia32-msvc/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-win32-ia32-msvc')
} catch (e) {
loadErrors.push(e)
}
@@ -147,12 +127,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-win32-arm64-msvc')
const bindingPackageVersion = require('@tursodatabase/sync-win32-arm64-msvc/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-win32-arm64-msvc')
} catch (e) {
loadErrors.push(e)
}
@@ -166,12 +141,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-darwin-universal')
const bindingPackageVersion = require('@tursodatabase/sync-darwin-universal/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-darwin-universal')
} catch (e) {
loadErrors.push(e)
}
@@ -182,12 +152,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-darwin-x64')
const bindingPackageVersion = require('@tursodatabase/sync-darwin-x64/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-darwin-x64')
} catch (e) {
loadErrors.push(e)
}
@@ -198,12 +163,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-darwin-arm64')
const bindingPackageVersion = require('@tursodatabase/sync-darwin-arm64/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-darwin-arm64')
} catch (e) {
loadErrors.push(e)
}
@@ -218,12 +178,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-freebsd-x64')
const bindingPackageVersion = require('@tursodatabase/sync-freebsd-x64/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-freebsd-x64')
} catch (e) {
loadErrors.push(e)
}
@@ -234,12 +189,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-freebsd-arm64')
const bindingPackageVersion = require('@tursodatabase/sync-freebsd-arm64/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-freebsd-arm64')
} catch (e) {
loadErrors.push(e)
}
@@ -255,12 +205,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-x64-musl')
const bindingPackageVersion = require('@tursodatabase/sync-linux-x64-musl/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-x64-musl')
} catch (e) {
loadErrors.push(e)
}
@@ -271,12 +216,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-x64-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-x64-gnu/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-x64-gnu')
} catch (e) {
loadErrors.push(e)
}
@@ -289,12 +229,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-arm64-musl')
const bindingPackageVersion = require('@tursodatabase/sync-linux-arm64-musl/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-arm64-musl')
} catch (e) {
loadErrors.push(e)
}
@@ -305,12 +240,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-arm64-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-arm64-gnu/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-arm64-gnu')
} catch (e) {
loadErrors.push(e)
}
@@ -323,12 +253,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-arm-musleabihf')
const bindingPackageVersion = require('@tursodatabase/sync-linux-arm-musleabihf/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-arm-musleabihf')
} catch (e) {
loadErrors.push(e)
}
@@ -339,12 +264,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-arm-gnueabihf')
const bindingPackageVersion = require('@tursodatabase/sync-linux-arm-gnueabihf/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-arm-gnueabihf')
} catch (e) {
loadErrors.push(e)
}
@@ -357,12 +277,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-riscv64-musl')
const bindingPackageVersion = require('@tursodatabase/sync-linux-riscv64-musl/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-riscv64-musl')
} catch (e) {
loadErrors.push(e)
}
@@ -373,12 +288,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-riscv64-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-riscv64-gnu/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-riscv64-gnu')
} catch (e) {
loadErrors.push(e)
}
@@ -390,12 +300,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-ppc64-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-ppc64-gnu/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-ppc64-gnu')
} catch (e) {
loadErrors.push(e)
}
@@ -406,12 +311,7 @@ function requireNative() {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-s390x-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-s390x-gnu/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-s390x-gnu')
} catch (e) {
loadErrors.push(e)
}
@@ -421,49 +321,34 @@ function requireNative() {
} else if (process.platform === 'openharmony') {
if (process.arch === 'arm64') {
try {
return require('./turso-sync-js.openharmony-arm64.node')
return require('./turso-sync-js.linux-arm64-ohos.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-openharmony-arm64')
const bindingPackageVersion = require('@tursodatabase/sync-openharmony-arm64/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-arm64-ohos')
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'x64') {
try {
return require('./turso-sync-js.openharmony-x64.node')
return require('./turso-sync-js.linux-x64-ohos.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-openharmony-x64')
const bindingPackageVersion = require('@tursodatabase/sync-openharmony-x64/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-x64-ohos')
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'arm') {
try {
return require('./turso-sync-js.openharmony-arm.node')
return require('./turso-sync-js.linux-arm-ohos.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-openharmony-arm')
const bindingPackageVersion = require('@tursodatabase/sync-openharmony-arm/package.json').version
if (bindingPackageVersion !== '0.1.4-pre.5') {
throw new Error(`Native binding package version mismatch, expected 0.1.4-pre.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
return require('@tursodatabase/sync-linux-arm-ohos')
} catch (e) {
loadErrors.push(e)
}
@@ -508,7 +393,7 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`)
}
const { Database, Statement, GeneratorHolder, JsDataCompletion, JsDataPollResult, JsProtocolIo, JsProtocolRequestData, SyncEngine } = nativeBinding
const { Database, Statement, GeneratorHolder, JsDataCompletion, JsDataPollResult, JsProtocolIo, JsProtocolRequestData, SyncEngine, DatabaseChangeTypeJs, SyncEngineProtocolVersion } = nativeBinding
export { Database }
export { Statement }
export { GeneratorHolder }
@@ -517,3 +402,5 @@ export { JsDataPollResult }
export { JsProtocolIo }
export { JsProtocolRequestData }
export { SyncEngine }
export { DatabaseChangeTypeJs }
export { SyncEngineProtocolVersion }

2
sync/javascript/package.json
View File

@@ -59,4 +59,4 @@
"dependencies": {
"@tursodatabase/database": "~0.1.4-pre.5"
}
}
}

28
sync/javascript/src/generator.rs
View File

@@ -1,5 +1,9 @@
use napi::Env;
use napi_derive::napi;
use std::{future::Future, sync::Mutex};
use std::{
future::Future,
sync::{Arc, Mutex},
};
use turso_sync_engine::types::ProtocolCommand;
@@ -7,18 +11,18 @@ pub const GENERATOR_RESUME_IO: u32 = 0;
pub const GENERATOR_RESUME_DONE: u32 = 1;
pub trait Generator {
fn resume(&mut self, result: Option<String>) -> napi::Result<u32>;
fn resume(&mut self, env: Env, result: Option<String>) -> napi::Result<u32>;
}
impl<F: Future<Output = turso_sync_engine::Result<()>>> Generator
for genawaiter::sync::Gen<ProtocolCommand, turso_sync_engine::Result<()>, F>
for genawaiter::sync::Gen<ProtocolCommand, turso_sync_engine::Result<Env>, F>
{
fn resume(&mut self, error: Option<String>) -> napi::Result<u32> {
fn resume(&mut self, env: Env, error: Option<String>) -> napi::Result<u32> {
let result = match error {
Some(err) => Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
format!("JsProtocolIo error: {err}"),
)),
None => Ok(()),
None => Ok(env),
};
match self.resume_with(result) {
genawaiter::GeneratorState::Yielded(ProtocolCommand::IO) => Ok(GENERATOR_RESUME_IO),
@@ -31,15 +35,25 @@ impl<F: Future<Output = turso_sync_engine::Result<()>>> Generator
}
}
#[napi(discriminant = "type")]
pub enum GeneratorResponse {
SyncEngineStats { operations: i64, wal: i64 },
}
#[napi]
pub struct GeneratorHolder {
pub(crate) inner: Box<Mutex<dyn Generator>>,
pub(crate) response: Arc<Mutex<Option<GeneratorResponse>>>,
}
#[napi]
impl GeneratorHolder {
#[napi]
pub fn resume(&self, error: Option<String>) -> napi::Result<u32> {
self.inner.lock().unwrap().resume(error)
pub fn resume(&self, env: Env, error: Option<String>) -> napi::Result<u32> {
self.inner.lock().unwrap().resume(env, error)
}
#[napi]
pub fn take(&self) -> Option<GeneratorResponse> {
self.response.lock().unwrap().take()
}
}

6
sync/javascript/src/js_protocol_io.rs
View File

@@ -15,6 +15,7 @@ pub enum JsProtocolRequest {
method: String,
path: String,
body: Option<Vec<u8>>,
headers: Vec<(String, String)>,
},
FullRead {
path: String,
@@ -130,11 +131,16 @@ impl ProtocolIO for JsProtocolIo {
method: &str,
path: &str,
body: Option<Vec<u8>>,
headers: &[(&str, &str)],
) -> turso_sync_engine::Result<JsDataCompletion> {
Ok(self.add_request(JsProtocolRequest::Http {
method: method.to_string(),
path: path.to_string(),
body,
headers: headers
.iter()
.map(|x| (x.0.to_string(), x.1.to_string()))
.collect(),
}))
}

286
sync/javascript/src/lib.rs
View File

@@ -3,19 +3,28 @@
pub mod generator;
pub mod js_protocol_io;
use std::sync::{Arc, Mutex, OnceLock};
use std::{
collections::HashMap,
sync::{Arc, Mutex, OnceLock, RwLock, RwLockReadGuard, RwLockWriteGuard},
};
use napi::bindgen_prelude::AsyncTask;
use napi::{
bindgen_prelude::{AsyncTask, Either5, Function, FunctionRef, Null},
Env,
};
use napi_derive::napi;
use tracing_subscriber::{filter::LevelFilter, fmt::format::FmtSpan};
use turso_node::IoLoopTask;
use turso_sync_engine::{
database_sync_engine::{DatabaseSyncEngine, DatabaseSyncEngineOpts},
types::Coro,
types::{
Coro, DatabaseChangeType, DatabaseRowMutation, DatabaseRowStatement,
DatabaseSyncEngineProtocolVersion,
},
};
use crate::{
generator::GeneratorHolder,
generator::{GeneratorHolder, GeneratorResponse},
js_protocol_io::{JsProtocolIo, JsProtocolRequestData},
};
@@ -29,18 +38,92 @@ pub struct SyncEngine {
path: String,
client_name: String,
wal_pull_batch_size: u32,
protocol_version: DatabaseSyncEngineProtocolVersion,
tables_ignore: Vec<String>,
transform: Option<FunctionRef<DatabaseRowMutationJs, Option<DatabaseRowStatementJs>>>,
io: Arc<dyn turso_core::IO>,
protocol: Arc<JsProtocolIo>,
sync_engine: Arc<Mutex<Option<DatabaseSyncEngine<JsProtocolIo>>>>,
sync_engine: Arc<RwLock<Option<DatabaseSyncEngine<JsProtocolIo, Env>>>>,
opened: Arc<Mutex<Option<turso_node::Database>>>,
}
#[napi]
pub enum DatabaseChangeTypeJs {
Insert,
Update,
Delete,
}
#[napi]
pub enum SyncEngineProtocolVersion {
Legacy,
V1,
}
fn core_change_type_to_js(value: DatabaseChangeType) -> DatabaseChangeTypeJs {
match value {
DatabaseChangeType::Delete => DatabaseChangeTypeJs::Delete,
DatabaseChangeType::Update => DatabaseChangeTypeJs::Update,
DatabaseChangeType::Insert => DatabaseChangeTypeJs::Insert,
}
}
fn js_value_to_core(value: Either5<Null, i64, f64, String, Vec<u8>>) -> turso_core::Value {
match value {
Either5::A(_) => turso_core::Value::Null,
Either5::B(value) => turso_core::Value::Integer(value as i64),
Either5::C(value) => turso_core::Value::Float(value),
Either5::D(value) => turso_core::Value::Text(turso_core::types::Text::new(&value)),
Either5::E(value) => turso_core::Value::Blob(value),
}
}
fn core_value_to_js(value: turso_core::Value) -> Either5<Null, i64, f64, String, Vec<u8>> {
match value {
turso_core::Value::Null => Either5::<Null, i64, f64, String, Vec<u8>>::A(Null),
turso_core::Value::Integer(value) => Either5::<Null, i64, f64, String, Vec<u8>>::B(value),
turso_core::Value::Float(value) => Either5::<Null, i64, f64, String, Vec<u8>>::C(value),
turso_core::Value::Text(value) => {
Either5::<Null, i64, f64, String, Vec<u8>>::D(value.as_str().to_string())
}
turso_core::Value::Blob(value) => Either5::<Null, i64, f64, String, Vec<u8>>::E(value),
}
}
fn core_values_map_to_js(
value: HashMap<String, turso_core::Value>,
) -> HashMap<String, Either5<Null, i64, f64, String, Vec<u8>>> {
let mut result = HashMap::new();
for (key, value) in value {
result.insert(key, core_value_to_js(value));
}
result
}
#[napi(object)]
pub struct DatabaseRowMutationJs {
pub change_time: i64,
pub table_name: String,
pub id: i64,
pub change_type: DatabaseChangeTypeJs,
pub before: Option<HashMap<String, Either5<Null, i64, f64, String, Vec<u8>>>>,
pub after: Option<HashMap<String, Either5<Null, i64, f64, String, Vec<u8>>>>,
pub updates: Option<HashMap<String, Either5<Null, i64, f64, String, Vec<u8>>>>,
}
#[napi(object)]
#[derive(Debug)]
pub struct DatabaseRowStatementJs {
pub sql: String,
pub values: Vec<Either5<Null, i64, f64, String, Vec<u8>>>,
}
#[napi(object, object_to_js = false)]
pub struct SyncEngineOpts {
pub path: String,
pub client_name: Option<String>,
pub wal_pull_batch_size: Option<u32>,
pub enable_tracing: Option<String>,
pub tables_ignore: Option<Vec<String>>,
pub transform: Option<Function<'static, DatabaseRowMutationJs, Option<DatabaseRowStatementJs>>>,
pub protocol_version: Option<SyncEngineProtocolVersion>,
}
static TRACING_INIT: OnceLock<()> = OnceLock::new();
@@ -81,19 +164,65 @@ impl SyncEngine {
path: opts.path,
client_name: opts.client_name.unwrap_or("turso-sync-js".to_string()),
wal_pull_batch_size: opts.wal_pull_batch_size.unwrap_or(100),
sync_engine: Arc::new(Mutex::new(None)),
tables_ignore: opts.tables_ignore.unwrap_or(Vec::new()),
transform: opts.transform.map(|x| x.create_ref().unwrap()),
sync_engine: Arc::new(RwLock::new(None)),
io,
protocol: Arc::new(JsProtocolIo::default()),
#[allow(clippy::arc_with_non_send_sync)]
opened: Arc::new(Mutex::new(None)),
protocol_version: match opts.protocol_version {
Some(SyncEngineProtocolVersion::Legacy) | None => {
DatabaseSyncEngineProtocolVersion::Legacy
}
_ => DatabaseSyncEngineProtocolVersion::V1,
},
})
}
#[napi]
pub fn init(&self) -> GeneratorHolder {
pub fn init(&mut self, env: Env) -> GeneratorHolder {
let transform: Option<
Arc<
dyn Fn(
&Env,
DatabaseRowMutation,
)
-> turso_sync_engine::Result<Option<DatabaseRowStatement>>
+ 'static,
>,
> = match self.transform.take() {
Some(f) => Some(Arc::new(move |env, mutation| {
let result = f
.borrow_back(&env)
.unwrap()
.call(DatabaseRowMutationJs {
change_time: mutation.change_time as i64,
table_name: mutation.table_name,
id: mutation.id,
change_type: core_change_type_to_js(mutation.change_type),
before: mutation.before.map(core_values_map_to_js),
after: mutation.after.map(core_values_map_to_js),
updates: mutation.updates.map(core_values_map_to_js),
})
.map_err(|e| {
turso_sync_engine::errors::Error::DatabaseSyncEngineError(format!(
"transform callback failed: {e}"
))
})?;
Ok(result.map(|statement| DatabaseRowStatement {
sql: statement.sql,
values: statement.values.into_iter().map(js_value_to_core).collect(),
}))
})),
None => None,
};
let opts = DatabaseSyncEngineOpts {
client_name: self.client_name.clone(),
wal_pull_batch_size: self.wal_pull_batch_size as u64,
tables_ignore: self.tables_ignore.clone(),
transform,
protocol_version_hint: self.protocol_version,
};
let protocol = self.protocol.clone();
@@ -102,17 +231,19 @@ impl SyncEngine {
let opened = self.opened.clone();
let path = self.path.clone();
let generator = genawaiter::sync::Gen::new(|coro| async move {
let coro = Coro::new(env, coro);
let initialized =
DatabaseSyncEngine::new(&coro, io.clone(), protocol, &path, opts).await?;
let connection = initialized.connect(&coro).await?;
let connection = initialized.connect_rw(&coro).await?;
let db = turso_node::Database::create(None, io.clone(), connection, false);
*sync_engine.lock().unwrap() = Some(initialized);
*sync_engine.write().unwrap() = Some(initialized);
*opened.lock().unwrap() = Some(db);
Ok(())
});
GeneratorHolder {
inner: Box::new(Mutex::new(generator)),
response: Arc::new(Mutex::new(None)),
}
}
@@ -137,18 +268,63 @@ impl SyncEngine {
}
#[napi]
pub fn sync(&self) -> GeneratorHolder {
self.run(async move |coro, sync_engine| sync_engine.sync(coro).await)
pub fn sync(&self, env: Env) -> GeneratorHolder {
self.run(env, async move |coro, sync_engine| {
let mut sync_engine = try_write(sync_engine)?;
let sync_engine = try_unwrap_mut(&mut sync_engine)?;
sync_engine.sync(coro).await?;
Ok(None)
})
}
#[napi]
pub fn push(&self) -> GeneratorHolder {
self.run(async move |coro, sync_engine| sync_engine.push(coro).await)
pub fn push(&self, env: Env) -> GeneratorHolder {
self.run(env, async move |coro, sync_engine| {
let sync_engine = try_read(sync_engine)?;
let sync_engine = try_unwrap(&sync_engine)?;
sync_engine.push_changes_to_remote(coro).await?;
Ok(None)
})
}
#[napi]
pub fn pull(&self) -> GeneratorHolder {
self.run(async move |coro, sync_engine| sync_engine.pull(coro).await)
pub fn stats(&self, env: Env) -> GeneratorHolder {
self.run(env, async move |coro, sync_engine| {
let sync_engine = try_read(sync_engine)?;
let sync_engine = try_unwrap(&sync_engine)?;
let changes = sync_engine.stats(coro).await?;
Ok(Some(GeneratorResponse::SyncEngineStats {
operations: changes.cdc_operations,
wal: changes.wal_size,
}))
})
}
#[napi]
pub fn pull(&self, env: Env) -> GeneratorHolder {
self.run(env, async move |coro, sync_engine| {
let changes = {
let sync_engine = try_read(sync_engine)?;
let sync_engine = try_unwrap(&sync_engine)?;
sync_engine.wait_changes_from_remote(coro).await?
};
if let Some(changes) = changes {
let mut sync_engine = try_write(sync_engine)?;
let sync_engine = try_unwrap_mut(&mut sync_engine)?;
sync_engine.apply_changes_from_remote(coro, changes).await?;
}
Ok(None)
})
}
#[napi]
pub fn checkpoint(&self, env: Env) -> GeneratorHolder {
self.run(env, async move |coro, sync_engine| {
let mut sync_engine = try_write(sync_engine)?;
let sync_engine = try_unwrap_mut(&mut sync_engine)?;
sync_engine.checkpoint(coro).await?;
Ok(None)
})
}
#[napi]
@@ -165,32 +341,76 @@ impl SyncEngine {
fn run(
&self,
env: Env,
f: impl AsyncFnOnce(
&Coro,
&mut DatabaseSyncEngine<JsProtocolIo>,
) -> turso_sync_engine::Result<()>
&Coro<Env>,
&Arc<RwLock<Option<DatabaseSyncEngine<JsProtocolIo, Env>>>>,
) -> turso_sync_engine::Result<Option<GeneratorResponse>>
+ 'static,
) -> GeneratorHolder {
let response = Arc::new(Mutex::new(None));
let sync_engine = self.sync_engine.clone();
#[allow(clippy::await_holding_lock)]
let generator = genawaiter::sync::Gen::new(|coro| async move {
let Ok(mut sync_engine) = sync_engine.try_lock() else {
let nasty_error = "sync_engine is busy".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
nasty_error,
));
};
let Some(sync_engine) = sync_engine.as_mut() else {
let error = "sync_engine must be initialized".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
error,
));
};
f(&coro, sync_engine).await?;
Ok(())
let generator = genawaiter::sync::Gen::new({
let response = response.clone();
|coro| async move {
let coro = Coro::new(env, coro);
*response.lock().unwrap() = f(&coro, &sync_engine).await?;
Ok(())
}
});
GeneratorHolder {
inner: Box::new(Mutex::new(generator)),
response,
}
}
}
fn try_read(
sync_engine: &RwLock<Option<DatabaseSyncEngine<JsProtocolIo, Env>>>,
) -> turso_sync_engine::Result<RwLockReadGuard<'_, Option<DatabaseSyncEngine<JsProtocolIo, Env>>>> {
let Ok(sync_engine) = sync_engine.try_read() else {
let nasty_error = "sync_engine is busy".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
nasty_error,
));
};
Ok(sync_engine)
}
fn try_write(
sync_engine: &RwLock<Option<DatabaseSyncEngine<JsProtocolIo, Env>>>,
) -> turso_sync_engine::Result<RwLockWriteGuard<'_, Option<DatabaseSyncEngine<JsProtocolIo, Env>>>>
{
let Ok(sync_engine) = sync_engine.try_write() else {
let nasty_error = "sync_engine is busy".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
nasty_error,
));
};
Ok(sync_engine)
}
fn try_unwrap<'a>(
sync_engine: &'a RwLockReadGuard<'_, Option<DatabaseSyncEngine<JsProtocolIo, Env>>>,
) -> turso_sync_engine::Result<&'a DatabaseSyncEngine<JsProtocolIo, Env>> {
let Some(sync_engine) = sync_engine.as_ref() else {
let error = "sync_engine must be initialized".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
error,
));
};
Ok(sync_engine)
}
fn try_unwrap_mut<'a>(
sync_engine: &'a mut RwLockWriteGuard<'_, Option<DatabaseSyncEngine<JsProtocolIo, Env>>>,
) -> turso_sync_engine::Result<&'a mut DatabaseSyncEngine<JsProtocolIo, Env>> {
let Some(sync_engine) = sync_engine.as_mut() else {
let error = "sync_engine must be initialized".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
error,
));
};
Ok(sync_engine)
}

31
sync/javascript/sync_engine.ts
View File

@@ -1,6 +1,6 @@
"use strict";
import { SyncEngine } from '#entry-point';
import { SyncEngine, DatabaseRowMutationJs, DatabaseRowStatementJs } from '#entry-point';
import { Database } from '@tursodatabase/database';
const GENERATOR_RESUME_IO = 0;
@@ -63,9 +63,16 @@ async function process(opts, request) {
const completion = request.completion();
if (requestType.type == 'Http') {
try {
let headers = opts.headers;
if (requestType.headers != null && requestType.headers.length > 0) {
headers = { ...opts.headers };
for (let header of requestType.headers) {
headers[header[0]] = header[1];
}
}
const response = await fetch(`${opts.url}${requestType.path}`, {
method: requestType.method,
headers: opts.headers,
headers: headers,
body: requestType.body != null ? new Uint8Array(requestType.body) : null,
});
completion.status(response.status);
@@ -101,7 +108,7 @@ async function process(opts, request) {
}
}
async function run(opts, engine, generator) {
async function run(opts, engine, generator): Promise<any> {
let tasks = [];
while (generator.resume(null) !== GENERATOR_RESUME_DONE) {
for (let request = engine.protocolIo(); request != null; request = engine.protocolIo()) {
@@ -113,6 +120,7 @@ async function run(opts, engine, generator) {
tasks = tasks.filter(t => !t.finished);
}
return generator.take();
}
interface ConnectOpts {
@@ -121,16 +129,27 @@ interface ConnectOpts {
url: string;
authToken?: string;
encryptionKey?: string;
tablesIgnore?: string[],
transform?: (arg: DatabaseRowMutationJs) => DatabaseRowStatementJs | null,
enableTracing?: string,
}
interface Sync {
sync(): Promise<void>;
push(): Promise<void>;
pull(): Promise<void>;
checkpoint(): Promise<void>;
stats(): Promise<{ operations: number, wal: number }>;
}
export async function connect(opts: ConnectOpts): Database & Sync {
const engine = new SyncEngine({ path: opts.path, clientName: opts.clientName });
const engine = new SyncEngine({
path: opts.path,
clientName: opts.clientName,
tablesIgnore: opts.tablesIgnore,
transform: opts.transform,
enableTracing: opts.enableTracing
});
const httpOpts = {
url: opts.url,
headers: {
@@ -147,5 +166,9 @@ export async function connect(opts: ConnectOpts): Database & Sync {
db.sync = async function () { await run(httpOpts, engine, engine.sync()); }
db.pull = async function () { await run(httpOpts, engine, engine.pull()); }
db.push = async function () { await run(httpOpts, engine, engine.push()); }
db.checkpoint = async function () { await run(httpOpts, engine, engine.checkpoint()); }
db.stats = async function () { return (await run(httpOpts, engine, engine.stats())); }
return db;
}
export { Database, Sync };

2
sync/javascript/turso-sync-js.wasi-browser.js
View File

@@ -64,3 +64,5 @@ export const JsDataPollResult = __napiModule.exports.JsDataPollResult
export const JsProtocolIo = __napiModule.exports.JsProtocolIo
export const JsProtocolRequestData = __napiModule.exports.JsProtocolRequestData
export const SyncEngine = __napiModule.exports.SyncEngine
export const DatabaseChangeTypeJs = __napiModule.exports.DatabaseChangeTypeJs
export const SyncEngineProtocolVersion = __napiModule.exports.SyncEngineProtocolVersion

2
sync/javascript/turso-sync-js.wasi.cjs
View File

@@ -116,3 +116,5 @@ module.exports.JsDataPollResult = __napiModule.exports.JsDataPollResult
module.exports.JsProtocolIo = __napiModule.exports.JsProtocolIo
module.exports.JsProtocolRequestData = __napiModule.exports.JsProtocolRequestData
module.exports.SyncEngine = __napiModule.exports.SyncEngine
module.exports.DatabaseChangeTypeJs = __napiModule.exports.DatabaseChangeTypeJs
module.exports.SyncEngineProtocolVersion = __napiModule.exports.SyncEngineProtocolVersion