From 2d517f9fd7454c529006cf41ddaa02636e37e06d Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Wed, 12 Nov 2025 14:20:26 +0400 Subject: [PATCH] use sparse io fir partial sync in case when file is used --- .../javascript/sync/packages/common/types.ts | 2 +- .../sync/packages/native/index.d.ts | 6 ++- .../sync/packages/native/promise.test.ts | 2 +- .../sync/packages/native/promise.ts | 13 ++++- bindings/javascript/sync/src/lib.rs | 47 ++++++++++++++----- 5 files changed, 55 insertions(+), 15 deletions(-) diff --git a/bindings/javascript/sync/packages/common/types.ts b/bindings/javascript/sync/packages/common/types.ts index 61cbd7f0b..06a48142e 100644 --- a/bindings/javascript/sync/packages/common/types.ts +++ b/bindings/javascript/sync/packages/common/types.ts @@ -110,7 +110,7 @@ export interface DatabaseOpts { /** * optional parameter to enable partial sync for the database */ - partial?: boolean; + partialBootstrapStrategy?: { kind: 'prefix', length: number } | { kind: 'query', query: string }; } export interface DatabaseStats { /** diff --git a/bindings/javascript/sync/packages/native/index.d.ts b/bindings/javascript/sync/packages/native/index.d.ts index 33187509d..4073cdd1e 100644 --- a/bindings/javascript/sync/packages/native/index.d.ts +++ b/bindings/javascript/sync/packages/native/index.d.ts @@ -224,6 +224,10 @@ export type GeneratorResponse = | { type: 'SyncEngineStats', operations: number, mainWal: number, revertWal: number, lastPullUnixTime?: number, lastPushUnixTime?: number, revision?: string, networkSentBytes: number, networkReceivedBytes: number } | { type: 'SyncEngineChanges', changes: SyncEngineChanges } +export type JsPartialBootstrapStrategy = + | { type: 'Prefix', length: number } + | { type: 'Query', query: string } + export type JsProtocolRequest = | { type: 'Http', method: string, path: string, body?: Array, headers: Array<[string, string]> } | { type: 'FullRead', path: string } @@ -241,7 +245,7 @@ export interface SyncEngineOpts { protocolVersion?: SyncEngineProtocolVersion bootstrapIfEmpty: boolean remoteEncryption?: string - partial?: boolean + partialBoostrapStrategy?: JsPartialBootstrapStrategy } export declare const enum SyncEngineProtocolVersion { diff --git a/bindings/javascript/sync/packages/native/promise.test.ts b/bindings/javascript/sync/packages/native/promise.test.ts index 7ab40ca96..d8e1bba5e 100644 --- a/bindings/javascript/sync/packages/native/promise.test.ts +++ b/bindings/javascript/sync/packages/native/promise.test.ts @@ -31,7 +31,7 @@ test('partial sync', async () => { path: ':memory:', url: process.env.VITE_TURSO_DB_URL, longPollTimeoutMs: 100, - partial: true, + partialBootstrapStrategy: { kind: 'prefix', length: 128 * 1024 }, }); // 128 pages plus some overhead (very rough estimation) diff --git a/bindings/javascript/sync/packages/native/promise.ts b/bindings/javascript/sync/packages/native/promise.ts index a75e13edf..aa2d90ac7 100644 --- a/bindings/javascript/sync/packages/native/promise.ts +++ b/bindings/javascript/sync/packages/native/promise.ts @@ -50,6 +50,17 @@ class Database extends DatabasePromise { return; } + let partialBoostrapStrategy = undefined; + if (opts.partialBootstrapStrategy != null) { + switch (opts.partialBootstrapStrategy.kind) { + case "prefix": + partialBoostrapStrategy = { type: "Prefix", length: opts.partialBootstrapStrategy.length }; + break; + case "query": + partialBoostrapStrategy = { type: "Query", length: opts.partialBootstrapStrategy.query }; + break; + } + } const engine = new SyncEngine({ path: opts.path, clientName: opts.clientName, @@ -59,7 +70,7 @@ class Database extends DatabasePromise { tracing: opts.tracing, bootstrapIfEmpty: typeof opts.url != "function" || opts.url() != null, remoteEncryption: opts.remoteEncryption?.cipher, - partial: opts.partial, + partialBoostrapStrategy: partialBoostrapStrategy }); let headers: { [K: string]: string } | (() => Promise<{ [K: string]: string }>); diff --git a/bindings/javascript/sync/src/lib.rs b/bindings/javascript/sync/src/lib.rs index ae133580f..0322f70a6 100644 --- a/bindings/javascript/sync/src/lib.rs +++ b/bindings/javascript/sync/src/lib.rs @@ -13,7 +13,7 @@ use napi::bindgen_prelude::{AsyncTask, Either5, Null}; use napi_derive::napi; use turso_node::{DatabaseOpts, IoLoopTask}; use turso_sync_engine::{ - database_sync_engine::{DatabaseSyncEngine, DatabaseSyncEngineOpts}, + database_sync_engine::{DatabaseSyncEngine, DatabaseSyncEngineOpts, PartialBootstrapStrategy}, protocol_io::ProtocolIO, types::{Coro, DatabaseChangeType, DatabaseSyncEngineProtocolVersion}, }; @@ -108,6 +108,13 @@ pub enum DatabaseRowTransformResultJs { Rewrite { stmt: DatabaseRowStatementJs }, } +#[napi(discriminant = "type")] +#[derive(Debug)] +pub enum JsPartialBootstrapStrategy { + Prefix { length: i64 }, + Query { query: String }, +} + #[napi(object, object_to_js = false)] pub struct SyncEngineOpts { pub path: String, @@ -120,7 +127,7 @@ pub struct SyncEngineOpts { pub protocol_version: Option, pub bootstrap_if_empty: bool, pub remote_encryption: Option, - pub partial: Option, + pub partial_boostrap_strategy: Option, } struct SyncEngineOptsFilled { @@ -133,7 +140,7 @@ struct SyncEngineOptsFilled { pub protocol_version: DatabaseSyncEngineProtocolVersion, pub bootstrap_if_empty: bool, pub remote_encryption: Option, - pub partial: bool, + pub partial_boostrap_strategy: Option, } #[derive(Debug, Clone, Copy)] @@ -175,12 +182,23 @@ impl SyncEngine { } else { #[cfg(not(feature = "browser"))] { - Arc::new(turso_core::PlatformIO::new().map_err(|e| { - napi::Error::new( - napi::Status::GenericFailure, - format!("Failed to create IO: {e}"), - ) - })?) + if opts.partial_boostrap_strategy.is_none() { + Arc::new(turso_core::PlatformIO::new().map_err(|e| { + napi::Error::new( + napi::Status::GenericFailure, + format!("Failed to create platform IO: {e}"), + ) + })?) + } else { + use turso_sync_engine::sparse_io::SparseLinuxIo; + + Arc::new(SparseLinuxIo::new().map_err(|e| { + napi::Error::new( + napi::Status::GenericFailure, + format!("Failed to create sparse IO: {e}"), + ) + })?) + } } #[cfg(feature = "browser")] { @@ -227,7 +245,14 @@ impl SyncEngine { )) } }, - partial: opts.partial.unwrap_or(false), + partial_boostrap_strategy: opts.partial_boostrap_strategy.map(|s| match s { + JsPartialBootstrapStrategy::Prefix { length } => PartialBootstrapStrategy::Prefix { + length: length as usize, + }, + JsPartialBootstrapStrategy::Query { query } => { + PartialBootstrapStrategy::Query { query } + } + }), }; Ok(SyncEngine { opts: opts_filled, @@ -255,7 +280,7 @@ impl SyncEngine { .remote_encryption .map(|x| x.required_metadata_size()) .unwrap_or(0), - partial: self.opts.partial, + partial_bootstrap_strategy: self.opts.partial_boostrap_strategy.clone(), }; let io = self.io()?;