aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2025-12-28 21:44:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

opfs for sync in one commit!

Browse Source
This commit is contained in:
Nikita Sivukhin
2025-09-10 03:01:37 +04:00
parent 8ab8b31cb1
commit d55026f84f
75 changed files with 3553 additions and 5535 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
bindings/javascript/sync/Cargo.toml Normal file
View File

@@ -0,0 +1,26 @@
[package]
name = "turso_sync_js"
version.workspace = true
authors.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
[lib]
crate-type = ["cdylib"]
[dependencies]
http = "1.3.1"
napi = { version = "3.1.3", default-features = false, features = ["napi6"] }
napi-derive = { version = "3.1.1", default-features = true }
turso_sync_engine = { workspace = true }
turso_core = { workspace = true }
turso_node = { workspace = true }
genawaiter = { version = "0.99.1", default-features = false }
tracing-subscriber = "0.3.19"
[build-dependencies]
napi-build = "2.2.3"
[features]
browser = ["turso_node/browser"]

60
bindings/javascript/sync/README.md Normal file
View File

@@ -0,0 +1,60 @@
<p align="center">
<h1 align="center">Turso Sync for JavaScript</h1>
</p>
<p align="center">
<a title="JavaScript" target="_blank" href="https://www.npmjs.com/package/@tursodatabase/sync"><img alt="npm" src="https://img.shields.io/npm/v/@tursodatabase/sync"></a>
<a title="MIT" target="_blank" href="https://github.com/tursodatabase/turso/blob/main/LICENSE.md"><img src="http://img.shields.io/badge/license-MIT-orange.svg?style=flat-square"></a>
</p>
<p align="center">
<a title="Users Discord" target="_blank" href="https://tur.so/discord"><img alt="Chat with other users of Turso on Discord" src="https://img.shields.io/discord/933071162680958986?label=Discord&logo=Discord&style=social"></a>
</p>
---
## About
This package is for syncing local Turso databases to the Turso Cloud and back.
> **⚠️ Warning:** This software is ALPHA, only use for development, testing, and experimentation. We are working to make it production ready, but do not use it for critical data right now.
## Installation
```bash
npm install @tursodatabase/sync
```
## Getting Started
To sync a database hosted at [Turso Cloud](https://turso.tech):
```js
import { connect } from '@tursodatabase/sync';
const db = await connect({
path: 'local.db', // path used as a prefix for local files created by sync-engine
url: 'https://<db>.turso.io', // URL of the remote database: turso db show <db>
authToken: '...', // auth token issued from the Turso Cloud: turso db tokens create <db>
clientName: 'turso-sync-example' // arbitrary client name
});
// db has same functions as Database class from @tursodatabase/database package but adds few more methods for sync:
await db.pull(); // pull changes from the remote
await db.push(); // push changes to the remote
await db.sync(); // pull & push changes
```
## Related Packages
* The [@tursodatabase/database](https://www.npmjs.com/package/@tursodatabase/database) package provides the Turso in-memory database, compatible with SQLite.
* The [@tursodatabase/serverless](https://www.npmjs.com/package/@tursodatabase/serverless) package provides a serverless driver with the same API.
## License
This project is licensed under the [MIT license](../../LICENSE.md).
## Support
- [GitHub Issues](https://github.com/tursodatabase/turso/issues)
- [Documentation](https://docs.turso.tech)
- [Discord Community](https://tur.so/discord)

3
bindings/javascript/sync/build.rs Normal file
View File

@@ -0,0 +1,3 @@
fn main() {
napi_build::setup();
}

124
bindings/javascript/sync/packages/browser/README.md Normal file
View File

@@ -0,0 +1,124 @@
<p align="center">
<h1 align="center">Turso Database for JavaScript in Browser</h1>
</p>
<p align="center">
<a title="JavaScript" target="_blank" href="https://www.npmjs.com/package/@tursodatabase/database"><img alt="npm" src="https://img.shields.io/npm/v/@tursodatabase/database"></a>
<a title="MIT" target="_blank" href="https://github.com/tursodatabase/turso/blob/main/LICENSE.md"><img src="http://img.shields.io/badge/license-MIT-orange.svg?style=flat-square"></a>
</p>
<p align="center">
<a title="Users Discord" target="_blank" href="https://tur.so/discord"><img alt="Chat with other users of Turso on Discord" src="https://img.shields.io/discord/933071162680958986?label=Discord&logo=Discord&style=social"></a>
</p>
---
## About
This package is the Turso embedded database library for JavaScript in Browser.
> **⚠️ Warning:** This software is ALPHA, only use for development, testing, and experimentation. We are working to make it production ready, but do not use it for critical data right now.
## Features
- **SQLite compatible:** SQLite query language and file format support ([status](https://github.com/tursodatabase/turso/blob/main/COMPAT.md)).
- **In-process**: No network overhead, runs directly in your Node.js process
- **TypeScript support**: Full TypeScript definitions included
## Installation
```bash
npm install @tursodatabase/database-browser
```
## Getting Started
### In-Memory Database
```javascript
import { connect } from '@tursodatabase/database-browser';
// Create an in-memory database
const db = await connect(':memory:');
// Create a table
await db.exec('CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, email TEXT)');
// Insert data
const insert = db.prepare('INSERT INTO users (name, email) VALUES (?, ?)');
await insert.run('Alice', 'alice@example.com');
await insert.run('Bob', 'bob@example.com');
// Query data
const users = await db.prepare('SELECT * FROM users').all();
console.log(users);
// Output: [
// { id: 1, name: 'Alice', email: 'alice@example.com' },
// { id: 2, name: 'Bob', email: 'bob@example.com' }
// ]
```
### File-Based Database
```javascript
import { connect } from '@tursodatabase/database-browser';
// Create or open a database file
const db = await connect('my-database.db');
// Create a table
await db.exec(`
CREATE TABLE IF NOT EXISTS posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
content TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
`);
// Insert a post
const insertPost = db.prepare('INSERT INTO posts (title, content) VALUES (?, ?)');
const result = await insertPost.run('Hello World', 'This is my first blog post!');
console.log(`Inserted post with ID: ${result.lastInsertRowid}`);
```
### Transactions
```javascript
import { connect } from '@tursodatabase/database-browser';
const db = await connect('transactions.db');
// Using transactions for atomic operations
const transaction = db.transaction(async (users) => {
const insert = db.prepare('INSERT INTO users (name, email) VALUES (?, ?)');
for (const user of users) {
await insert.run(user.name, user.email);
}
});
// Execute transaction
await transaction([
{ name: 'Alice', email: 'alice@example.com' },
{ name: 'Bob', email: 'bob@example.com' }
]);
```
## API Reference
For complete API documentation, see [JavaScript API Reference](../../../../docs/javascript-api-reference.md).
## Related Packages
* The [@tursodatabase/serverless](https://www.npmjs.com/package/@tursodatabase/serverless) package provides a serverless driver with the same API.
* The [@tursodatabase/sync](https://www.npmjs.com/package/@tursodatabase/sync) package provides bidirectional sync between a local Turso database and Turso Cloud.
## License
This project is licensed under the [MIT license](../../LICENSE.md).
## Support
- [GitHub Issues](https://github.com/tursodatabase/turso/issues)
- [Documentation](https://docs.turso.tech)
- [Discord Community](https://tur.so/discord)

76
bindings/javascript/sync/packages/browser/index.js Normal file
View File

@@ -0,0 +1,76 @@
import {
createOnMessage as __wasmCreateOnMessageForFsProxy,
getDefaultContext as __emnapiGetDefaultContext,
instantiateNapiModule as __emnapiInstantiateNapiModule,
WASI as __WASI,
} from '@napi-rs/wasm-runtime'
import { MainDummyImports } from "@tursodatabase/database-browser-common";
const __wasi = new __WASI({
version: 'preview1',
})
const __wasmUrl = new URL('./sync.wasm32-wasi.wasm', import.meta.url).href
const __emnapiContext = __emnapiGetDefaultContext()
const __sharedMemory = new WebAssembly.Memory({
initial: 4000,
maximum: 65536,
shared: true,
})
const __wasmFile = await fetch(__wasmUrl).then((res) => res.arrayBuffer())
export let MainWorker = null;
const {
instance: __napiInstance,
module: __wasiModule,
napiModule: __napiModule,
} = await __emnapiInstantiateNapiModule(__wasmFile, {
context: __emnapiContext,
asyncWorkPoolSize: 1,
wasi: __wasi,
onCreateWorker() {
const worker = new Worker(new URL('./worker.mjs', import.meta.url), {
type: 'module',
})
MainWorker = worker;
return worker
},
overwriteImports(importObject) {
importObject.env = {
...importObject.env,
...importObject.napi,
...importObject.emnapi,
...MainDummyImports,
memory: __sharedMemory,
}
return importObject
},
beforeInit({ instance }) {
for (const name of Object.keys(instance.exports)) {
if (name.startsWith('__napi_register__')) {
instance.exports[name]()
}
}
},
})
export default __napiModule.exports
export const Database = __napiModule.exports.Database
export const Statement = __napiModule.exports.Statement
export const Opfs = __napiModule.exports.Opfs
export const OpfsFile = __napiModule.exports.OpfsFile
export const connect = __napiModule.exports.connect
export const initThreadPool = __napiModule.exports.initThreadPool
export const GeneratorHolder = __napiModule.exports.GeneratorHolder
export const JsDataCompletion = __napiModule.exports.JsDataCompletion
export const JsProtocolIo = __napiModule.exports.JsProtocolIo
export const JsProtocolRequestBytes = __napiModule.exports.JsProtocolRequestBytes
export const SyncEngine = __napiModule.exports.SyncEngine
export const DatabaseChangeTypeJs = __napiModule.exports.DatabaseChangeTypeJs
export const SyncEngineProtocolVersion = __napiModule.exports.SyncEngineProtocolVersion

46
bindings/javascript/sync/packages/browser/package.json Normal file
View File

@@ -0,0 +1,46 @@
{
"name": "@tursodatabase/sync-browser",
"version": "0.1.5",
"repository": {
"type": "git",
"url": "https://github.com/tursodatabase/turso"
},
"type": "module",
"license": "MIT",
"main": "dist/promise.js",
"packageManager": "yarn@4.9.2",
"files": [
"index.js",
"worker.mjs",
"sync.wasm32-wasi.wasm",
"dist/**",
"README.md"
],
"devDependencies": {
"@napi-rs/cli": "^3.1.5",
"@vitest/browser": "^3.2.4",
"playwright": "^1.55.0",
"typescript": "^5.9.2",
"vitest": "^3.2.4"
},
"scripts": {
"napi-build": "napi build --features browser --release --platform --target wasm32-wasip1-threads --no-js --manifest-path ../../Cargo.toml --output-dir . && rm index.d.ts sync.wasi* wasi* browser.js",
"tsc-build": "npm exec tsc",
"build": "npm run napi-build && npm run tsc-build",
"test": "VITE_TURSO_DB_URL=http://b--a--a.localhost:10000 CI=1 vitest --browser=chromium --run && VITE_TURSO_DB_URL=http://b--a--a.localhost:10000 CI=1 vitest --browser=firefox --run"
},
"napi": {
"binaryName": "sync",
"targets": [
"wasm32-wasip1-threads"
]
},
"imports": {
"#index": "./index.js"
},
"dependencies": {
"@napi-rs/wasm-runtime": "^1.0.3",
"@tursodatabase/sync-common": "^0.1.5",
"@tursodatabase/database-common": "^0.1.5"
}
}

281
bindings/javascript/sync/packages/browser/promise.test.ts Normal file
View File

@@ -0,0 +1,281 @@
import { expect, test } from 'vitest'
import { connect, DatabaseRowMutation, DatabaseRowTransformResult } from './promise.js'
const localeCompare = (a, b) => a.x.localeCompare(b.x);
test('select-after-push', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS t(x)");
await db.exec("DELETE FROM t");
await db.push();
await db.close();
}
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("INSERT INTO t VALUES (1), (2), (3)");
await db.push();
}
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
const rows = await db.prepare('SELECT * FROM t').all();
expect(rows).toEqual([{ x: 1 }, { x: 2 }, { x: 3 }])
}
})
test('select-without-push', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS t(x)");
await db.exec("DELETE FROM t");
await db.push();
await db.close();
}
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("INSERT INTO t VALUES (1), (2), (3)");
}
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
const rows = await db.prepare('SELECT * FROM t').all();
expect(rows).toEqual([])
}
})
test('merge-non-overlapping-keys', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db1.exec("INSERT INTO q VALUES ('k1', 'value1'), ('k2', 'value2')");
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db2.exec("INSERT INTO q VALUES ('k3', 'value3'), ('k4', 'value4'), ('k5', 'value5')");
await Promise.all([db1.push(), db2.push()]);
await Promise.all([db1.pull(), db2.pull()]);
const rows1 = await db1.prepare('SELECT * FROM q').all();
const rows2 = await db1.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k1', y: 'value1' }, { x: 'k2', y: 'value2' }, { x: 'k3', y: 'value3' }, { x: 'k4', y: 'value4' }, { x: 'k5', y: 'value5' }];
expect(rows1.sort(localeCompare)).toEqual(expected.sort(localeCompare))
expect(rows2.sort(localeCompare)).toEqual(expected.sort(localeCompare))
})
test('last-push-wins', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db1.exec("INSERT INTO q VALUES ('k1', 'value1'), ('k2', 'value2'), ('k4', 'value4')");
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db2.exec("INSERT INTO q VALUES ('k1', 'value3'), ('k2', 'value4'), ('k3', 'value5')");
await db2.push();
await db1.push();
await Promise.all([db1.pull(), db2.pull()]);
const rows1 = await db1.prepare('SELECT * FROM q').all();
const rows2 = await db1.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k1', y: 'value1' }, { x: 'k2', y: 'value2' }, { x: 'k3', y: 'value5' }, { x: 'k4', y: 'value4' }];
expect(rows1.sort(localeCompare)).toEqual(expected.sort(localeCompare))
expect(rows2.sort(localeCompare)).toEqual(expected.sort(localeCompare))
})
test('last-push-wins-with-delete', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db1.exec("INSERT INTO q VALUES ('k1', 'value1'), ('k2', 'value2'), ('k4', 'value4')");
await db1.exec("DELETE FROM q")
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db2.exec("INSERT INTO q VALUES ('k1', 'value3'), ('k2', 'value4'), ('k3', 'value5')");
await db2.push();
await db1.push();
await Promise.all([db1.pull(), db2.pull()]);
const rows1 = await db1.prepare('SELECT * FROM q').all();
const rows2 = await db1.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k3', y: 'value5' }];
expect(rows1).toEqual(expected)
expect(rows2).toEqual(expected)
})
test('constraint-conflict', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS u(x TEXT PRIMARY KEY, y UNIQUE)");
await db.exec("DELETE FROM u");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db1.exec("INSERT INTO u VALUES ('k1', 'value1')");
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db2.exec("INSERT INTO u VALUES ('k2', 'value1')");
await db1.push();
await expect(async () => await db2.push()).rejects.toThrow('SQLite error: UNIQUE constraint failed: u.y');
})
test('checkpoint', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
for (let i = 0; i < 1000; i++) {
await db1.exec(`INSERT INTO q VALUES ('k${i}', 'v${i}')`);
}
expect((await db1.stats()).mainWal).toBeGreaterThan(4096 * 1000);
await db1.checkpoint();
expect((await db1.stats()).mainWal).toBe(0);
let revertWal = (await db1.stats()).revertWal;
expect(revertWal).toBeLessThan(4096 * 1000 / 100);
for (let i = 0; i < 1000; i++) {
await db1.exec(`UPDATE q SET y = 'u${i}' WHERE x = 'k${i}'`);
}
await db1.checkpoint();
expect((await db1.stats()).revertWal).toBe(revertWal);
})
test('persistence', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const path = `test-${(Math.random() * 10000) | 0}.db`;
{
const db1 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL });
await db1.exec(`INSERT INTO q VALUES ('k1', 'v1')`);
await db1.exec(`INSERT INTO q VALUES ('k2', 'v2')`);
await db1.close();
}
{
const db2 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL });
await db2.exec(`INSERT INTO q VALUES ('k3', 'v3')`);
await db2.exec(`INSERT INTO q VALUES ('k4', 'v4')`);
const stmt = db2.prepare('SELECT * FROM q');
const rows = await stmt.all();
const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }, { x: 'k3', y: 'v3' }, { x: 'k4', y: 'v4' }];
expect(rows).toEqual(expected)
stmt.close();
await db2.close();
}
{
const db3 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL });
await db3.push();
await db3.close();
}
{
const db4 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL });
const rows = await db4.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }, { x: 'k3', y: 'v3' }, { x: 'k4', y: 'v4' }];
expect(rows).toEqual(expected)
await db4.close();
}
})
test('transform', async () => {
{
const db = await connect({
path: ':memory:',
url: process.env.VITE_TURSO_DB_URL,
});
await db.exec("CREATE TABLE IF NOT EXISTS counter(key TEXT PRIMARY KEY, value INTEGER)");
await db.exec("DELETE FROM counter");
await db.exec("INSERT INTO counter VALUES ('1', 0)")
await db.push();
await db.close();
}
const transform = (m: DatabaseRowMutation) => ({
operation: 'rewrite',
stmt: {
sql: `UPDATE counter SET value = value + ? WHERE key = ?`,
values: [m.after.value - m.before.value, m.after.key]
}
} as DatabaseRowTransformResult);
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL, transform: transform });
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL, transform: transform });
await db1.exec("UPDATE counter SET value = value + 1 WHERE key = '1'");
await db2.exec("UPDATE counter SET value = value + 1 WHERE key = '1'");
await Promise.all([db1.push(), db2.push()]);
await Promise.all([db1.pull(), db2.pull()]);
const rows1 = await db1.prepare('SELECT * FROM counter').all();
const rows2 = await db2.prepare('SELECT * FROM counter').all();
expect(rows1).toEqual([{ key: '1', value: 2 }]);
expect(rows2).toEqual([{ key: '1', value: 2 }]);
})
test('transform-many', async () => {
{
const db = await connect({
path: ':memory:',
url: process.env.VITE_TURSO_DB_URL,
});
await db.exec("CREATE TABLE IF NOT EXISTS counter(key TEXT PRIMARY KEY, value INTEGER)");
await db.exec("DELETE FROM counter");
await db.exec("INSERT INTO counter VALUES ('1', 0)")
await db.push();
await db.close();
}
const transform = (m: DatabaseRowMutation) => ({
operation: 'rewrite',
stmt: {
sql: `UPDATE counter SET value = value + ? WHERE key = ?`,
values: [m.after.value - m.before.value, m.after.key]
}
} as DatabaseRowTransformResult);
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL, transform: transform });
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL, transform: transform });
for (let i = 0; i < 1002; i++) {
await db1.exec("UPDATE counter SET value = value + 1 WHERE key = '1'");
}
for (let i = 0; i < 1001; i++) {
await db2.exec("UPDATE counter SET value = value + 1 WHERE key = '1'");
}
let start = performance.now();
await Promise.all([db1.push(), db2.push()]);
console.info('push', performance.now() - start);
start = performance.now();
await Promise.all([db1.pull(), db2.pull()]);
console.info('pull', performance.now() - start);
const rows1 = await db1.prepare('SELECT * FROM counter').all();
const rows2 = await db2.prepare('SELECT * FROM counter').all();
expect(rows1).toEqual([{ key: '1', value: 1001 + 1002 }]);
expect(rows2).toEqual([{ key: '1', value: 1001 + 1002 }]);
})

113
bindings/javascript/sync/packages/browser/promise.ts Normal file
View File

@@ -0,0 +1,113 @@
import { registerFileAtWorker, unregisterFileAtWorker } from "@tursodatabase/database-browser-common"
import { DatabasePromise, DatabaseOpts, NativeDatabase } from "@tursodatabase/database-common"
import { ProtocolIo, run, SyncOpts, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult, memoryIO } from "@tursodatabase/sync-common";
import { connect as nativeConnect, initThreadPool, MainWorker } from "#index";
import { Database as NativeDB, SyncEngine } from "#index";
let BrowserIo: ProtocolIo = {
async read(path: string): Promise<Buffer | Uint8Array | null> {
const result = localStorage.getItem(path);
if (result == null) {
return null;
}
return new TextEncoder().encode(result);
},
async write(path: string, data: Buffer | Uint8Array): Promise<void> {
const array = new Uint8Array(data);
const value = new TextDecoder('utf-8').decode(array);
localStorage.setItem(path, value);
}
};
class Database extends DatabasePromise {
runOpts: RunOpts;
engine: any;
io: ProtocolIo;
fsPath: string | null;
constructor(db: NativeDatabase, io: ProtocolIo, runOpts: RunOpts, engine: any, fsPath: string | null, opts: DatabaseOpts = {}) {
super(db, opts)
this.runOpts = runOpts;
this.engine = engine;
this.fsPath = fsPath;
this.io = io;
}
async sync() {
await run(this.runOpts, this.io, this.engine, this.engine.sync());
}
async pull() {
await run(this.runOpts, this.io, this.engine, this.engine.pull());
}
async push() {
await run(this.runOpts, this.io, this.engine, this.engine.push());
}
async checkpoint() {
await run(this.runOpts, this.io, this.engine, this.engine.checkpoint());
}
async stats(): Promise<{ operations: number, mainWal: number, revertWal: number, lastPullUnixTime: number, lastPushUnixTime: number | null }> {
return (await run(this.runOpts, this.io, this.engine, this.engine.stats()));
}
override async close(): Promise<void> {
this.db.close();
this.engine.close();
if (this.fsPath != null) {
await Promise.all([
unregisterFileAtWorker(MainWorker, this.fsPath),
unregisterFileAtWorker(MainWorker, `${this.fsPath}-wal`),
unregisterFileAtWorker(MainWorker, `${this.fsPath}-revert`),
unregisterFileAtWorker(MainWorker, `${this.fsPath}-info`),
unregisterFileAtWorker(MainWorker, `${this.fsPath}-changes`),
]);
}
}
}
/**
* Creates a new database connection asynchronously.
*
* @param {string} path - Path to the database file.
* @param {Object} opts - Options for database behavior.
* @returns {Promise<Database>} - A promise that resolves to a Database instance.
*/
async function connect(opts: SyncOpts): Promise<Database> {
const engine = new SyncEngine({
path: opts.path,
clientName: opts.clientName,
tablesIgnore: opts.tablesIgnore,
useTransform: opts.transform != null,
tracing: opts.tracing,
protocolVersion: 1
});
const runOpts: RunOpts = {
url: opts.url,
headers: {
...(opts.authToken != null && { "Authorization": `Bearer ${opts.authToken}` }),
...(opts.encryptionKey != null && { "x-turso-encryption-key": opts.encryptionKey })
},
preemptionMs: 1,
transform: opts.transform,
};
const isMemory = opts.path == ':memory:';
let io = isMemory ? memoryIO() : BrowserIo;
await initThreadPool();
if (MainWorker == null) {
throw new Error("panic: MainWorker is not set");
}
if (!isMemory) {
await Promise.all([
registerFileAtWorker(MainWorker, opts.path),
registerFileAtWorker(MainWorker, `${opts.path}-wal`),
registerFileAtWorker(MainWorker, `${opts.path}-revert`),
registerFileAtWorker(MainWorker, `${opts.path}-info`),
registerFileAtWorker(MainWorker, `${opts.path}-changes`),
]);
}
await run(runOpts, io, engine, engine.init());
const nativeDb = engine.open();
return new Database(nativeDb as any, io, runOpts, engine, isMemory ? null : opts.path, {});
}
export { connect, Database, }
export type { DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult }

24
bindings/javascript/sync/packages/browser/tsconfig.json Normal file
View File

@@ -0,0 +1,24 @@
{
"compilerOptions": {
"skipLibCheck": true,
"declaration": true,
"declarationMap": true,
"module": "nodenext",
"target": "esnext",
"moduleResolution": "nodenext",
"outDir": "dist/",
"lib": [
"es2020",
"DOM",
"WebWorker"
],
"paths": {
"#index": [
"./index.js"
]
}
},
"include": [
"*"
]
}

23
bindings/javascript/sync/packages/browser/vitest.config.ts Normal file
View File

@@ -0,0 +1,23 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
define: {
'process.env.NODE_DEBUG_NATIVE': 'false',
},
server: {
headers: {
"Cross-Origin-Embedder-Policy": "require-corp",
"Cross-Origin-Opener-Policy": "same-origin"
},
},
test: {
browser: {
enabled: true,
provider: 'playwright',
instances: [
{ browser: 'chromium' },
{ browser: 'firefox' }
],
},
},
})

55
bindings/javascript/sync/packages/browser/worker.mjs Normal file
View File

@@ -0,0 +1,55 @@
import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'
import { OpfsDirectory, workerImports } from "@tursodatabase/database-browser-common";
var opfs = new OpfsDirectory();
var memory = null;
const handler = new MessageHandler({
onLoad({ wasmModule, wasmMemory }) {
memory = wasmMemory;
const wasi = new WASI({
print: function () {
// eslint-disable-next-line no-console
console.log.apply(console, arguments)
},
printErr: function () {
// eslint-disable-next-line no-console
console.error.apply(console, arguments)
},
})
return instantiateNapiModuleSync(wasmModule, {
childThread: true,
wasi,
overwriteImports(importObject) {
importObject.env = {
...importObject.env,
...importObject.napi,
...importObject.emnapi,
...workerImports(opfs, memory),
memory: wasmMemory,
}
},
})
},
})
globalThis.onmessage = async function (e) {
if (e.data.__turso__ == 'register') {
try {
await opfs.registerFile(e.data.path);
self.postMessage({ id: e.data.id });
} catch (error) {
self.postMessage({ id: e.data.id, error: error });
}
return;
} else if (e.data.__turso__ == 'unregister') {
try {
await opfs.unregisterFile(e.data.path);
self.postMessage({ id: e.data.id });
} catch (error) {
self.postMessage({ id: e.data.id, error: error });
}
return;
}
handler.handle(e)
}

8
bindings/javascript/sync/packages/common/README.md Normal file
View File

@@ -0,0 +1,8 @@
## About
This package is the Turso Sync common JS library which is shared between final builds for Node and Browser.
Do not use this package directly - instead you must use `@tursodatabase/sync` or `@tursodatabase/sync-browser`.
> **⚠️ Warning:** This software is ALPHA, only use for development, testing, and experimentation. We are working to make it production ready, but do not use it for critical data right now.

5
bindings/javascript/sync/packages/common/index.ts Normal file
View File

@@ -0,0 +1,5 @@
import { run, memoryIO } from "./run.js"
import { SyncOpts, ProtocolIo, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult } from "./types.js"
export { run, memoryIO, }
export type { SyncOpts, ProtocolIo, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult }

25
bindings/javascript/sync/packages/common/package.json Normal file
View File

@@ -0,0 +1,25 @@
{
"name": "@tursodatabase/sync-common",
"version": "0.1.5",
"repository": {
"type": "git",
"url": "https://github.com/tursodatabase/turso"
},
"type": "module",
"license": "MIT",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"packageManager": "yarn@4.9.2",
"files": [
"dist/**",
"README.md"
],
"devDependencies": {
"typescript": "^5.9.2"
},
"scripts": {
"tsc-build": "npm exec tsc",
"build": "npm run tsc-build",
"test": "echo 'no tests'"
}
}

127
bindings/javascript/sync/packages/common/run.ts Normal file
View File

@@ -0,0 +1,127 @@
"use strict";
import { GeneratorResponse, ProtocolIo, RunOpts } from "./types.js";
const GENERATOR_RESUME_IO = 0;
const GENERATOR_RESUME_DONE = 1;
interface TrackPromise<T> {
promise: Promise<T>,
finished: boolean
}
function trackPromise<T>(p: Promise<T>): TrackPromise<T> {
let status = { promise: null, finished: false };
status.promise = p.finally(() => status.finished = true);
return status;
}
function timeoutMs(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms))
}
async function process(opts: RunOpts, io: ProtocolIo, request: any) {
const requestType = request.request();
const completion = request.completion();
if (requestType.type == 'Http') {
try {
let headers = opts.headers;
if (requestType.headers != null && requestType.headers.length > 0) {
headers = { ...opts.headers };
for (let header of requestType.headers) {
headers[header[0]] = header[1];
}
}
const response = await fetch(`${opts.url}${requestType.path}`, {
method: requestType.method,
headers: headers,
body: requestType.body != null ? new Uint8Array(requestType.body) : null,
});
completion.status(response.status);
const reader = response.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) {
completion.done();
break;
}
completion.pushBuffer(value);
}
} catch (error) {
completion.poison(`fetch error: ${error}`);
}
} else if (requestType.type == 'FullRead') {
try {
const metadata = await io.read(requestType.path);
if (metadata != null) {
completion.pushBuffer(metadata);
}
completion.done();
} catch (error) {
completion.poison(`metadata read error: ${error}`);
}
} else if (requestType.type == 'FullWrite') {
try {
await io.write(requestType.path, requestType.content);
completion.done();
} catch (error) {
completion.poison(`metadata write error: ${error}`);
}
} else if (requestType.type == 'Transform') {
if (opts.transform == null) {
completion.poison("transform is not set");
return;
}
const results = [];
for (const mutation of requestType.mutations) {
const result = opts.transform(mutation);
if (result == null) {
results.push({ type: 'Keep' });
} else if (result.operation == 'skip') {
results.push({ type: 'Skip' });
} else if (result.operation == 'rewrite') {
results.push({ type: 'Rewrite', stmt: result.stmt });
} else {
completion.poison("unexpected transform operation");
return;
}
}
completion.pushTransform(results);
completion.done();
}
}
export function memoryIO(): ProtocolIo {
let values = new Map();
return {
async read(path: string): Promise<Buffer | Uint8Array | null> {
return values.get(path);
},
async write(path: string, data: Buffer | Uint8Array): Promise<void> {
values.set(path, data);
}
}
};
export async function run(opts: RunOpts, io: ProtocolIo, engine: any, generator: any): Promise<any> {
let tasks = [];
while (true) {
const { type, ...rest }: GeneratorResponse = await generator.resumeAsync(null);
if (type == 'Done') {
return null;
}
if (type == 'SyncEngineStats') {
return rest;
}
for (let request = engine.protocolIo(); request != null; request = engine.protocolIo()) {
tasks.push(trackPromise(process(opts, io, request)));
}
const tasksRace = tasks.length == 0 ? Promise.resolve() : Promise.race([timeoutMs(opts.preemptionMs), ...tasks.map(t => t.promise)]);
await Promise.all([engine.ioLoopAsync(), tasksRace]);
tasks = tasks.filter(t => !t.finished);
}
return generator.take();
}

17
bindings/javascript/sync/packages/common/tsconfig.json Normal file
View File

@@ -0,0 +1,17 @@
{
"compilerOptions": {
"skipLibCheck": true,
"declaration": true,
"declarationMap": true,
"module": "esnext",
"target": "esnext",
"outDir": "dist/",
"lib": [
"es2020",
"dom"
],
},
"include": [
"*"
]
}

50
bindings/javascript/sync/packages/common/types.ts Normal file
View File

@@ -0,0 +1,50 @@
export declare const enum DatabaseChangeType {
Insert = 0,
Update = 1,
Delete = 2
}
export interface DatabaseRowMutation {
changeTime: number
tableName: string
id: number
changeType: DatabaseChangeType
before?: Record<string, any>
after?: Record<string, any>
updates?: Record<string, any>
}
export type DatabaseRowTransformResult = { operation: 'skip' } | { operation: 'rewrite', stmt: DatabaseRowStatement } | null;
export type Transform = (arg: DatabaseRowMutation) => DatabaseRowTransformResult;
export interface RunOpts {
preemptionMs: number,
url: string,
headers: { [K: string]: string }
transform?: Transform,
}
export interface ProtocolIo {
read(path: string): Promise<Buffer | Uint8Array | null>;
write(path: string, content: Buffer | Uint8Array): Promise<void>;
}
export interface SyncOpts {
path: string;
clientName?: string;
url: string;
authToken?: string;
encryptionKey?: string;
tablesIgnore?: string[],
transform?: Transform,
tracing?: string,
}
export interface DatabaseRowStatement {
sql: string
values: Array<any>
}
export type GeneratorResponse =
| { type: 'IO' }
| { type: 'Done' }
| { type: 'SyncEngineStats', operations: number, mainWal: number, revertWal: number, lastPullUnixTime: number, lastPushUnixTime: number | null }

125
bindings/javascript/sync/packages/native/README.md Normal file
View File

@@ -0,0 +1,125 @@
<p align="center">
<h1 align="center">Turso Database for JavaScript in Node</h1>
</p>
<p align="center">
<a title="JavaScript" target="_blank" href="https://www.npmjs.com/package/@tursodatabase/database"><img alt="npm" src="https://img.shields.io/npm/v/@tursodatabase/database"></a>
<a title="MIT" target="_blank" href="https://github.com/tursodatabase/turso/blob/main/LICENSE.md"><img src="http://img.shields.io/badge/license-MIT-orange.svg?style=flat-square"></a>
</p>
<p align="center">
<a title="Users Discord" target="_blank" href="https://tur.so/discord"><img alt="Chat with other users of Turso on Discord" src="https://img.shields.io/discord/933071162680958986?label=Discord&logo=Discord&style=social"></a>
</p>
---
## About
This package is the Turso embedded database library for JavaScript in Node.
> **⚠️ Warning:** This software is ALPHA, only use for development, testing, and experimentation. We are working to make it production ready, but do not use it for critical data right now.
## Features
- **SQLite compatible:** SQLite query language and file format support ([status](https://github.com/tursodatabase/turso/blob/main/COMPAT.md)).
- **In-process**: No network overhead, runs directly in your Node.js process
- **TypeScript support**: Full TypeScript definitions included
- **Cross-platform**: Supports Linux (x86 and arm64), macOS, Windows (browser is supported in the separate package `@tursodatabase/database-browser` package)
## Installation
```bash
npm install @tursodatabase/database
```
## Getting Started
### In-Memory Database
```javascript
import { connect } from '@tursodatabase/database';
// Create an in-memory database
const db = await connect(':memory:');
// Create a table
await db.exec('CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, email TEXT)');
// Insert data
const insert = db.prepare('INSERT INTO users (name, email) VALUES (?, ?)');
await insert.run('Alice', 'alice@example.com');
await insert.run('Bob', 'bob@example.com');
// Query data
const users = await db.prepare('SELECT * FROM users').all();
console.log(users);
// Output: [
// { id: 1, name: 'Alice', email: 'alice@example.com' },
// { id: 2, name: 'Bob', email: 'bob@example.com' }
// ]
```
### File-Based Database
```javascript
import { connect } from '@tursodatabase/database';
// Create or open a database file
const db = await connect('my-database.db');
// Create a table
await db.exec(`
CREATE TABLE IF NOT EXISTS posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
content TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
`);
// Insert a post
const insertPost = db.prepare('INSERT INTO posts (title, content) VALUES (?, ?)');
const result = await insertPost.run('Hello World', 'This is my first blog post!');
console.log(`Inserted post with ID: ${result.lastInsertRowid}`);
```
### Transactions
```javascript
import { connect } from '@tursodatabase/database';
const db = await connect('transactions.db');
// Using transactions for atomic operations
const transaction = db.transaction(async (users) => {
const insert = db.prepare('INSERT INTO users (name, email) VALUES (?, ?)');
for (const user of users) {
await insert.run(user.name, user.email);
}
});
// Execute transaction
await transaction([
{ name: 'Alice', email: 'alice@example.com' },
{ name: 'Bob', email: 'bob@example.com' }
]);
```
## API Reference
For complete API documentation, see [JavaScript API Reference](../../../../docs/javascript-api-reference.md).
## Related Packages
* The [@tursodatabase/serverless](https://www.npmjs.com/package/@tursodatabase/serverless) package provides a serverless driver with the same API.
* The [@tursodatabase/sync](https://www.npmjs.com/package/@tursodatabase/sync) package provides bidirectional sync between a local Turso database and Turso Cloud.
## License
This project is licensed under the [MIT license](../../LICENSE.md).
## Support
- [GitHub Issues](https://github.com/tursodatabase/turso/issues)
- [Documentation](https://docs.turso.tech)
- [Discord Community](https://tur.so/discord)

244
bindings/javascript/sync/packages/native/index.d.ts vendored Normal file
View File

@@ -0,0 +1,244 @@
/* auto-generated by NAPI-RS */
/* eslint-disable */
/** A database connection. */
export declare class Database {
/**
* Creates a new database instance.
*
* # Arguments
* * `path` - The path to the database file.
*/
constructor(path: string, opts?: DatabaseOpts | undefined | null)
/** Returns whether the database is in memory-only mode. */
get memory(): boolean
/** Returns whether the database is in memory-only mode. */
get path(): string
/** Returns whether the database connection is open. */
get open(): boolean
/**
* Executes a batch of SQL statements on main thread
*
* # Arguments
*
* * `sql` - The SQL statements to execute.
*
* # Returns
*/
batchSync(sql: string): void
/**
* Executes a batch of SQL statements outside of main thread
*
* # Arguments
*
* * `sql` - The SQL statements to execute.
*
* # Returns
*/
batchAsync(sql: string): Promise<void>
/**
* Prepares a statement for execution.
*
* # Arguments
*
* * `sql` - The SQL statement to prepare.
*
* # Returns
*
* A `Statement` instance.
*/
prepare(sql: string): Statement
/**
* Returns the rowid of the last row inserted.
*
* # Returns
*
* The rowid of the last row inserted.
*/
lastInsertRowid(): number
/**
* Returns the number of changes made by the last statement.
*
* # Returns
*
* The number of changes made by the last statement.
*/
changes(): number
/**
* Returns the total number of changes made by all statements.
*
* # Returns
*
* The total number of changes made by all statements.
*/
totalChanges(): number
/**
* Closes the database connection.
*
* # Returns
*
* `Ok(())` if the database is closed successfully.
*/
close(): void
/**
* Sets the default safe integers mode for all statements from this database.
*
* # Arguments
*
* * `toggle` - Whether to use safe integers by default.
*/
defaultSafeIntegers(toggle?: boolean | undefined | null): void
/** Runs the I/O loop synchronously. */
ioLoopSync(): void
/** Runs the I/O loop asynchronously, returning a Promise. */
ioLoopAsync(): Promise<void>
}
/** A prepared statement. */
export declare class Statement {
reset(): void
/** Returns the number of parameters in the statement. */
parameterCount(): number
/**
* Returns the name of a parameter at a specific 1-based index.
*
* # Arguments
*
* * `index` - The 1-based parameter index.
*/
parameterName(index: number): string | null
/**
* Binds a parameter at a specific 1-based index with explicit type.
*
* # Arguments
*
* * `index` - The 1-based parameter index.
* * `value_type` - The type constant (0=null, 1=int, 2=float, 3=text, 4=blob).
* * `value` - The value to bind.
*/
bindAt(index: number, value: unknown): void
/**
* Step the statement and return result code (executed on the main thread):
* 1 = Row available, 2 = Done, 3 = I/O needed
*/
stepSync(): number
/**
* Step the statement and return result code (executed on the background thread):
* 1 = Row available, 2 = Done, 3 = I/O needed
*/
stepAsync(): Promise<number>
/** Get the current row data according to the presentation mode */
row(): unknown
/** Sets the presentation mode to raw. */
raw(raw?: boolean | undefined | null): void
/** Sets the presentation mode to pluck. */
pluck(pluck?: boolean | undefined | null): void
/**
* Sets safe integers mode for this statement.
*
* # Arguments
*
* * `toggle` - Whether to use safe integers.
*/
safeIntegers(toggle?: boolean | undefined | null): void
/** Get column information for the statement */
columns(): Promise<any>
/** Finalizes the statement. */
finalize(): void
}
export interface DatabaseOpts {
tracing?: string
}
export declare class GeneratorHolder {
resumeSync(error?: string | undefined | null): GeneratorResponse
resumeAsync(error?: string | undefined | null): Promise<unknown>
}
export declare class JsDataCompletion {
poison(err: string): void
status(value: number): void
pushBuffer(value: Buffer): void
pushTransform(values: Array<DatabaseRowTransformResultJs>): void
done(): void
}
export declare class JsProtocolIo {
takeRequest(): JsProtocolRequestBytes | null
}
export declare class JsProtocolRequestBytes {
request(): JsProtocolRequest
completion(): JsDataCompletion
}
export declare class SyncEngine {
constructor(opts: SyncEngineOpts)
init(): GeneratorHolder
ioLoopSync(): void
/** Runs the I/O loop asynchronously, returning a Promise. */
ioLoopAsync(): Promise<void>
protocolIo(): JsProtocolRequestBytes | null
sync(): GeneratorHolder
push(): GeneratorHolder
stats(): GeneratorHolder
pull(): GeneratorHolder
checkpoint(): GeneratorHolder
open(): Database
close(): void
}
export declare const enum DatabaseChangeTypeJs {
Insert = 0,
Update = 1,
Delete = 2
}
export interface DatabaseOpts {
path: string
}
export interface DatabaseRowMutationJs {
changeTime: number
tableName: string
id: number
changeType: DatabaseChangeTypeJs
before?: Record<string, any>
after?: Record<string, any>
updates?: Record<string, any>
}
export interface DatabaseRowStatementJs {
sql: string
values: Array<any>
}
export type DatabaseRowTransformResultJs =
| { type: 'Keep' }
| { type: 'Skip' }
| { type: 'Rewrite', stmt: DatabaseRowStatementJs }
export type GeneratorResponse =
| { type: 'IO' }
| { type: 'Done' }
| { type: 'SyncEngineStats', operations: number, mainWal: number, revertWal: number, lastPullUnixTime: number, lastPushUnixTime?: number }
export type JsProtocolRequest =
| { type: 'Http', method: string, path: string, body?: Array<number>, headers: Array<[string, string]> }
| { type: 'FullRead', path: string }
| { type: 'FullWrite', path: string, content: Array<number> }
| { type: 'Transform', mutations: Array<DatabaseRowMutationJs> }
export interface SyncEngineOpts {
path: string
clientName?: string
walPullBatchSize?: number
tracing?: string
tablesIgnore?: Array<string>
useTransform: boolean
protocolVersion?: SyncEngineProtocolVersion
}
export declare const enum SyncEngineProtocolVersion {
Legacy = 0,
V1 = 1
}

520
bindings/javascript/sync/packages/native/index.js Normal file
View File

@@ -0,0 +1,520 @@
// prettier-ignore
/* eslint-disable */
// @ts-nocheck
/* auto-generated by NAPI-RS */
import { createRequire } from 'node:module'
const require = createRequire(import.meta.url)
const __dirname = new URL('.', import.meta.url).pathname
const { readFileSync } = require('node:fs')
let nativeBinding = null
const loadErrors = []
const isMusl = () => {
let musl = false
if (process.platform === 'linux') {
musl = isMuslFromFilesystem()
if (musl === null) {
musl = isMuslFromReport()
}
if (musl === null) {
musl = isMuslFromChildProcess()
}
}
return musl
}
const isFileMusl = (f) => f.includes('libc.musl-') || f.includes('ld-musl-')
const isMuslFromFilesystem = () => {
try {
return readFileSync('/usr/bin/ldd', 'utf-8').includes('musl')
} catch {
return null
}
}
const isMuslFromReport = () => {
let report = null
if (typeof process.report?.getReport === 'function') {
process.report.excludeNetwork = true
report = process.report.getReport()
}
if (!report) {
return null
}
if (report.header && report.header.glibcVersionRuntime) {
return false
}
if (Array.isArray(report.sharedObjects)) {
if (report.sharedObjects.some(isFileMusl)) {
return true
}
}
return false
}
const isMuslFromChildProcess = () => {
try {
return require('child_process').execSync('ldd --version', { encoding: 'utf8' }).includes('musl')
} catch (e) {
// If we reach this case, we don't know if the system is musl or not, so is better to just fallback to false
return false
}
}
function requireNative() {
if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) {
try {
nativeBinding = require(process.env.NAPI_RS_NATIVE_LIBRARY_PATH);
} catch (err) {
loadErrors.push(err)
}
} else if (process.platform === 'android') {
if (process.arch === 'arm64') {
try {
return require('./sync.android-arm64.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-android-arm64')
const bindingPackageVersion = require('@tursodatabase/sync-android-arm64/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'arm') {
try {
return require('./sync.android-arm-eabi.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-android-arm-eabi')
const bindingPackageVersion = require('@tursodatabase/sync-android-arm-eabi/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
loadErrors.push(new Error(`Unsupported architecture on Android ${process.arch}`))
}
} else if (process.platform === 'win32') {
if (process.arch === 'x64') {
try {
return require('./sync.win32-x64-msvc.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-win32-x64-msvc')
const bindingPackageVersion = require('@tursodatabase/sync-win32-x64-msvc/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'ia32') {
try {
return require('./sync.win32-ia32-msvc.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-win32-ia32-msvc')
const bindingPackageVersion = require('@tursodatabase/sync-win32-ia32-msvc/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'arm64') {
try {
return require('./sync.win32-arm64-msvc.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-win32-arm64-msvc')
const bindingPackageVersion = require('@tursodatabase/sync-win32-arm64-msvc/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
loadErrors.push(new Error(`Unsupported architecture on Windows: ${process.arch}`))
}
} else if (process.platform === 'darwin') {
try {
return require('./sync.darwin-universal.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-darwin-universal')
const bindingPackageVersion = require('@tursodatabase/sync-darwin-universal/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
if (process.arch === 'x64') {
try {
return require('./sync.darwin-x64.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-darwin-x64')
const bindingPackageVersion = require('@tursodatabase/sync-darwin-x64/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'arm64') {
try {
return require('./sync.darwin-arm64.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-darwin-arm64')
const bindingPackageVersion = require('@tursodatabase/sync-darwin-arm64/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
loadErrors.push(new Error(`Unsupported architecture on macOS: ${process.arch}`))
}
} else if (process.platform === 'freebsd') {
if (process.arch === 'x64') {
try {
return require('./sync.freebsd-x64.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-freebsd-x64')
const bindingPackageVersion = require('@tursodatabase/sync-freebsd-x64/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'arm64') {
try {
return require('./sync.freebsd-arm64.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-freebsd-arm64')
const bindingPackageVersion = require('@tursodatabase/sync-freebsd-arm64/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
loadErrors.push(new Error(`Unsupported architecture on FreeBSD: ${process.arch}`))
}
} else if (process.platform === 'linux') {
if (process.arch === 'x64') {
if (isMusl()) {
try {
return require('./sync.linux-x64-musl.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-x64-musl')
const bindingPackageVersion = require('@tursodatabase/sync-linux-x64-musl/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
try {
return require('./sync.linux-x64-gnu.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-x64-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-x64-gnu/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
}
} else if (process.arch === 'arm64') {
if (isMusl()) {
try {
return require('./sync.linux-arm64-musl.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-arm64-musl')
const bindingPackageVersion = require('@tursodatabase/sync-linux-arm64-musl/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
try {
return require('./sync.linux-arm64-gnu.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-arm64-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-arm64-gnu/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
}
} else if (process.arch === 'arm') {
if (isMusl()) {
try {
return require('./sync.linux-arm-musleabihf.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-arm-musleabihf')
const bindingPackageVersion = require('@tursodatabase/sync-linux-arm-musleabihf/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
try {
return require('./sync.linux-arm-gnueabihf.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-arm-gnueabihf')
const bindingPackageVersion = require('@tursodatabase/sync-linux-arm-gnueabihf/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
}
} else if (process.arch === 'riscv64') {
if (isMusl()) {
try {
return require('./sync.linux-riscv64-musl.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-riscv64-musl')
const bindingPackageVersion = require('@tursodatabase/sync-linux-riscv64-musl/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
try {
return require('./sync.linux-riscv64-gnu.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-riscv64-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-riscv64-gnu/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
}
} else if (process.arch === 'ppc64') {
try {
return require('./sync.linux-ppc64-gnu.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-ppc64-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-ppc64-gnu/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 's390x') {
try {
return require('./sync.linux-s390x-gnu.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-linux-s390x-gnu')
const bindingPackageVersion = require('@tursodatabase/sync-linux-s390x-gnu/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
loadErrors.push(new Error(`Unsupported architecture on Linux: ${process.arch}`))
}
} else if (process.platform === 'openharmony') {
if (process.arch === 'arm64') {
try {
return require('./sync.openharmony-arm64.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-openharmony-arm64')
const bindingPackageVersion = require('@tursodatabase/sync-openharmony-arm64/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'x64') {
try {
return require('./sync.openharmony-x64.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-openharmony-x64')
const bindingPackageVersion = require('@tursodatabase/sync-openharmony-x64/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else if (process.arch === 'arm') {
try {
return require('./sync.openharmony-arm.node')
} catch (e) {
loadErrors.push(e)
}
try {
const binding = require('@tursodatabase/sync-openharmony-arm')
const bindingPackageVersion = require('@tursodatabase/sync-openharmony-arm/package.json').version
if (bindingPackageVersion !== '0.1.5-pre.3' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
throw new Error(`Native binding package version mismatch, expected 0.1.5-pre.3 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
}
return binding
} catch (e) {
loadErrors.push(e)
}
} else {
loadErrors.push(new Error(`Unsupported architecture on OpenHarmony: ${process.arch}`))
}
} else {
loadErrors.push(new Error(`Unsupported OS: ${process.platform}, architecture: ${process.arch}`))
}
}
nativeBinding = requireNative()
if (!nativeBinding || process.env.NAPI_RS_FORCE_WASI) {
try {
nativeBinding = require('./sync.wasi.cjs')
} catch (err) {
if (process.env.NAPI_RS_FORCE_WASI) {
loadErrors.push(err)
}
}
if (!nativeBinding) {
try {
nativeBinding = require('@tursodatabase/sync-wasm32-wasi')
} catch (err) {
if (process.env.NAPI_RS_FORCE_WASI) {
loadErrors.push(err)
}
}
}
}
if (!nativeBinding) {
if (loadErrors.length > 0) {
throw new Error(
`Cannot find native binding. ` +
`npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` +
'Please try `npm i` again after removing both package-lock.json and node_modules directory.',
{ cause: loadErrors }
)
}
throw new Error(`Failed to load native binding`)
}
const { Database, Statement, GeneratorHolder, JsDataCompletion, JsProtocolIo, JsProtocolRequestBytes, SyncEngine, DatabaseChangeTypeJs, SyncEngineProtocolVersion } = nativeBinding
export { Database }
export { Statement }
export { GeneratorHolder }
export { JsDataCompletion }
export { JsProtocolIo }
export { JsProtocolRequestBytes }
export { SyncEngine }
export { DatabaseChangeTypeJs }
export { SyncEngineProtocolVersion }

53
bindings/javascript/sync/packages/native/package.json Normal file
View File

@@ -0,0 +1,53 @@
{
"name": "@tursodatabase/sync",
"version": "0.1.5",
"repository": {
"type": "git",
"url": "https://github.com/tursodatabase/turso"
},
"license": "MIT",
"module": "./dist/promise.js",
"main": "./dist/promise.js",
"type": "module",
"exports": {
".": "./dist/promise.js",
"./compat": "./dist/compat.js"
},
"files": [
"index.js",
"dist/**",
"README.md"
],
"packageManager": "yarn@4.9.2",
"devDependencies": {
"@napi-rs/cli": "^3.1.5",
"@types/node": "^24.3.1",
"typescript": "^5.9.2",
"vitest": "^3.2.4"
},
"scripts": {
"napi-build": "napi build --platform --release --esm --manifest-path ../../Cargo.toml --output-dir .",
"napi-dirs": "napi create-npm-dirs",
"napi-artifacts": "napi artifacts --output-dir .",
"tsc-build": "npm exec tsc",
"build": "npm run napi-build && npm run tsc-build",
"test": "vitest --run",
"prepublishOnly": "npm run napi-dirs && npm run napi-artifacts && napi prepublish -t npm"
},
"napi": {
"binaryName": "sync",
"targets": [
"x86_64-unknown-linux-gnu",
"x86_64-pc-windows-msvc",
"universal-apple-darwin",
"aarch64-unknown-linux-gnu"
]
},
"dependencies": {
"@tursodatabase/database-common": "^0.1.5",
"@tursodatabase/sync-common": "^0.1.5"
},
"imports": {
"#index": "./index.js"
}
}

288
bindings/javascript/sync/packages/native/promise.test.ts Normal file
View File

@@ -0,0 +1,288 @@
import { unlinkSync } from "node:fs";
import { expect, test } from 'vitest'
import { connect, DatabaseRowMutation, DatabaseRowTransformResult } from './promise.js'
const localeCompare = (a, b) => a.x.localeCompare(b.x);
test('select-after-push', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS t(x)");
await db.exec("DELETE FROM t");
await db.push();
await db.close();
}
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("INSERT INTO t VALUES (1), (2), (3)");
await db.push();
}
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
const rows = await db.prepare('SELECT * FROM t').all();
expect(rows).toEqual([{ x: 1 }, { x: 2 }, { x: 3 }])
}
})
test('select-without-push', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS t(x)");
await db.exec("DELETE FROM t");
await db.push();
await db.close();
}
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("INSERT INTO t VALUES (1), (2), (3)");
}
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
const rows = await db.prepare('SELECT * FROM t').all();
expect(rows).toEqual([])
}
})
test('merge-non-overlapping-keys', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db1.exec("INSERT INTO q VALUES ('k1', 'value1'), ('k2', 'value2')");
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db2.exec("INSERT INTO q VALUES ('k3', 'value3'), ('k4', 'value4'), ('k5', 'value5')");
await Promise.all([db1.push(), db2.push()]);
await Promise.all([db1.pull(), db2.pull()]);
const rows1 = await db1.prepare('SELECT * FROM q').all();
const rows2 = await db1.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k1', y: 'value1' }, { x: 'k2', y: 'value2' }, { x: 'k3', y: 'value3' }, { x: 'k4', y: 'value4' }, { x: 'k5', y: 'value5' }];
expect(rows1.sort(localeCompare)).toEqual(expected.sort(localeCompare))
expect(rows2.sort(localeCompare)).toEqual(expected.sort(localeCompare))
})
test('last-push-wins', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db1.exec("INSERT INTO q VALUES ('k1', 'value1'), ('k2', 'value2'), ('k4', 'value4')");
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db2.exec("INSERT INTO q VALUES ('k1', 'value3'), ('k2', 'value4'), ('k3', 'value5')");
await db2.push();
await db1.push();
await Promise.all([db1.pull(), db2.pull()]);
const rows1 = await db1.prepare('SELECT * FROM q').all();
const rows2 = await db1.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k1', y: 'value1' }, { x: 'k2', y: 'value2' }, { x: 'k3', y: 'value5' }, { x: 'k4', y: 'value4' }];
expect(rows1.sort(localeCompare)).toEqual(expected.sort(localeCompare))
expect(rows2.sort(localeCompare)).toEqual(expected.sort(localeCompare))
})
test('last-push-wins-with-delete', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db1.exec("INSERT INTO q VALUES ('k1', 'value1'), ('k2', 'value2'), ('k4', 'value4')");
await db1.exec("DELETE FROM q")
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db2.exec("INSERT INTO q VALUES ('k1', 'value3'), ('k2', 'value4'), ('k3', 'value5')");
await db2.push();
await db1.push();
await Promise.all([db1.pull(), db2.pull()]);
const rows1 = await db1.prepare('SELECT * FROM q').all();
const rows2 = await db1.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k3', y: 'value5' }];
expect(rows1).toEqual(expected)
expect(rows2).toEqual(expected)
})
test('constraint-conflict', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS u(x TEXT PRIMARY KEY, y UNIQUE)");
await db.exec("DELETE FROM u");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db1.exec("INSERT INTO u VALUES ('k1', 'value1')");
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db2.exec("INSERT INTO u VALUES ('k2', 'value1')");
await db1.push();
await expect(async () => await db2.push()).rejects.toThrow('SQLite error: UNIQUE constraint failed: u.y');
})
test('checkpoint', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
for (let i = 0; i < 1000; i++) {
await db1.exec(`INSERT INTO q VALUES ('k${i}', 'v${i}')`);
}
expect((await db1.stats()).mainWal).toBeGreaterThan(4096 * 1000);
await db1.checkpoint();
expect((await db1.stats()).mainWal).toBe(0);
let revertWal = (await db1.stats()).revertWal;
expect(revertWal).toBeLessThan(4096 * 1000 / 100);
for (let i = 0; i < 1000; i++) {
await db1.exec(`UPDATE q SET y = 'u${i}' WHERE x = 'k${i}'`);
}
await db1.checkpoint();
expect((await db1.stats()).revertWal).toBe(revertWal);
})
test('persistence', async () => {
{
const db = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL });
await db.exec("CREATE TABLE IF NOT EXISTS q(x TEXT PRIMARY KEY, y)");
await db.exec("DELETE FROM q");
await db.push();
await db.close();
}
const path = `test-${(Math.random() * 10000) | 0}.db`;
try {
{
const db1 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL });
await db1.exec(`INSERT INTO q VALUES ('k1', 'v1')`);
await db1.exec(`INSERT INTO q VALUES ('k2', 'v2')`);
await db1.close();
}
{
const db2 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL });
await db2.exec(`INSERT INTO q VALUES ('k3', 'v3')`);
await db2.exec(`INSERT INTO q VALUES ('k4', 'v4')`);
const rows = await db2.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }, { x: 'k3', y: 'v3' }, { x: 'k4', y: 'v4' }];
expect(rows).toEqual(expected)
await db2.close();
}
{
const db3 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL });
await db3.push();
await db3.close();
}
{
const db4 = await connect({ path: path, url: process.env.VITE_TURSO_DB_URL });
const rows = await db4.prepare('SELECT * FROM q').all();
const expected = [{ x: 'k1', y: 'v1' }, { x: 'k2', y: 'v2' }, { x: 'k3', y: 'v3' }, { x: 'k4', y: 'v4' }];
expect(rows).toEqual(expected)
await db4.close();
}
} finally {
unlinkSync(path);
unlinkSync(`${path}-wal`);
unlinkSync(`${path}-info`);
unlinkSync(`${path}-changes`);
try { unlinkSync(`${path}-revert`) } catch (e) { }
}
})
test('transform', async () => {
{
const db = await connect({
path: ':memory:',
url: process.env.VITE_TURSO_DB_URL,
});
await db.exec("CREATE TABLE IF NOT EXISTS counter(key TEXT PRIMARY KEY, value INTEGER)");
await db.exec("DELETE FROM counter");
await db.exec("INSERT INTO counter VALUES ('1', 0)")
await db.push();
await db.close();
}
const transform = (m: DatabaseRowMutation) => ({
operation: 'rewrite',
stmt: {
sql: `UPDATE counter SET value = value + ? WHERE key = ?`,
values: [m.after.value - m.before.value, m.after.key]
}
} as DatabaseRowTransformResult);
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL, transform: transform });
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL, transform: transform });
await db1.exec("UPDATE counter SET value = value + 1 WHERE key = '1'");
await db2.exec("UPDATE counter SET value = value + 1 WHERE key = '1'");
await Promise.all([db1.push(), db2.push()]);
await Promise.all([db1.pull(), db2.pull()]);
const rows1 = await db1.prepare('SELECT * FROM counter').all();
const rows2 = await db2.prepare('SELECT * FROM counter').all();
expect(rows1).toEqual([{ key: '1', value: 2 }]);
expect(rows2).toEqual([{ key: '1', value: 2 }]);
})
test('transform-many', async () => {
{
const db = await connect({
path: ':memory:',
url: process.env.VITE_TURSO_DB_URL,
});
await db.exec("CREATE TABLE IF NOT EXISTS counter(key TEXT PRIMARY KEY, value INTEGER)");
await db.exec("DELETE FROM counter");
await db.exec("INSERT INTO counter VALUES ('1', 0)")
await db.push();
await db.close();
}
const transform = (m: DatabaseRowMutation) => ({
operation: 'rewrite',
stmt: {
sql: `UPDATE counter SET value = value + ? WHERE key = ?`,
values: [m.after.value - m.before.value, m.after.key]
}
} as DatabaseRowTransformResult);
const db1 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL, transform: transform });
const db2 = await connect({ path: ':memory:', url: process.env.VITE_TURSO_DB_URL, transform: transform });
for (let i = 0; i < 1002; i++) {
await db1.exec("UPDATE counter SET value = value + 1 WHERE key = '1'");
}
for (let i = 0; i < 1001; i++) {
await db2.exec("UPDATE counter SET value = value + 1 WHERE key = '1'");
}
let start = performance.now();
await Promise.all([db1.push(), db2.push()]);
console.info('push', performance.now() - start);
start = performance.now();
await Promise.all([db1.pull(), db2.pull()]);
console.info('pull', performance.now() - start);
const rows1 = await db1.prepare('SELECT * FROM counter').all();
const rows2 = await db2.prepare('SELECT * FROM counter').all();
expect(rows1).toEqual([{ key: '1', value: 1001 + 1002 }]);
expect(rows2).toEqual([{ key: '1', value: 1001 + 1002 }]);
})

104
bindings/javascript/sync/packages/native/promise.ts Normal file
View File

@@ -0,0 +1,104 @@
import { DatabasePromise, DatabaseOpts, NativeDatabase } from "@tursodatabase/database-common"
import { ProtocolIo, run, SyncOpts, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult } from "@tursodatabase/sync-common";
import { Database as NativeDB, SyncEngine } from "#index";
import { promises } from "node:fs";
let NodeIO: ProtocolIo = {
async read(path: string): Promise<Buffer | Uint8Array | null> {
try {
return await promises.readFile(path);
} catch (error) {
if (error.code === 'ENOENT') {
return null;
}
throw error;
}
},
async write(path: string, data: Buffer | Uint8Array): Promise<void> {
const unix = Math.floor(Date.now() / 1000);
const nonce = Math.floor(Math.random() * 1000000000);
const tmp = `${path}.tmp.${unix}.${nonce}`;
await promises.writeFile(tmp, new Uint8Array(data));
try {
await promises.rename(tmp, path);
} catch (err) {
await promises.unlink(tmp);
throw err;
}
}
};
function memoryIO(): ProtocolIo {
let values = new Map();
return {
async read(path: string): Promise<Buffer | Uint8Array | null> {
return values.get(path);
},
async write(path: string, data: Buffer | Uint8Array): Promise<void> {
values.set(path, data);
}
}
};
class Database extends DatabasePromise {
runOpts: RunOpts;
engine: any;
io: ProtocolIo;
constructor(db: NativeDatabase, io: ProtocolIo, runOpts: RunOpts, engine: any, opts: DatabaseOpts = {}) {
super(db, opts)
this.runOpts = runOpts;
this.engine = engine;
this.io = io;
}
async sync() {
await run(this.runOpts, this.io, this.engine, this.engine.sync());
}
async pull() {
await run(this.runOpts, this.io, this.engine, this.engine.pull());
}
async push() {
await run(this.runOpts, this.io, this.engine, this.engine.push());
}
async checkpoint() {
await run(this.runOpts, this.io, this.engine, this.engine.checkpoint());
}
async stats(): Promise<{ operations: number, mainWal: number, revertWal: number, lastPullUnixTime: number, lastPushUnixTime: number | null }> {
return (await run(this.runOpts, this.io, this.engine, this.engine.stats()));
}
override async close(): Promise<void> {
this.engine.close();
}
}
/**
* Creates a new database connection asynchronously.
*
* @param {string} path - Path to the database file.
* @param {Object} opts - Options for database behavior.
* @returns {Promise<Database>} - A promise that resolves to a Database instance.
*/
async function connect(opts: SyncOpts): Promise<Database> {
const engine = new SyncEngine({
path: opts.path,
clientName: opts.clientName,
tablesIgnore: opts.tablesIgnore,
useTransform: opts.transform != null,
tracing: opts.tracing,
protocolVersion: 1
});
const runOpts: RunOpts = {
url: opts.url,
headers: {
...(opts.authToken != null && { "Authorization": `Bearer ${opts.authToken}` }),
...(opts.encryptionKey != null && { "x-turso-encryption-key": opts.encryptionKey })
},
preemptionMs: 1,
transform: opts.transform,
};
let io = opts.path == ':memory:' ? memoryIO() : NodeIO;
await run(runOpts, io, engine, engine.init());
const nativeDb = engine.open();
return new Database(nativeDb as any, io, runOpts, engine, {});
}
export { connect, Database, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult }

21
bindings/javascript/sync/packages/native/tsconfig.json Normal file
View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"skipLibCheck": true,
"declaration": true,
"declarationMap": true,
"module": "nodenext",
"target": "esnext",
"outDir": "dist/",
"lib": [
"es2020"
],
"paths": {
"#index": [
"./index.d.ts"
]
}
},
"include": [
"*"
]
}

102
bindings/javascript/sync/src/generator.rs Normal file
View File

@@ -0,0 +1,102 @@
use napi::{bindgen_prelude::AsyncTask, Env, Task};
use napi_derive::napi;
use std::{
future::Future,
sync::{Arc, Mutex},
};
use turso_sync_engine::types::ProtocolCommand;
pub const GENERATOR_RESUME_IO: u32 = 0;
pub const GENERATOR_RESUME_DONE: u32 = 1;
pub trait Generator {
fn resume(&mut self, result: Option<String>) -> napi::Result<GeneratorResponse>;
}
impl<F: Future<Output = turso_sync_engine::Result<()>>> Generator
for genawaiter::sync::Gen<ProtocolCommand, turso_sync_engine::Result<()>, F>
{
fn resume(&mut self, error: Option<String>) -> napi::Result<GeneratorResponse> {
let result = match error {
Some(err) => Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
format!("JsProtocolIo error: {err}"),
)),
None => Ok(()),
};
match self.resume_with(result) {
genawaiter::GeneratorState::Yielded(ProtocolCommand::IO) => Ok(GeneratorResponse::IO),
genawaiter::GeneratorState::Complete(Ok(())) => Ok(GeneratorResponse::Done),
genawaiter::GeneratorState::Complete(Err(err)) => Err(napi::Error::new(
napi::Status::GenericFailure,
format!("sync engine operation failed: {err}"),
)),
}
}
}
#[napi(discriminant = "type")]
pub enum GeneratorResponse {
IO,
Done,
SyncEngineStats {
operations: i64,
main_wal: i64,
revert_wal: i64,
last_pull_unix_time: i64,
last_push_unix_time: Option<i64>,
},
}
#[napi]
#[derive(Clone)]
pub struct GeneratorHolder {
pub(crate) generator: Arc<Mutex<dyn Generator>>,
pub(crate) response: Arc<Mutex<Option<GeneratorResponse>>>,
}
pub struct ResumeTask {
holder: GeneratorHolder,
error: Option<String>,
}
unsafe impl Send for ResumeTask {}
impl Task for ResumeTask {
type Output = GeneratorResponse;
type JsValue = GeneratorResponse;
fn compute(&mut self) -> napi::Result<Self::Output> {
resume_sync(&self.holder, self.error.take())
}
fn resolve(&mut self, _: Env, output: Self::Output) -> napi::Result<Self::JsValue> {
Ok(output)
}
}
fn resume_sync(holder: &GeneratorHolder, error: Option<String>) -> napi::Result<GeneratorResponse> {
let result = holder.generator.lock().unwrap().resume(error)?;
if let GeneratorResponse::Done = result {
let response = holder.response.lock().unwrap().take();
Ok(response.unwrap_or(GeneratorResponse::Done))
} else {
Ok(result)
}
}
#[napi]
impl GeneratorHolder {
#[napi]
pub fn resume_sync(&self, error: Option<String>) -> napi::Result<GeneratorResponse> {
resume_sync(&self, error)
}
#[napi]
pub fn resume_async(&self, error: Option<String>) -> napi::Result<AsyncTask<ResumeTask>> {
Ok(AsyncTask::new(ResumeTask {
holder: self.clone(),
error,
}))
}
}

283
bindings/javascript/sync/src/js_protocol_io.rs Normal file
View File

@@ -0,0 +1,283 @@
#![deny(clippy::all)]
use std::{
collections::VecDeque,
sync::{Arc, Mutex, MutexGuard},
};
use napi::bindgen_prelude::*;
use napi_derive::napi;
use turso_sync_engine::{
protocol_io::{DataCompletion, DataPollResult, ProtocolIO},
types::{DatabaseRowTransformResult, DatabaseStatementReplay},
};
use crate::{
core_change_type_to_js, core_values_map_to_js, js_value_to_core, DatabaseRowMutationJs,
DatabaseRowTransformResultJs,
};
#[napi]
pub enum JsProtocolRequest {
Http {
method: String,
path: String,
body: Option<Vec<u8>>,
headers: Vec<(String, String)>,
},
FullRead {
path: String,
},
FullWrite {
path: String,
content: Vec<u8>,
},
Transform {
mutations: Vec<DatabaseRowMutationJs>,
},
}
#[derive(Clone)]
#[napi]
pub struct JsDataCompletion(Arc<Mutex<JsDataCompletionInner>>);
pub struct JsBytesPollResult(Buffer);
impl DataPollResult<u8> for JsBytesPollResult {
fn data(&self) -> &[u8] {
&self.0
}
}
pub struct JsTransformPollResult(Vec<DatabaseRowTransformResult>);
impl DataPollResult<DatabaseRowTransformResult> for JsTransformPollResult {
fn data(&self) -> &[DatabaseRowTransformResult] {
&self.0
}
}
struct JsDataCompletionInner {
status: Option<u16>,
chunks: VecDeque<Buffer>,
transformed: VecDeque<DatabaseRowTransformResult>,
finished: bool,
err: Option<String>,
}
impl JsDataCompletion {
fn inner(&self) -> turso_sync_engine::Result<MutexGuard<JsDataCompletionInner>> {
let inner = self.0.lock().unwrap();
if let Some(err) = &inner.err {
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
err.clone(),
));
}
Ok(inner)
}
}
impl DataCompletion<u8> for JsDataCompletion {
type DataPollResult = JsBytesPollResult;
fn status(&self) -> turso_sync_engine::Result<Option<u16>> {
let inner = self.inner()?;
Ok(inner.status)
}
fn poll_data(&self) -> turso_sync_engine::Result<Option<Self::DataPollResult>> {
let mut inner = self.inner()?;
let chunk = inner.chunks.pop_front();
Ok(chunk.map(JsBytesPollResult))
}
fn is_done(&self) -> turso_sync_engine::Result<bool> {
let inner = self.inner()?;
Ok(inner.finished)
}
}
impl DataCompletion<DatabaseRowTransformResult> for JsDataCompletion {
type DataPollResult = JsTransformPollResult;
fn status(&self) -> turso_sync_engine::Result<Option<u16>> {
let inner = self.inner()?;
Ok(inner.status)
}
fn poll_data(&self) -> turso_sync_engine::Result<Option<Self::DataPollResult>> {
let mut inner = self.inner()?;
let chunk = inner.transformed.drain(..).collect::<Vec<_>>();
if chunk.is_empty() {
Ok(None)
} else {
Ok(Some(JsTransformPollResult(chunk)))
}
}
fn is_done(&self) -> turso_sync_engine::Result<bool> {
let inner = self.inner()?;
Ok(inner.finished)
}
}
#[napi]
impl JsDataCompletion {
#[napi]
pub fn poison(&self, err: String) {
let mut completion = self.0.lock().unwrap();
completion.err = Some(err);
}
#[napi]
pub fn status(&self, value: u32) {
let mut completion = self.0.lock().unwrap();
completion.status = Some(value as u16);
}
#[napi]
pub fn push_buffer(&self, value: Buffer) {
let mut completion = self.0.lock().unwrap();
completion.chunks.push_back(value);
}
#[napi]
pub fn push_transform(&self, values: Vec<DatabaseRowTransformResultJs>) {
let mut completion = self.0.lock().unwrap();
for value in values {
completion.transformed.push_back(match value {
DatabaseRowTransformResultJs::Keep => DatabaseRowTransformResult::Keep,
DatabaseRowTransformResultJs::Skip => DatabaseRowTransformResult::Skip,
DatabaseRowTransformResultJs::Rewrite { stmt } => {
DatabaseRowTransformResult::Rewrite(DatabaseStatementReplay {
sql: stmt.sql,
values: stmt.values.into_iter().map(js_value_to_core).collect(),
})
}
});
}
}
#[napi]
pub fn done(&self) {
let mut completion = self.0.lock().unwrap();
completion.finished = true;
}
}
#[napi]
pub struct JsProtocolRequestBytes {
request: Arc<Mutex<Option<JsProtocolRequest>>>,
completion: JsDataCompletion,
}
#[napi]
impl JsProtocolRequestBytes {
#[napi]
pub fn request(&self) -> JsProtocolRequest {
let mut request = self.request.lock().unwrap();
request.take().unwrap()
}
#[napi]
pub fn completion(&self) -> JsDataCompletion {
self.completion.clone()
}
}
impl ProtocolIO for JsProtocolIo {
type DataCompletionBytes = JsDataCompletion;
type DataCompletionTransform = JsDataCompletion;
fn http(
&self,
method: &str,
path: &str,
body: Option<Vec<u8>>,
headers: &[(&str, &str)],
) -> turso_sync_engine::Result<JsDataCompletion> {
Ok(self.add_request(JsProtocolRequest::Http {
method: method.to_string(),
path: path.to_string(),
body,
headers: headers
.iter()
.map(|x| (x.0.to_string(), x.1.to_string()))
.collect(),
}))
}
fn full_read(&self, path: &str) -> turso_sync_engine::Result<Self::DataCompletionBytes> {
Ok(self.add_request(JsProtocolRequest::FullRead {
path: path.to_string(),
}))
}
fn full_write(
&self,
path: &str,
content: Vec<u8>,
) -> turso_sync_engine::Result<Self::DataCompletionBytes> {
Ok(self.add_request(JsProtocolRequest::FullWrite {
path: path.to_string(),
content,
}))
}
fn transform(
&self,
mutations: Vec<turso_sync_engine::types::DatabaseRowMutation>,
) -> turso_sync_engine::Result<Self::DataCompletionTransform> {
Ok(self.add_request(JsProtocolRequest::Transform {
mutations: mutations
.into_iter()
.map(|mutation| DatabaseRowMutationJs {
change_time: mutation.change_time as i64,
table_name: mutation.table_name,
id: mutation.id,
change_type: core_change_type_to_js(mutation.change_type),
before: mutation.before.map(core_values_map_to_js),
after: mutation.after.map(core_values_map_to_js),
updates: mutation.updates.map(core_values_map_to_js),
})
.collect(),
}))
}
}
#[napi]
pub struct JsProtocolIo {
requests: Mutex<Vec<JsProtocolRequestBytes>>,
}
impl Default for JsProtocolIo {
fn default() -> Self {
Self {
requests: Mutex::new(Vec::new()),
}
}
}
#[napi]
impl JsProtocolIo {
#[napi]
pub fn take_request(&self) -> Option<JsProtocolRequestBytes> {
self.requests.lock().unwrap().pop()
}
fn add_request(&self, request: JsProtocolRequest) -> JsDataCompletion {
let completion = JsDataCompletionInner {
chunks: VecDeque::new(),
transformed: VecDeque::new(),
finished: false,
err: None,
status: None,
};
let completion = JsDataCompletion(Arc::new(Mutex::new(completion)));
let mut requests = self.requests.lock().unwrap();
requests.push(JsProtocolRequestBytes {
request: Arc::new(Mutex::new(Some(request))),
completion: completion.clone(),
});
completion
}
}

413
bindings/javascript/sync/src/lib.rs Normal file
View File

@@ -0,0 +1,413 @@
#![allow(clippy::await_holding_lock)]
#![allow(clippy::type_complexity)]
pub mod generator;
pub mod js_protocol_io;
use std::{
collections::HashMap,
sync::{Arc, Mutex, OnceLock, RwLock, RwLockReadGuard, RwLockWriteGuard},
};
use napi::bindgen_prelude::{AsyncTask, Either5, Null};
use napi_derive::napi;
use tracing_subscriber::{filter::LevelFilter, fmt::format::FmtSpan};
use turso_node::IoLoopTask;
use turso_sync_engine::{
database_sync_engine::{DatabaseSyncEngine, DatabaseSyncEngineOpts},
types::{Coro, DatabaseChangeType, DatabaseSyncEngineProtocolVersion},
};
use crate::{
generator::{GeneratorHolder, GeneratorResponse},
js_protocol_io::{JsProtocolIo, JsProtocolRequestBytes},
};
#[napi(object)]
pub struct DatabaseOpts {
pub path: String,
}
#[napi]
pub struct SyncEngine {
path: String,
client_name: String,
wal_pull_batch_size: u32,
protocol_version: DatabaseSyncEngineProtocolVersion,
tables_ignore: Vec<String>,
use_transform: bool,
io: Option<Arc<dyn turso_core::IO>>,
protocol: Option<Arc<JsProtocolIo>>,
sync_engine: Arc<RwLock<Option<DatabaseSyncEngine<JsProtocolIo>>>>,
opened: Arc<Mutex<Option<turso_node::Database>>>,
}
#[napi]
pub enum DatabaseChangeTypeJs {
Insert,
Update,
Delete,
}
#[napi]
pub enum SyncEngineProtocolVersion {
Legacy,
V1,
}
fn core_change_type_to_js(value: DatabaseChangeType) -> DatabaseChangeTypeJs {
match value {
DatabaseChangeType::Delete => DatabaseChangeTypeJs::Delete,
DatabaseChangeType::Update => DatabaseChangeTypeJs::Update,
DatabaseChangeType::Insert => DatabaseChangeTypeJs::Insert,
}
}
fn js_value_to_core(value: Either5<Null, i64, f64, String, Vec<u8>>) -> turso_core::Value {
match value {
Either5::A(_) => turso_core::Value::Null,
Either5::B(value) => turso_core::Value::Integer(value),
Either5::C(value) => turso_core::Value::Float(value),
Either5::D(value) => turso_core::Value::Text(turso_core::types::Text::new(&value)),
Either5::E(value) => turso_core::Value::Blob(value),
}
}
fn core_value_to_js(value: turso_core::Value) -> Either5<Null, i64, f64, String, Vec<u8>> {
match value {
turso_core::Value::Null => Either5::<Null, i64, f64, String, Vec<u8>>::A(Null),
turso_core::Value::Integer(value) => Either5::<Null, i64, f64, String, Vec<u8>>::B(value),
turso_core::Value::Float(value) => Either5::<Null, i64, f64, String, Vec<u8>>::C(value),
turso_core::Value::Text(value) => {
Either5::<Null, i64, f64, String, Vec<u8>>::D(value.as_str().to_string())
}
turso_core::Value::Blob(value) => Either5::<Null, i64, f64, String, Vec<u8>>::E(value),
}
}
fn core_values_map_to_js(
value: HashMap<String, turso_core::Value>,
) -> HashMap<String, Either5<Null, i64, f64, String, Vec<u8>>> {
let mut result = HashMap::new();
for (key, value) in value {
result.insert(key, core_value_to_js(value));
}
result
}
#[napi(object)]
pub struct DatabaseRowMutationJs {
pub change_time: i64,
pub table_name: String,
pub id: i64,
pub change_type: DatabaseChangeTypeJs,
pub before: Option<HashMap<String, Either5<Null, i64, f64, String, Vec<u8>>>>,
pub after: Option<HashMap<String, Either5<Null, i64, f64, String, Vec<u8>>>>,
pub updates: Option<HashMap<String, Either5<Null, i64, f64, String, Vec<u8>>>>,
}
#[napi(object)]
#[derive(Debug)]
pub struct DatabaseRowStatementJs {
pub sql: String,
pub values: Vec<Either5<Null, i64, f64, String, Vec<u8>>>,
}
#[napi(discriminant = "type")]
#[derive(Debug)]
pub enum DatabaseRowTransformResultJs {
Keep,
Skip,
Rewrite { stmt: DatabaseRowStatementJs },
}
#[napi(object, object_to_js = false)]
pub struct SyncEngineOpts {
pub path: String,
pub client_name: Option<String>,
pub wal_pull_batch_size: Option<u32>,
pub tracing: Option<String>,
pub tables_ignore: Option<Vec<String>>,
pub use_transform: bool,
pub protocol_version: Option<SyncEngineProtocolVersion>,
}
static TRACING_INIT: OnceLock<()> = OnceLock::new();
pub fn init_tracing(level_filter: LevelFilter) {
TRACING_INIT.get_or_init(|| {
tracing_subscriber::fmt()
.with_ansi(false)
.with_thread_ids(true)
.with_span_events(FmtSpan::ACTIVE)
.with_max_level(level_filter)
.init();
});
}
#[napi]
impl SyncEngine {
#[napi(constructor)]
pub fn new(opts: SyncEngineOpts) -> napi::Result<Self> {
// helpful for local debugging
match opts.tracing.as_deref() {
Some("info") => init_tracing(LevelFilter::INFO),
Some("debug") => init_tracing(LevelFilter::DEBUG),
Some("trace") => init_tracing(LevelFilter::TRACE),
_ => {}
}
let is_memory = opts.path == ":memory:";
let io: Arc<dyn turso_core::IO> = if is_memory {
Arc::new(turso_core::MemoryIO::new())
} else {
#[cfg(not(feature = "browser"))]
{
Arc::new(turso_core::PlatformIO::new().map_err(|e| {
napi::Error::new(
napi::Status::GenericFailure,
format!("Failed to create IO: {e}"),
)
})?)
}
#[cfg(feature = "browser")]
{
Arc::new(turso_node::browser::Opfs::new()?)
}
};
Ok(SyncEngine {
path: opts.path,
client_name: opts.client_name.unwrap_or("turso-sync-js".to_string()),
wal_pull_batch_size: opts.wal_pull_batch_size.unwrap_or(100),
tables_ignore: opts.tables_ignore.unwrap_or_default(),
use_transform: opts.use_transform,
#[allow(clippy::arc_with_non_send_sync)]
sync_engine: Arc::new(RwLock::new(None)),
io: Some(io),
protocol: Some(Arc::new(JsProtocolIo::default())),
#[allow(clippy::arc_with_non_send_sync)]
opened: Arc::new(Mutex::new(None)),
protocol_version: match opts.protocol_version {
Some(SyncEngineProtocolVersion::Legacy) | None => {
DatabaseSyncEngineProtocolVersion::Legacy
}
_ => DatabaseSyncEngineProtocolVersion::V1,
},
})
}
#[napi]
pub fn init(&mut self) -> napi::Result<GeneratorHolder> {
let opts = DatabaseSyncEngineOpts {
client_name: self.client_name.clone(),
wal_pull_batch_size: self.wal_pull_batch_size as u64,
tables_ignore: self.tables_ignore.clone(),
use_transform: self.use_transform,
protocol_version_hint: self.protocol_version,
};
let io = self.io()?;
let protocol = self.protocol()?;
let sync_engine = self.sync_engine.clone();
let opened = self.opened.clone();
let path = self.path.clone();
let generator = genawaiter::sync::Gen::new(|coro| async move {
let coro = Coro::new((), coro);
let initialized =
DatabaseSyncEngine::new(&coro, io.clone(), protocol, &path, opts).await?;
let connection = initialized.connect_rw(&coro).await?;
let db = turso_node::Database::create(None, io.clone(), connection, path);
*sync_engine.write().unwrap() = Some(initialized);
*opened.lock().unwrap() = Some(db);
Ok(())
});
Ok(GeneratorHolder {
generator: Arc::new(Mutex::new(generator)),
response: Arc::new(Mutex::new(None)),
})
}
#[napi]
pub fn io_loop_sync(&self) -> napi::Result<()> {
self.io()?.step().map_err(|e| {
napi::Error::new(napi::Status::GenericFailure, format!("IO error: {e}"))
})?;
Ok(())
}
/// Runs the I/O loop asynchronously, returning a Promise.
#[napi(ts_return_type = "Promise<void>")]
pub fn io_loop_async(&self) -> napi::Result<AsyncTask<IoLoopTask>> {
let io = self.io()?;
Ok(AsyncTask::new(IoLoopTask { io }))
}
#[napi]
pub fn protocol_io(&self) -> napi::Result<Option<JsProtocolRequestBytes>> {
Ok(self.protocol()?.take_request())
}
#[napi]
pub fn sync(&self) -> GeneratorHolder {
self.run(async move |coro, sync_engine| {
let mut sync_engine = try_write(sync_engine)?;
let sync_engine = try_unwrap_mut(&mut sync_engine)?;
sync_engine.sync(coro).await?;
Ok(None)
})
}
#[napi]
pub fn push(&self) -> GeneratorHolder {
self.run(async move |coro, sync_engine| {
let sync_engine = try_read(sync_engine)?;
let sync_engine = try_unwrap(&sync_engine)?;
sync_engine.push_changes_to_remote(coro).await?;
Ok(None)
})
}
#[napi]
pub fn stats(&self) -> GeneratorHolder {
self.run(async move |coro, sync_engine| {
let sync_engine = try_read(sync_engine)?;
let sync_engine = try_unwrap(&sync_engine)?;
let changes = sync_engine.stats(coro).await?;
Ok(Some(GeneratorResponse::SyncEngineStats {
operations: changes.cdc_operations,
main_wal: changes.main_wal_size as i64,
revert_wal: changes.revert_wal_size as i64,
last_pull_unix_time: changes.last_pull_unix_time,
last_push_unix_time: changes.last_push_unix_time,
}))
})
}
#[napi]
pub fn pull(&self) -> GeneratorHolder {
self.run(async move |coro, sync_engine| {
let mut sync_engine = try_write(sync_engine)?;
let sync_engine = try_unwrap_mut(&mut sync_engine)?;
sync_engine.pull_changes_from_remote(coro).await?;
Ok(None)
})
}
#[napi]
pub fn checkpoint(&self) -> GeneratorHolder {
self.run(async move |coro, sync_engine| {
let mut sync_engine = try_write(sync_engine)?;
let sync_engine = try_unwrap_mut(&mut sync_engine)?;
sync_engine.checkpoint(coro).await?;
Ok(None)
})
}
#[napi]
pub fn open(&self) -> napi::Result<turso_node::Database> {
let opened = self.opened.lock().unwrap();
let Some(opened) = opened.as_ref() else {
return Err(napi::Error::new(
napi::Status::GenericFailure,
"sync_engine must be initialized".to_string(),
));
};
Ok(opened.clone())
}
#[napi]
pub fn close(&mut self) {
let _ = self.sync_engine.write().unwrap().take();
let _ = self.opened.lock().unwrap().take().unwrap();
let _ = self.io.take();
let _ = self.protocol.take();
}
fn io(&self) -> napi::Result<Arc<dyn turso_core::IO>> {
if self.io.is_none() {
return Err(napi::Error::new(
napi::Status::GenericFailure,
"sync engine was closed",
));
}
Ok(self.io.as_ref().unwrap().clone())
}
fn protocol(&self) -> napi::Result<Arc<JsProtocolIo>> {
if self.protocol.is_none() {
return Err(napi::Error::new(
napi::Status::GenericFailure,
"sync engine was closed",
));
}
Ok(self.protocol.as_ref().unwrap().clone())
}
fn run(
&self,
f: impl AsyncFnOnce(
&Coro<()>,
&Arc<RwLock<Option<DatabaseSyncEngine<JsProtocolIo>>>>,
) -> turso_sync_engine::Result<Option<GeneratorResponse>>
+ 'static,
) -> GeneratorHolder {
let response = Arc::new(Mutex::new(None));
let sync_engine = self.sync_engine.clone();
#[allow(clippy::await_holding_lock)]
let generator = genawaiter::sync::Gen::new({
let response = response.clone();
|coro| async move {
let coro = Coro::new((), coro);
*response.lock().unwrap() = f(&coro, &sync_engine).await?;
Ok(())
}
});
GeneratorHolder {
generator: Arc::new(Mutex::new(generator)),
response,
}
}
}
fn try_read(
sync_engine: &RwLock<Option<DatabaseSyncEngine<JsProtocolIo>>>,
) -> turso_sync_engine::Result<RwLockReadGuard<'_, Option<DatabaseSyncEngine<JsProtocolIo>>>> {
let Ok(sync_engine) = sync_engine.try_read() else {
let nasty_error = "sync_engine is busy".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
nasty_error,
));
};
Ok(sync_engine)
}
fn try_write(
sync_engine: &RwLock<Option<DatabaseSyncEngine<JsProtocolIo>>>,
) -> turso_sync_engine::Result<RwLockWriteGuard<'_, Option<DatabaseSyncEngine<JsProtocolIo>>>> {
let Ok(sync_engine) = sync_engine.try_write() else {
let nasty_error = "sync_engine is busy".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
nasty_error,
));
};
Ok(sync_engine)
}
fn try_unwrap<'a>(
sync_engine: &'a RwLockReadGuard<'_, Option<DatabaseSyncEngine<JsProtocolIo>>>,
) -> turso_sync_engine::Result<&'a DatabaseSyncEngine<JsProtocolIo>> {
let Some(sync_engine) = sync_engine.as_ref() else {
let error = "sync_engine must be initialized".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
error,
));
};
Ok(sync_engine)
}
fn try_unwrap_mut<'a>(
sync_engine: &'a mut RwLockWriteGuard<'_, Option<DatabaseSyncEngine<JsProtocolIo>>>,
) -> turso_sync_engine::Result<&'a mut DatabaseSyncEngine<JsProtocolIo>> {
let Some(sync_engine) = sync_engine.as_mut() else {
let error = "sync_engine must be initialized".to_string();
return Err(turso_sync_engine::errors::Error::DatabaseSyncEngineError(
error,
));
};
Ok(sync_engine)
}