opfs for sync in one commit!

bindings/javascript/sync/packages/common/README.md

@@ -0,0 +1,8 @@
+## About
+
+This package is the Turso Sync common JS library which is shared between final builds for Node and Browser.
+
+Do not use this package directly - instead you must use `@tursodatabase/sync` or `@tursodatabase/sync-browser`.
+
+> **⚠️ Warning:** This software is ALPHA, only use for development, testing, and experimentation. We are working to make it production ready, but do not use it for critical data right now.
+

bindings/javascript/sync/packages/common/index.ts

@@ -0,0 +1,5 @@
+import { run, memoryIO } from "./run.js"
+import { SyncOpts, ProtocolIo, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult } from "./types.js"
+
+export { run, memoryIO, }
+export type { SyncOpts, ProtocolIo, RunOpts, DatabaseRowMutation, DatabaseRowStatement, DatabaseRowTransformResult }

bindings/javascript/sync/packages/common/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@tursodatabase/sync-common",
+  "version": "0.1.5",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/tursodatabase/turso"
+  },
+  "type": "module",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "packageManager": "yarn@4.9.2",
+  "files": [
+    "dist/**",
+    "README.md"
+  ],
+  "devDependencies": {
+    "typescript": "^5.9.2"
+  },
+  "scripts": {
+    "tsc-build": "npm exec tsc",
+    "build": "npm run tsc-build",
+    "test": "echo 'no tests'"
+  }
+}

bindings/javascript/sync/packages/common/run.ts

@@ -0,0 +1,127 @@
+"use strict";
+
+import { GeneratorResponse, ProtocolIo, RunOpts } from "./types.js";
+
+const GENERATOR_RESUME_IO = 0;
+const GENERATOR_RESUME_DONE = 1;
+
+interface TrackPromise<T> {
+    promise: Promise<T>,
+    finished: boolean
+}
+
+function trackPromise<T>(p: Promise<T>): TrackPromise<T> {
+    let status = { promise: null, finished: false };
+    status.promise = p.finally(() => status.finished = true);
+    return status;
+}
+
+function timeoutMs(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+async function process(opts: RunOpts, io: ProtocolIo, request: any) {
+    const requestType = request.request();
+    const completion = request.completion();
+    if (requestType.type == 'Http') {
+        try {
+            let headers = opts.headers;
+            if (requestType.headers != null && requestType.headers.length > 0) {
+                headers = { ...opts.headers };
+                for (let header of requestType.headers) {
+                    headers[header[0]] = header[1];
+                }
+            }
+            const response = await fetch(`${opts.url}${requestType.path}`, {
+                method: requestType.method,
+                headers: headers,
+                body: requestType.body != null ? new Uint8Array(requestType.body) : null,
+            });
+            completion.status(response.status);
+            const reader = response.body.getReader();
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    completion.done();
+                    break;
+                }
+                completion.pushBuffer(value);
+            }
+        } catch (error) {
+            completion.poison(`fetch error: ${error}`);
+        }
+    } else if (requestType.type == 'FullRead') {
+        try {
+            const metadata = await io.read(requestType.path);
+            if (metadata != null) {
+                completion.pushBuffer(metadata);
+            }
+            completion.done();
+        } catch (error) {
+            completion.poison(`metadata read error: ${error}`);
+        }
+    } else if (requestType.type == 'FullWrite') {
+        try {
+            await io.write(requestType.path, requestType.content);
+            completion.done();
+        } catch (error) {
+            completion.poison(`metadata write error: ${error}`);
+        }
+    } else if (requestType.type == 'Transform') {
+        if (opts.transform == null) {
+            completion.poison("transform is not set");
+            return;
+        }
+        const results = [];
+        for (const mutation of requestType.mutations) {
+            const result = opts.transform(mutation);
+            if (result == null) {
+                results.push({ type: 'Keep' });
+            } else if (result.operation == 'skip') {
+                results.push({ type: 'Skip' });
+            } else if (result.operation == 'rewrite') {
+                results.push({ type: 'Rewrite', stmt: result.stmt });
+            } else {
+                completion.poison("unexpected transform operation");
+                return;
+            }
+        }
+        completion.pushTransform(results);
+        completion.done();
+    }
+}
+
+export function memoryIO(): ProtocolIo {
+    let values = new Map();
+    return {
+        async read(path: string): Promise<Buffer | Uint8Array | null> {
+            return values.get(path);
+        },
+        async write(path: string, data: Buffer | Uint8Array): Promise<void> {
+            values.set(path, data);
+        }
+    }
+};
+
+
+export async function run(opts: RunOpts, io: ProtocolIo, engine: any, generator: any): Promise<any> {
+    let tasks = [];
+    while (true) {
+        const { type, ...rest }: GeneratorResponse = await generator.resumeAsync(null);
+        if (type == 'Done') {
+            return null;
+        }
+        if (type == 'SyncEngineStats') {
+            return rest;
+        }
+        for (let request = engine.protocolIo(); request != null; request = engine.protocolIo()) {
+            tasks.push(trackPromise(process(opts, io, request)));
+        }
+
+        const tasksRace = tasks.length == 0 ? Promise.resolve() : Promise.race([timeoutMs(opts.preemptionMs), ...tasks.map(t => t.promise)]);
+        await Promise.all([engine.ioLoopAsync(), tasksRace]);
+
+        tasks = tasks.filter(t => !t.finished);
+    }
+    return generator.take();
+}

bindings/javascript/sync/packages/common/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "skipLibCheck": true,
+    "declaration": true,
+    "declarationMap": true,
+    "module": "esnext",
+    "target": "esnext",
+    "outDir": "dist/",
+    "lib": [
+      "es2020",
+      "dom"
+    ],
+  },
+  "include": [
+    "*"
+  ]
+}

bindings/javascript/sync/packages/common/types.ts

@@ -0,0 +1,50 @@
+export declare const enum DatabaseChangeType {
+    Insert = 0,
+    Update = 1,
+    Delete = 2
+}
+
+export interface DatabaseRowMutation {
+    changeTime: number
+    tableName: string
+    id: number
+    changeType: DatabaseChangeType
+    before?: Record<string, any>
+    after?: Record<string, any>
+    updates?: Record<string, any>
+}
+
+export type DatabaseRowTransformResult = { operation: 'skip' } | { operation: 'rewrite', stmt: DatabaseRowStatement } | null;
+export type Transform = (arg: DatabaseRowMutation) => DatabaseRowTransformResult;
+export interface RunOpts {
+    preemptionMs: number,
+    url: string,
+    headers: { [K: string]: string }
+    transform?: Transform,
+}
+
+export interface ProtocolIo {
+    read(path: string): Promise<Buffer | Uint8Array | null>;
+    write(path: string, content: Buffer | Uint8Array): Promise<void>;
+}
+
+export interface SyncOpts {
+    path: string;
+    clientName?: string;
+    url: string;
+    authToken?: string;
+    encryptionKey?: string;
+    tablesIgnore?: string[],
+    transform?: Transform,
+    tracing?: string,
+}
+
+export interface DatabaseRowStatement {
+    sql: string
+    values: Array<any>
+}
+
+export type GeneratorResponse =
+    | { type: 'IO' }
+    | { type: 'Done' }
+    | { type: 'SyncEngineStats', operations: number, mainWal: number, revertWal: number, lastPullUnixTime: number, lastPushUnixTime: number | null }