opfs for sync in one commit!

bindings/javascript/packages/browser-common/README.md

@@ -0,0 +1,8 @@
+## About
+
+This package is the Turso embedded database common JS library which is shared between final builds for Node and Browser.
+
+Do not use this package directly - instead you must use `@tursodatabase/database` or `@tursodatabase/database-browser`.
+
+> **⚠️ Warning:** This software is ALPHA, only use for development, testing, and experimentation. We are working to make it production ready, but do not use it for critical data right now.
+

bindings/javascript/packages/browser-common/index.ts

@@ -0,0 +1,239 @@
+function getUint8ArrayFromMemory(memory: WebAssembly.Memory, ptr: number, len: number): Uint8Array {
+    ptr = ptr >>> 0;
+    return new Uint8Array(memory.buffer).subarray(ptr, ptr + len);
+}
+
+function getStringFromMemory(memory: WebAssembly.Memory, ptr: number, len: number): string {
+    const shared = getUint8ArrayFromMemory(memory, ptr, len);
+    const copy = new Uint8Array(shared.length);
+    copy.set(shared);
+    const decoder = new TextDecoder('utf-8');
+    return decoder.decode(copy);
+}
+
+interface BrowserImports {
+    is_web_worker(): boolean;
+    lookup_file(ptr: number, len: number): number;
+    read(handle: number, ptr: number, len: number, offset: number): number;
+    write(handle: number, ptr: number, len: number, offset: number): number;
+    sync(handle: number): number;
+    truncate(handle: number, len: number): number;
+    size(handle: number): number;
+}
+
+function panic(name): never {
+    throw new Error(`method ${name} must be invoked only from the main thread`);
+}
+
+const MainDummyImports: BrowserImports = {
+    is_web_worker: function (): boolean {
+        return false;
+    },
+    lookup_file: function (ptr: number, len: number): number {
+        panic("lookup_file")
+    },
+    read: function (handle: number, ptr: number, len: number, offset: number): number {
+        panic("read")
+    },
+    write: function (handle: number, ptr: number, len: number, offset: number): number {
+        panic("write")
+    },
+    sync: function (handle: number): number {
+        panic("sync")
+    },
+    truncate: function (handle: number, len: number): number {
+        panic("truncate")
+    },
+    size: function (handle: number): number {
+        panic("size")
+    }
+};
+
+function workerImports(opfs: OpfsDirectory, memory: WebAssembly.Memory): BrowserImports {
+    return {
+        is_web_worker: function (): boolean {
+            return true;
+        },
+        lookup_file: function (ptr: number, len: number): number {
+            try {
+                const handle = opfs.lookupFileHandle(getStringFromMemory(memory, ptr, len));
+                return handle == null ? -404 : handle;
+            } catch (e) {
+                return -1;
+            }
+        },
+        read: function (handle: number, ptr: number, len: number, offset: number): number {
+            try {
+                return opfs.read(handle, getUint8ArrayFromMemory(memory, ptr, len), offset);
+            } catch (e) {
+                return -1;
+            }
+        },
+        write: function (handle: number, ptr: number, len: number, offset: number): number {
+            try {
+                return opfs.write(handle, getUint8ArrayFromMemory(memory, ptr, len), offset)
+            } catch (e) {
+                return -1;
+            }
+        },
+        sync: function (handle: number): number {
+            try {
+                opfs.sync(handle);
+                return 0;
+            } catch (e) {
+                return -1;
+            }
+        },
+        truncate: function (handle: number, len: number): number {
+            try {
+                opfs.truncate(handle, len);
+                return 0;
+            } catch (e) {
+                return -1;
+            }
+        },
+        size: function (handle: number): number {
+            try {
+                return opfs.size(handle);
+            } catch (e) {
+                return -1;
+            }
+        }
+    }
+}
+
+class OpfsDirectory {
+    fileByPath: Map<String, { handle: number, sync: FileSystemSyncAccessHandle }>;
+    fileByHandle: Map<number, FileSystemSyncAccessHandle>;
+    fileHandleNo: number;
+
+    constructor() {
+        this.fileByPath = new Map();
+        this.fileByHandle = new Map();
+        this.fileHandleNo = 0;
+    }
+
+    async registerFile(path: string) {
+        if (this.fileByPath.has(path)) {
+            return;
+        }
+        const opfsRoot = await navigator.storage.getDirectory();
+        const opfsHandle = await opfsRoot.getFileHandle(path, { create: true });
+        const opfsSync = await opfsHandle.createSyncAccessHandle();
+        this.fileHandleNo += 1;
+        this.fileByPath.set(path, { handle: this.fileHandleNo, sync: opfsSync });
+        this.fileByHandle.set(this.fileHandleNo, opfsSync);
+    }
+
+    async unregisterFile(path: string) {
+        const file = this.fileByPath.get(path);
+        if (file == null) {
+            return;
+        }
+        this.fileByPath.delete(path);
+        this.fileByHandle.delete(file.handle);
+        file.sync.close();
+    }
+    lookupFileHandle(path: string): number | null {
+        try {
+            const file = this.fileByPath.get(path);
+            if (file == null) {
+                return null;
+            }
+            return file.handle;
+        } catch (e) {
+            console.error('lookupFile', path, e);
+            throw e;
+        }
+    }
+    read(handle: number, buffer: Uint8Array, offset: number): number {
+        try {
+            const file = this.fileByHandle.get(handle);
+            const result = file.read(buffer, { at: Number(offset) });
+            return result;
+        } catch (e) {
+            console.error('read', handle, buffer.length, offset, e);
+            throw e;
+        }
+    }
+    write(handle: number, buffer: Uint8Array, offset: number): number {
+        try {
+            const file = this.fileByHandle.get(handle);
+            const result = file.write(buffer, { at: Number(offset) });
+            return result;
+        } catch (e) {
+            console.error('write', handle, buffer.length, offset, e);
+            throw e;
+        }
+    }
+    sync(handle: number) {
+        try {
+            const file = this.fileByHandle.get(handle);
+            file.flush();
+        } catch (e) {
+            console.error('sync', handle, e);
+            throw e;
+        }
+    }
+    truncate(handle: number, size: number) {
+        try {
+            const file = this.fileByHandle.get(handle);
+            const result = file.truncate(size);
+            return result;
+        } catch (e) {
+            console.error('truncate', handle, size, e);
+            throw e;
+        }
+    }
+    size(handle: number): number {
+        try {
+            const file = this.fileByHandle.get(handle);
+            const size = file.getSize()
+            return size;
+        } catch (e) {
+            console.error('size', handle, e);
+            throw e;
+        }
+    }
+}
+
+var workerRequestId = 0;
+function waitForWorkerResponse(worker: Worker, id: number): Promise<any> {
+    let waitResolve, waitReject;
+    const callback = msg => {
+        if (msg.data.id == id) {
+            if (msg.data.error != null) {
+                waitReject(msg.data.error)
+            } else {
+                waitResolve()
+            }
+            cleanup();
+        }
+    };
+    const cleanup = () => worker.removeEventListener("message", callback);
+
+    worker.addEventListener("message", callback);
+    const result = new Promise((resolve, reject) => {
+        waitResolve = resolve;
+        waitReject = reject;
+    });
+    return result;
+}
+
+function registerFileAtWorker(worker: Worker, path: string): Promise<void> {
+    workerRequestId += 1;
+    const currentId = workerRequestId;
+    const promise = waitForWorkerResponse(worker, currentId);
+    worker.postMessage({ __turso__: "register", path: path, id: currentId });
+    return promise;
+}
+
+function unregisterFileAtWorker(worker: Worker, path: string): Promise<void> {
+    workerRequestId += 1;
+    const currentId = workerRequestId;
+    const promise = waitForWorkerResponse(worker, currentId);
+    worker.postMessage({ __turso__: "unregister", path: path, id: currentId });
+    return promise;
+}
+
+export { OpfsDirectory, workerImports, MainDummyImports, waitForWorkerResponse, registerFileAtWorker, unregisterFileAtWorker }

bindings/javascript/packages/browser-common/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@tursodatabase/database-browser-common",
+  "version": "0.1.5",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/tursodatabase/turso"
+  },
+  "type": "module",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "packageManager": "yarn@4.9.2",
+  "files": [
+    "dist/**",
+    "README.md"
+  ],
+  "devDependencies": {
+    "typescript": "^5.9.2"
+  },
+  "scripts": {
+    "tsc-build": "npm exec tsc",
+    "build": "npm run tsc-build",
+    "test": "echo 'no tests'"
+  }
+}

bindings/javascript/packages/browser-common/tsconfig.json

@@ -0,0 +1,18 @@
+{
+  "compilerOptions": {
+    "skipLibCheck": true,
+    "declaration": true,
+    "declarationMap": true,
+    "module": "esnext",
+    "target": "esnext",
+    "outDir": "dist/",
+    "lib": [
+      "es2020",
+      "DOM",
+      "WebWorker"
+    ],
+  },
+  "include": [
+    "*"
+  ]
+}

bindings/javascript/packages/browser/index.js

@@ -5,6 +5,7 @@ import {
   WASI as __WASI,
 } from '@napi-rs/wasm-runtime'
 
+import { MainDummyImports } from "@tursodatabase/database-browser-common";
 
 
 const __wasi = new __WASI({
@@ -25,10 +26,6 @@ const __wasmFile = await fetch(__wasmUrl).then((res) => res.arrayBuffer())
 
 export let MainWorker = null;
 
-function panic(name) {
-  throw new Error(`method ${name} must be invoked only from the main thread`);
-}
-
 const {
   instance: __napiInstance,
   module: __wasiModule,
@@ -49,14 +46,8 @@ const {
       ...importObject.env,
       ...importObject.napi,
       ...importObject.emnapi,
+      ...MainDummyImports,
       memory: __sharedMemory,
-      is_web_worker: () => false,
-      lookup_file: () => panic("lookup_file"),
-      read: () => panic("read"),
-      write: () => panic("write"),
-      sync: () => panic("sync"),
-      truncate: () => panic("truncate"),
-      size: () => panic("size"),
     }
     return importObject
   },

bindings/javascript/packages/browser/package.json

@@ -40,6 +40,7 @@
   },
   "dependencies": {
     "@napi-rs/wasm-runtime": "^1.0.3",
+    "@tursodatabase/database-browser-common": "^0.1.5",
     "@tursodatabase/database-common": "^0.1.5"
   }
 }

bindings/javascript/packages/browser/promise.ts

@@ -1,50 +1,24 @@
-import { DatabasePromise, NativeDatabase, DatabaseOpts, SqliteError } from "@tursodatabase/database-common"
+import { registerFileAtWorker, unregisterFileAtWorker } from "@tursodatabase/database-browser-common"
+import { DatabasePromise, NativeDatabase, DatabaseOpts, SqliteError, } from "@tursodatabase/database-common"
 import { connect as nativeConnect, initThreadPool, MainWorker } from "#index";
 
-let workerRequestId = 0;
 class Database extends DatabasePromise {
-    files: string[];
-    constructor(db: NativeDatabase, files: string[], opts: DatabaseOpts = {}) {
+    path: string | null;
+    constructor(db: NativeDatabase, fsPath: string | null, opts: DatabaseOpts = {}) {
         super(db, opts)
-        this.files = files;
+        this.path = fsPath;
     }
     async close() {
-        let currentId = workerRequestId;
-        workerRequestId += this.files.length;
-
-        let tasks = [];
-        for (const file of this.files) {
-            (MainWorker as any).postMessage({ __turso__: "unregister", path: file, id: currentId });
-            tasks.push(waitFor(currentId));
-            currentId += 1;
+        if (this.path != null) {
+            await Promise.all([
+                unregisterFileAtWorker(MainWorker, this.path),
+                unregisterFileAtWorker(MainWorker, `${this.path}-wal`)
+            ]);
         }
-        await Promise.all(tasks);
         this.db.close();
     }
 }
 
-function waitFor(id: number): Promise<any> {
-    let waitResolve, waitReject;
-    const callback = msg => {
-        if (msg.data.id == id) {
-            if (msg.data.error != null) {
-                waitReject(msg.data.error)
-            } else {
-                waitResolve()
-            }
-            cleanup();
-        }
-    };
-    const cleanup = () => (MainWorker as any).removeEventListener("message", callback);
-
-    (MainWorker as any).addEventListener("message", callback);
-    const result = new Promise((resolve, reject) => {
-        waitResolve = resolve;
-        waitReject = reject;
-    });
-    return result;
-}
-
 /**
  * Creates a new database connection asynchronously.
  * 
@@ -55,24 +29,18 @@ function waitFor(id: number): Promise<any> {
 async function connect(path: string, opts: DatabaseOpts = {}): Promise<Database> {
     if (path == ":memory:") {
         const db = await nativeConnect(path, { tracing: opts.tracing });
-        return new Database(db, [], opts);
+        return new Database(db, null, opts);
     }
     await initThreadPool();
     if (MainWorker == null) {
         throw new Error("panic: MainWorker is not set");
     }
-
-    let currentId = workerRequestId;
-    workerRequestId += 2;
-
-    let dbHandlePromise = waitFor(currentId);
-    let walHandlePromise = waitFor(currentId + 1);
-    (MainWorker as any).postMessage({ __turso__: "register", path: `${path}`, id: currentId });
-    (MainWorker as any).postMessage({ __turso__: "register", path: `${path}-wal`, id: currentId + 1 });
-    await Promise.all([dbHandlePromise, walHandlePromise]);
+    await Promise.all([
+        registerFileAtWorker(MainWorker, path),
+        registerFileAtWorker(MainWorker, `${path}-wal`)
+    ]);
     const db = await nativeConnect(path, { tracing: opts.tracing });
-    const files = [path, `${path}-wal`];
-    return new Database(db, files, opts);
+    return new Database(db, path, opts);
 }
 
 export { connect, Database, SqliteError }

bindings/javascript/packages/browser/tsconfig.json

@@ -5,6 +5,7 @@
     "declarationMap": true,
     "module": "nodenext",
     "target": "esnext",
+    "moduleResolution": "nodenext",
     "outDir": "dist/",
     "lib": [
       "es2020"

bindings/javascript/packages/browser/worker.mjs

@@ -1,108 +1,9 @@
 import { instantiateNapiModuleSync, MessageHandler, WASI } from '@napi-rs/wasm-runtime'
+import { OpfsDirectory, workerImports } from '@tursodatabase/database-browser-common';
 
-var fileByPath = new Map();
-var fileByHandle = new Map();
-let fileHandles = 0;
+var opfs = new OpfsDirectory();
 var memory = null;
 
-function getUint8ArrayFromWasm(ptr, len) {
-  ptr = ptr >>> 0;
-  return new Uint8Array(memory.buffer).subarray(ptr, ptr + len);
-}
-
-
-async function registerFile(path) {
-  if (fileByPath.has(path)) {
-    return;
-  }
-  const opfsRoot = await navigator.storage.getDirectory();
-  const opfsHandle = await opfsRoot.getFileHandle(path, { create: true });
-  const opfsSync = await opfsHandle.createSyncAccessHandle();
-  fileHandles += 1;
-  fileByPath.set(path, { handle: fileHandles, sync: opfsSync });
-  fileByHandle.set(fileHandles, opfsSync);
-}
-
-async function unregisterFile(path) {
-  const file = fileByPath.get(path);
-  if (file == null) {
-    return;
-  }
-  fileByPath.delete(path);
-  fileByHandle.delete(file.handle);
-  file.sync.close();
-}
-
-function lookup_file(pathPtr, pathLen) {
-  try {
-    const buffer = getUint8ArrayFromWasm(pathPtr, pathLen);
-    const notShared = new Uint8Array(buffer.length);
-    notShared.set(buffer);
-    const decoder = new TextDecoder('utf-8');
-    const path = decoder.decode(notShared);
-    const file = fileByPath.get(path);
-    if (file == null) {
-      return -404;
-    }
-    return file.handle;
-  } catch (e) {
-    console.error('lookupFile', pathPtr, pathLen, e);
-    return -1;
-  }
-}
-function read(handle, bufferPtr, bufferLen, offset) {
-  try {
-    const buffer = getUint8ArrayFromWasm(bufferPtr, bufferLen);
-    const file = fileByHandle.get(Number(handle));
-    const result = file.read(buffer, { at: Number(offset) });
-    return result;
-  } catch (e) {
-    console.error('read', handle, bufferPtr, bufferLen, offset, e);
-    return -1;
-  }
-}
-function write(handle, bufferPtr, bufferLen, offset) {
-  try {
-    const buffer = getUint8ArrayFromWasm(bufferPtr, bufferLen);
-    const file = fileByHandle.get(Number(handle));
-    const result = file.write(buffer, { at: Number(offset) });
-    return result;
-  } catch (e) {
-    console.error('write', handle, bufferPtr, bufferLen, offset, e);
-    return -1;
-  }
-}
-function sync(handle) {
-  try {
-    const file = fileByHandle.get(Number(handle));
-    file.flush();
-    return 0;
-  } catch (e) {
-    console.error('sync', handle, e);
-    return -1;
-  }
-}
-function truncate(handle, size) {
-  try {
-    const file = fileByHandle.get(Number(handle));
-    const result = file.truncate(size);
-    return result;
-  } catch (e) {
-    console.error('truncate', handle, size, e);
-    return -1;
-  }
-}
-function size(handle) {
-  try {
-    const file = fileByHandle.get(Number(handle));
-    const size = file.getSize()
-    return size;
-  } catch (e) {
-    console.error('size', handle, e);
-    return -1;
-  }
-}
-
 const handler = new MessageHandler({
   onLoad({ wasmModule, wasmMemory }) {
     memory = wasmMemory;
@@ -124,14 +25,8 @@ const handler = new MessageHandler({
           ...importObject.env,
           ...importObject.napi,
           ...importObject.emnapi,
+          ...workerImports(opfs, memory),
           memory: wasmMemory,
-          is_web_worker: () => true,
-          lookup_file: lookup_file,
-          read: read,
-          write: write,
-          sync: sync,
-          truncate: truncate,
-          size: size,
         }
       },
     })
@@ -141,16 +36,16 @@ const handler = new MessageHandler({
 globalThis.onmessage = async function (e) {
   if (e.data.__turso__ == 'register') {
     try {
-      await registerFile(e.data.path)
-      self.postMessage({ id: e.data.id })
+      await opfs.registerFile(e.data.path);
+      self.postMessage({ id: e.data.id });
     } catch (error) {
       self.postMessage({ id: e.data.id, error: error });
     }
     return;
   } else if (e.data.__turso__ == 'unregister') {
     try {
-      await unregisterFile(e.data.path)
-      self.postMessage({ id: e.data.id })
+      await opfs.unregisterFile(e.data.path);
+      self.postMessage({ id: e.data.id });
     } catch (error) {
       self.postMessage({ id: e.data.id, error: error });
     }

bindings/javascript/packages/common/types.ts

@@ -18,7 +18,6 @@ export interface NativeDatabase {
 
     prepare(sql: string): NativeStatement;
 
-    pluck(pluckMode: boolean);
     defaultSafeIntegers(toggle: boolean);
     totalChanges(): number;
     changes(): number;
@@ -32,6 +31,11 @@ export const STEP_ROW = 1;
 export const STEP_DONE = 2;
 export const STEP_IO = 3;
 
+export interface TableColumn {
+    name: string,
+    type: string
+}
+
 export interface NativeStatement {
     stepAsync(): Promise<number>;
     stepSync(): number;
@@ -39,7 +43,7 @@ export interface NativeStatement {
     pluck(pluckMode: boolean);
     safeIntegers(toggle: boolean);
     raw(toggle: boolean);
-    columns(): string[];
+    columns(): TableColumn[];
     row(): any;
     reset();
     finalize();

bindings/javascript/packages/native/index.d.ts

@@ -91,6 +91,14 @@ export declare class Database {
   ioLoopAsync(): Promise<void>
 }
 
+export declare class Opfs {
+  constructor()
+}
+
+export declare class OpfsFile {
+
+}
+
 /** A prepared statement. */
 export declare class Statement {
   reset(): void
@@ -144,6 +152,14 @@ export declare class Statement {
   finalize(): void
 }
 
+export declare function connect(path: string, opts?: DatabaseOpts | undefined | null): Promise<unknown>
+
 export interface DatabaseOpts {
   tracing?: string
 }
+
+/**
+ * turso-db in the the browser requires explicit thread pool initialization
+ * so, we just put no-op task on the thread pool and force emnapi to allocate web worker
+ */
+export declare function initThreadPool(): Promise<unknown>

bindings/javascript/packages/native/index.js

@@ -508,6 +508,10 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { Database, Statement } = nativeBinding
+const { Database, Opfs, OpfsFile, Statement, connect, initThreadPool } = nativeBinding
 export { Database }
+export { Opfs }
+export { OpfsFile }
 export { Statement }
+export { connect }
+export { initThreadPool }