large refactor

package.json

@@ -64,6 +64,7 @@
     "@swc-node/register": "^1.10.9",
     "@swc/core": "^1.10.18",
     "@types/better-sqlite3": "^7.6.12",
+    "@types/bun": "^1.2.4",
     "@types/cors": "^2.8.17",
     "@types/debug": "^4.1.12",
     "@types/express": "^4.17.21",

src/app/database.ts

@@ -34,14 +34,14 @@ export default class LocalDatabase extends EventEmitter {
     // Detect architecture to pass the correct native sqlite module
     this.db = new Database(this.path.main);
 
-    if (this.config.wal) this.db.pragma("journal_mode = WAL");
+    if (this.config.wal) this.db.exec("PRAGMA journal_mode = WAL");
   }
 
   hasTable(table: string) {
     const result = this.db
-      .prepare(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='table' AND name=?`)
+      .prepare<[string], { count: number }>(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='table' AND name=?`)
-      .get([table]) as { count: number };
+      .get(table);
-    return result.count > 0;
+    return !!result && result.count > 0;
   }
 
   // Delete all events in the database

src/app/index.ts

@@ -1,8 +1,6 @@
 import { WebSocketServer } from "ws";
-import { createServer, Server } from "http";
+import { Server } from "http";
 import { SimpleSigner } from "applesauce-signers/signers/simple-signer";
-import { IEventStore, NostrRelay, SQLiteEventStore } from "@satellite-earth/core";
-import { getDMRecipient } from "@satellite-earth/core/helpers/nostr";
 import { kinds } from "nostr-tools";
 import { AbstractRelay } from "nostr-tools/abstract-relay";
 import express, { Express } from "express";
@@ -48,9 +46,12 @@ import secrets from "../services/secrets.js";
 import config from "../services/config.js";
 import logStore from "../services/log-store.js";
 import stateManager from "../services/state.js";
-import eventCache from "../services/event-cache.js";
+import sqliteEventStore from "../services/event-cache.js";
 import { inboundNetwork, outboundNetwork } from "../services/network.js";
 import { server } from "../services/server.js";
+import { SQLiteEventStore } from "../sqlite/event-store.js";
+import { NostrRelay } from "../relay/nostr-relay.js";
+import { getDMRecipient } from "../helpers/nostr/dms.js";
 
 type EventMap = {
   listening: [];
@@ -71,7 +72,7 @@ export default class App extends EventEmitter<EventMap> {
   outboundNetwork: OutboundNetworkManager;
 
   database: Database;
-  eventStore: IEventStore;
+  eventStore: SQLiteEventStore;
   logStore: LogStore;
   relay: NostrRelay;
   receiver: Receiver;
@@ -114,7 +115,7 @@ export default class App extends EventEmitter<EventMap> {
 
     // create http and ws server interface
     this.server = server;
-    this.inboundNetwork = inboundNetwork
+    this.inboundNetwork = inboundNetwork;
     this.outboundNetwork = outboundNetwork;
 
     /** make the outbound network reflect the app config */
@@ -153,7 +154,7 @@ export default class App extends EventEmitter<EventMap> {
     });
 
     // Initialize the event store
-    this.eventStore = eventCache;
+    this.eventStore = sqliteEventStore;
 
     // setup decryption cache
     this.decryptionCache = new DecryptionCache(this.database.db);

src/classes/json-file.ts

@@ -0,0 +1,100 @@
+import { EventEmitter } from "events";
+import { Adapter, Low, LowSync, SyncAdapter } from "lowdb";
+
+type EventMap<T> = {
+  /** fires when file is loaded */
+  loaded: [T];
+  /** fires when a field is set */
+  changed: [T, string, any];
+  /** fires when file is loaded or changed */
+  updated: [T];
+  saved: [T];
+};
+
+export class ReactiveJsonFile<T extends object> extends EventEmitter<EventMap<T>> implements Low<T> {
+  protected db: Low<T>;
+  adapter: Adapter<T>;
+
+  data: T;
+
+  constructor(adapter: Adapter<T>, defaultData: T) {
+    super();
+
+    this.adapter = adapter;
+    this.db = new Low<T>(adapter, defaultData);
+
+    this.data = this.createProxy();
+  }
+
+  private createProxy() {
+    return (this.data = new Proxy(this.db.data, {
+      get(target, prop, receiver) {
+        return Reflect.get(target, prop, receiver);
+      },
+      set: (target, p, newValue, receiver) => {
+        Reflect.set(target, p, newValue, receiver);
+        this.emit("changed", target as T, String(p), newValue);
+        this.emit("updated", target as T);
+        return true;
+      },
+    }));
+  }
+
+  async read() {
+    await this.db.read();
+    this.emit("loaded", this.db.data);
+    this.emit("updated", this.db.data);
+    this.createProxy();
+  }
+  async write() {
+    await this.db.write();
+    this.emit("saved", this.db.data);
+  }
+  update(fn: (data: T) => unknown) {
+    return this.db.update(fn);
+  }
+}
+
+export class ReactiveJsonFileSync<T extends object> extends EventEmitter<EventMap<T>> implements LowSync<T> {
+  protected db: LowSync<T>;
+  adapter: SyncAdapter<T>;
+
+  data: T;
+
+  constructor(adapter: SyncAdapter<T>, defaultData: T) {
+    super();
+
+    this.adapter = adapter;
+    this.db = new LowSync<T>(adapter, defaultData);
+
+    this.data = this.createProxy();
+  }
+
+  private createProxy() {
+    return (this.data = new Proxy(this.db.data, {
+      get(target, prop, receiver) {
+        return Reflect.get(target, prop, receiver);
+      },
+      set: (target, p, newValue, receiver) => {
+        Reflect.set(target, p, newValue, receiver);
+        this.emit("changed", target as T, String(p), newValue);
+        this.emit("updated", target as T);
+        return true;
+      },
+    }));
+  }
+
+  read() {
+    this.db.read();
+    this.emit("loaded", this.db.data);
+    this.emit("updated", this.db.data);
+    this.createProxy();
+  }
+  write() {
+    this.db.write();
+    this.emit("saved", this.db.data);
+  }
+  update(fn: (data: T) => unknown) {
+    return this.db.update(fn);
+  }
+}

src/helpers/nostr/channel.ts

@@ -0,0 +1,21 @@
+import { NostrEvent } from 'nostr-tools';
+
+import { getTagValue } from './event.js';
+import { COMMUNITY_CHANNEL_KIND } from './kinds.js';
+
+export const CHANNEL_KIND = COMMUNITY_CHANNEL_KIND;
+
+export function getChannelId(channel: NostrEvent) {
+	const id = getTagValue(channel, 'd');
+	if (!id) throw new Error('Channel missing id');
+	return id;
+}
+export function getChannelName(channel: NostrEvent) {
+	return getTagValue(channel, 'name');
+}
+export function getChannelAbout(channel: NostrEvent) {
+	return getTagValue(channel, 'about');
+}
+export function getChannelPicture(channel: NostrEvent) {
+	return getTagValue(channel, 'picture');
+}

src/helpers/nostr/communities.ts

@@ -0,0 +1,18 @@
+import { NostrEvent } from 'nostr-tools';
+
+const tagValue = (e: NostrEvent, k: string) => e.tags.find((t) => t[0] === k)?.[1];
+export function getCommunityName(definition: NostrEvent) {
+	return tagValue(definition, 'name');
+}
+export function getCommunityBanner(definition: NostrEvent) {
+	return tagValue(definition, 'banner');
+}
+export function getCommunityImage(definition: NostrEvent) {
+	return tagValue(definition, 'image');
+}
+export function getCommunityRelay(definition: NostrEvent) {
+	return tagValue(definition, 'r');
+}
+export function getCommunityCDN(definition: NostrEvent) {
+	return tagValue(definition, 'cdn');
+}

src/helpers/nostr/contacts.ts

@@ -0,0 +1,20 @@
+import { NostrEvent } from 'nostr-tools';
+import { safeRelayUrl } from './relays.js';
+
+export function getRelaysFromContactList(event: NostrEvent) {
+	try {
+		const json = JSON.parse(event.content) as Record<string, { write?: boolean; read?: boolean }>;
+		const relays: { url: string; write?: boolean; read?: boolean }[] = [];
+
+		for (const [url, value] of Object.entries(json)) {
+			const safeUrl = safeRelayUrl(url);
+			if (safeUrl) {
+				relays.push({ url: safeUrl, write: value.write, read: value.read });
+			}
+		}
+
+		return relays;
+	} catch (error) {
+		return null;
+	}
+}

src/helpers/nostr/dms.ts

@@ -0,0 +1,10 @@
+import { NostrEvent } from 'nostr-tools';
+
+export function getDMSender(event: NostrEvent) {
+	return event.pubkey;
+}
+export function getDMRecipient(event: NostrEvent) {
+	const pubkey = event.tags.find((t) => t[0] === 'p')?.[1];
+	if (!pubkey) throw new Error('Missing recipient pubkey');
+	return pubkey;
+}

src/helpers/nostr/event.ts

@@ -0,0 +1,63 @@
+import { NostrEvent, kinds, nip19 } from 'nostr-tools';
+
+export function isReplaceable(kind: number) {
+	return kinds.isReplaceableKind(kind) || kinds.isParameterizedReplaceableKind(kind);
+}
+
+export function sortByDate(a: NostrEvent, b: NostrEvent) {
+	return b.created_at - a.created_at;
+}
+
+export function getEventCoordinate(event: NostrEvent) {
+	const d = event.tags.find((t) => t[0] === 'd')?.[1];
+	return d ? `${event.kind}:${event.pubkey}:${d}` : `${event.kind}:${event.pubkey}`;
+}
+
+export function getTagValue(event: NostrEvent, tag: string) {
+	return event.tags.find((t) => t[0] === tag)?.[1];
+}
+
+export function doesEventMatchCoordinate(event: NostrEvent, coordinate: string) {
+	const [kind, pubkey, d] = coordinate.split(':');
+	if (!kind || !pubkey || !d) return false;
+	return (
+		event.kind === parseInt(kind) && event.pubkey === event.pubkey && event.tags.find((t) => t[0] === 'd')?.[1] === d
+	);
+}
+
+export type CustomAddressPointer = Omit<nip19.AddressPointer, 'identifier'> & {
+	identifier?: string;
+};
+
+export function parseCoordinate(a: string): CustomAddressPointer | null;
+export function parseCoordinate(a: string, requireD: false): CustomAddressPointer | null;
+export function parseCoordinate(a: string, requireD: true): nip19.AddressPointer | null;
+export function parseCoordinate(a: string, requireD: false, silent: false): CustomAddressPointer;
+export function parseCoordinate(a: string, requireD: true, silent: false): nip19.AddressPointer;
+export function parseCoordinate(a: string, requireD: true, silent: true): nip19.AddressPointer | null;
+export function parseCoordinate(a: string, requireD: false, silent: true): CustomAddressPointer | null;
+export function parseCoordinate(a: string, requireD = false, silent = true): CustomAddressPointer | null {
+	const parts = a.split(':') as (string | undefined)[];
+	const kind = parts[0] && parseInt(parts[0]);
+	const pubkey = parts[1];
+	const d = parts[2];
+
+	if (!kind) {
+		if (silent) return null;
+		else throw new Error('Missing kind');
+	}
+	if (!pubkey) {
+		if (silent) return null;
+		else throw new Error('Missing pubkey');
+	}
+	if (requireD && d === undefined) {
+		if (silent) return null;
+		else throw new Error('Missing identifier');
+	}
+
+	return {
+		kind,
+		pubkey,
+		identifier: d,
+	};
+}

src/helpers/nostr/index.ts

@@ -0,0 +1,7 @@
+export * from './channel.js';
+export * from './communities.js';
+export * from './dms.js';
+export * from './event.js';
+export * from './kinds.js';
+export * from './profile.js';
+export * from './nip19.js';

src/helpers/nostr/kinds.ts

@@ -0,0 +1,2 @@
+export const COMMUNITY_CHANNEL_KIND = 39000;
+export const COMMUNITY_CHAT_MESSAGE = 9;

src/helpers/nostr/lists.ts

@@ -0,0 +1,104 @@
+import { EventTemplate, nip19, NostrEvent } from 'nostr-tools';
+import { parseCoordinate } from './event.js';
+
+function unixNow() {
+	return Math.round(Date.now() / 1000);
+}
+
+export function getPubkeysFromList(event: NostrEvent | EventTemplate) {
+	return event.tags.filter((t) => t[0] === 'p' && t[1]).map((t) => ({ pubkey: t[1], relay: t[2], petname: t[3] }));
+}
+export function getEventPointersFromList(event: NostrEvent | EventTemplate): nip19.EventPointer[] {
+	return event.tags
+		.filter((t) => t[0] === 'e' && t[1])
+		.map((t) => (t[2] ? { id: t[1], relays: [t[2]] } : { id: t[1] }));
+}
+export function getCoordinatesFromList(event: NostrEvent | EventTemplate) {
+	return event.tags.filter((t) => t[0] === 'a' && t[1] && t[2]).map((t) => ({ coordinate: t[1], relay: t[2] }));
+}
+export function getAddressPointersFromList(event: NostrEvent | EventTemplate): nip19.AddressPointer[] {
+	const pointers: nip19.AddressPointer[] = [];
+
+	for (const tag of event.tags) {
+		if (!tag[1]) continue;
+		const relay = tag[2];
+		const parsed = parseCoordinate(tag[1]);
+		if (!parsed?.identifier) continue;
+
+		pointers.push({ ...parsed, identifier: parsed?.identifier, relays: relay ? [relay] : undefined });
+	}
+
+	return pointers;
+}
+
+export function isPubkeyInList(list?: NostrEvent, pubkey?: string) {
+	if (!pubkey || !list) return false;
+	return list.tags.some((t) => t[0] === 'p' && t[1] === pubkey);
+}
+
+export function listAddPerson(
+	list: NostrEvent | EventTemplate,
+	pubkey: string,
+	relay?: string,
+	petname?: string,
+): EventTemplate {
+	if (list.tags.some((t) => t[0] === 'p' && t[1] === pubkey)) throw new Error('Person already in list');
+	const pTag = ['p', pubkey, relay ?? '', petname ?? ''];
+	while (pTag[pTag.length - 1] === '') pTag.pop();
+
+	return {
+		created_at: unixNow(),
+		kind: list.kind,
+		content: list.content,
+		tags: [...list.tags, pTag],
+	};
+}
+
+export function listRemovePerson(list: NostrEvent | EventTemplate, pubkey: string): EventTemplate {
+	return {
+		created_at: unixNow(),
+		kind: list.kind,
+		content: list.content,
+		tags: list.tags.filter((t) => !(t[0] === 'p' && t[1] === pubkey)),
+	};
+}
+
+export function listAddEvent(list: NostrEvent | EventTemplate, event: string, relay?: string): EventTemplate {
+	if (list.tags.some((t) => t[0] === 'e' && t[1] === event)) throw new Error('Event already in list');
+
+	return {
+		created_at: unixNow(),
+		kind: list.kind,
+		content: list.content,
+		tags: [...list.tags, relay ? ['e', event, relay] : ['e', event]],
+	};
+}
+
+export function listRemoveEvent(list: NostrEvent | EventTemplate, event: string): EventTemplate {
+	return {
+		created_at: unixNow(),
+		kind: list.kind,
+		content: list.content,
+		tags: list.tags.filter((t) => !(t[0] === 'e' && t[1] === event)),
+	};
+}
+
+export function listAddCoordinate(list: NostrEvent | EventTemplate, coordinate: string, relay?: string): EventTemplate {
+	if (list.tags.some((t) => t[0] === 'a' && t[1] === coordinate)) throw new Error('Event already in list');
+
+	return {
+		created_at: unixNow(),
+		kind: list.kind,
+		content: list.content,
+		tags: [...list.tags, relay ? ['a', coordinate, relay] : ['a', coordinate]],
+	};
+}
+
+export function listRemoveCoordinate(list: NostrEvent | EventTemplate, coordinate: string): EventTemplate {
+	return {
+		created_at: unixNow(),
+		kind: list.kind,
+		content: list.content,
+		tags: list.tags.filter((t) => !(t[0] === 'a' && t[1] === coordinate)),
+	};
+}

src/helpers/nostr/long-form.ts

@@ -0,0 +1,15 @@
+import { NostrEvent } from 'nostr-tools';
+
+export function getArticleTitle(event: NostrEvent) {
+	return event.tags.find((t) => t[0] === 'title')?.[1];
+}
+export function getArticleSummary(event: NostrEvent) {
+	return event.tags.find((t) => t[0] === 'summary')?.[1];
+}
+export function getArticleImage(event: NostrEvent) {
+	return event.tags.find((t) => t[0] === 'image')?.[1];
+}
+export function getArticlePublishDate(event: NostrEvent) {
+	const timestamp = event.tags.find((t) => t[0] === 'published_at')?.[1];
+	return timestamp ? parseInt(timestamp) : undefined;
+}

src/helpers/nostr/mailboxes.ts

@@ -0,0 +1,17 @@
+import { NostrEvent } from 'nostr-tools';
+import { safeRelayUrls } from './relays.js';
+
+// inbox relays can be ["r", <url>, "read"] or ["r", <url>]
+export function getInboxes(event?: NostrEvent | null, fallback?: string[]) {
+	const tags = event ? event.tags.filter((t) => (t[0] === 'r' && t[2] === 'read') || t[2] === undefined) : [];
+	const urls = safeRelayUrls(tags.map((t) => t[1]));
+	if (fallback && urls.length === 0) return fallback;
+	return urls;
+}
+
+export function getOutboxes(event?: NostrEvent | null, fallback?: string[]) {
+	const tags = event ? event.tags.filter((t) => (t[0] === 'r' && t[2] === 'write') || t[2] === undefined) : [];
+	const urls = safeRelayUrls(tags.map((t) => t[1]));
+	if (fallback && urls.length === 0) return fallback;
+	return urls;
+}

src/helpers/nostr/nip19.ts

@@ -0,0 +1,22 @@
+import { nip19, NostrEvent } from 'nostr-tools';
+import { isReplaceable } from './event.js';
+
+export function getSharableEventAddress(event: NostrEvent, relays?: Iterable<string>) {
+	if (isReplaceable(event.kind)) {
+		const d = event.tags.find((t) => t[0] === 'd' && t[1])?.[1];
+		if (!d) return null;
+		return nip19.naddrEncode({
+			kind: event.kind,
+			identifier: d,
+			pubkey: event.pubkey,
+			relays: relays && Array.from(relays),
+		});
+	} else {
+		return nip19.neventEncode({
+			id: event.id,
+			kind: event.kind,
+			relays: relays && Array.from(relays),
+			author: event.pubkey,
+		});
+	}
+}

src/helpers/nostr/profile.ts

@@ -0,0 +1,51 @@
+import { NostrEvent, nip19 } from 'nostr-tools';
+
+export type Kind0ParsedContent = {
+	pubkey?: string;
+	name?: string;
+	display_name?: string;
+	displayName?: string;
+	about?: string;
+	/** @deprecated */
+	image?: string;
+	picture?: string;
+	banner?: string;
+	website?: string;
+	lud16?: string;
+	lud06?: string;
+	nip05?: string;
+};
+
+export function parseKind0Event(event: NostrEvent): Kind0ParsedContent {
+	if (event.kind !== 0) throw new Error('expected a kind 0 event');
+	try {
+		const metadata = JSON.parse(event.content) as Kind0ParsedContent;
+		metadata.pubkey = event.pubkey;
+
+		// ensure nip05 is a string
+		if (metadata.nip05 && typeof metadata.nip05 !== 'string') metadata.nip05 = String(metadata.nip05);
+
+		// fix user website
+		if (metadata.website) metadata.website = fixWebsiteUrl(metadata.website);
+
+		return metadata;
+	} catch (e) {}
+	return {};
+}
+
+export function getSearchNames(metadata: Kind0ParsedContent) {
+	if (!metadata) return [];
+
+	return [metadata.displayName, metadata.display_name, metadata.name].filter(Boolean) as string[];
+}
+
+export function getUserDisplayName(metadata: Kind0ParsedContent | undefined, pubkey: string) {
+	return metadata?.displayName || metadata?.display_name || metadata?.name || nip19.npubEncode(pubkey).slice(0, 8);
+}
+
+export function fixWebsiteUrl(website: string) {
+	if (website.match(/^http?s:\/\//)) {
+		return website;
+	}
+	return 'https://' + website;
+}

src/helpers/nostr/relays.ts

@@ -0,0 +1,24 @@
+export function validateRelayURL(relay: string | URL) {
+	if (typeof relay === 'string' && relay.includes(',ws')) throw new Error('Can not have multiple relays in one string');
+	const url = typeof relay === 'string' ? new URL(relay) : relay;
+	if (url.protocol !== 'wss:' && url.protocol !== 'ws:') throw new Error('Incorrect protocol');
+	return url;
+}
+export function isValidRelayURL(relay: string | URL) {
+	try {
+		validateRelayURL(relay);
+		return true;
+	} catch (e) {
+		return false;
+	}
+}
+export function safeRelayUrl(relayUrl: string | URL) {
+	try {
+		return validateRelayURL(relayUrl).toString();
+	} catch (e) {
+		return null;
+	}
+}
+export function safeRelayUrls(urls: Iterable<string>): string[] {
+	return Array.from(urls).map(safeRelayUrl).filter(Boolean) as string[];
+}

src/helpers/sql.ts

@@ -0,0 +1,3 @@
+export function mapParams(params: any[]) {
+  return `(${params.map(() => "?").join(", ")})`;
+}

src/helpers/super-map.ts

@@ -0,0 +1,17 @@
+export default class SuperMap<Key, Value> extends Map<Key, Value> {
+	newValue: (key: Key) => Value;
+
+	constructor(newValue: (key: Key) => Value) {
+		super();
+		this.newValue = newValue;
+	}
+
+	get(key: Key) {
+		let value = super.get(key);
+		if (value === undefined) {
+			value = this.newValue(key);
+			this.set(key, value);
+		}
+		return value;
+	}
+}

src/modules/community-multiplexer.ts

@@ -1,89 +0,0 @@
-import { type Database } from "better-sqlite3";
-import { WebSocket, WebSocketServer } from "ws";
-import { type IncomingMessage } from "http";
-import { randomBytes } from "crypto";
-import { NostrEvent, SimplePool } from "nostr-tools";
-
-import { HyperConnectionManager } from "./hyper-connection-manager.js";
-import { logger } from "../logger.js";
-import { CommunityProxy } from "./community-proxy.js";
-import { IEventStore } from "@satellite-earth/core";
-
-export class CommunityMultiplexer {
-  log = logger.extend("community-multiplexer");
-  db: Database;
-  eventStore: IEventStore;
-  pool: SimplePool;
-  connectionManager: HyperConnectionManager;
-
-  communities = new Map<string, CommunityProxy>();
-
-  constructor(db: Database, eventStore: IEventStore) {
-    this.db = db;
-    this.eventStore = eventStore;
-    this.pool = new SimplePool();
-
-    this.connectionManager = new HyperConnectionManager(randomBytes(32).toString("hex"));
-
-    this.syncCommunityDefinitions();
-  }
-
-  attachToServer(wss: WebSocketServer) {
-    wss.on("connection", this.handleConnection.bind(this));
-  }
-
-  handleConnection(ws: WebSocket, req: IncomingMessage) {
-    if (!req.url) return false;
-
-    const url = new URL(req.url, `http://${req.headers.host}`);
-    const pubkey = url.pathname.split("/")[1] as string | undefined;
-    if (!pubkey || pubkey.length !== 64) return false;
-
-    try {
-      let community = this.communities.get(pubkey);
-      if (!community) community = this.getCommunityProxy(pubkey);
-
-      // connect the socket to the relay
-      community.relay.handleConnection(ws, req);
-      return true;
-    } catch (error) {
-      this.log("Failed handle ws connection to", pubkey);
-      console.log(error);
-      return false;
-    }
-  }
-
-  syncCommunityDefinitions() {
-    this.log("Syncing community definitions");
-    const sub = this.pool.subscribeMany(["wss://nostrue.com"], [{ kinds: [12012] }], {
-      onevent: (event) => this.eventStore.addEvent(event),
-      oneose: () => sub.close(),
-    });
-  }
-
-  getCommunityProxy(pubkey: string) {
-    this.log("Looking for community definition", pubkey);
-    let definition: NostrEvent | undefined = undefined;
-
-    const local = this.eventStore.getEventsForFilters([{ kinds: [12012], authors: [pubkey] }]);
-    if (local[0]) definition = local[0];
-
-    if (!definition) throw new Error("Failed to find community definition");
-
-    this.log("Creating community proxy", pubkey);
-    const community = new CommunityProxy(this.db, definition, this.connectionManager);
-
-    community.connect();
-    this.communities.set(pubkey, community);
-
-    return community;
-  }
-
-  stop() {
-    for (const [pubkey, community] of this.communities) {
-      community.stop();
-    }
-    this.communities.clear();
-    this.connectionManager.stop();
-  }
-}

src/modules/community-proxy.ts

@@ -1,175 +0,0 @@
-import { type Database } from "better-sqlite3";
-import { Debugger } from "debug";
-import { Filter, NostrEvent, Relay, kinds } from "nostr-tools";
-import { NostrRelay, RelayActions } from "@satellite-earth/core";
-import { Subscription } from "nostr-tools/abstract-relay";
-
-import { LabeledEventStore } from "./labeled-event-store.js";
-import { HyperConnectionManager } from "./hyper-connection-manager.js";
-import { logger } from "../logger.js";
-
-/** Used to connect to and sync with remote communities */
-export class CommunityProxy {
-  log: Debugger;
-  database: Database;
-  connectionManager: HyperConnectionManager;
-  definition: NostrEvent;
-
-  upstream?: Relay;
-  eventStore: LabeledEventStore;
-  relay: NostrRelay;
-
-  get addresses() {
-    return this.definition.tags.filter((t) => t[0] === "r" && t[1]).map((t) => t[1]);
-  }
-
-  constructor(database: Database, communityDefinition: NostrEvent, connectionManager: HyperConnectionManager) {
-    this.database = database;
-    this.connectionManager = connectionManager;
-    this.definition = communityDefinition;
-    this.log = logger.extend("community-proxy:" + communityDefinition.pubkey);
-
-    this.eventStore = new LabeledEventStore(this.database, communityDefinition.pubkey);
-    this.eventStore.setup();
-    this.relay = new NostrRelay(this.eventStore);
-
-    // handle incoming events and pass them to the upstream relay
-    this.relay.registerEventHandler(async (ctx, next) => {
-      // send event to upstream relay
-      if (this.upstream) {
-        const result = this.upstream.publish(ctx.event);
-        this.log("Sent event to upstream", ctx.event.id);
-        return result;
-      } else throw new Error("Not connected to upstream");
-    });
-
-    this.relay.on("subscription:created", (subscription, ws) => {
-      this.syncChannelsFromFilters(subscription.filters);
-    });
-    this.relay.on("subscription:updated", (subscription, ws) => {
-      this.syncChannelsFromFilters(subscription.filters);
-    });
-  }
-
-  protected async connectUpstream() {
-    if (this.upstream) {
-      if (this.upstream.connected) this.upstream.close();
-      this.upstream = undefined;
-    }
-
-    const hyperAddress = this.definition.tags.find((t) => t[0] === "r" && t[1] && t[2] === "hyper")?.[1];
-    let address = this.definition.tags.find((t) => t[0] === "r" && t[1].startsWith("ws"))?.[1];
-
-    if (hyperAddress) {
-      const serverInfo = await this.connectionManager.getLocalAddress(hyperAddress);
-      address = new URL(`ws://${serverInfo.address}:${serverInfo.port}`).toString();
-    }
-
-    if (!address) throw new Error("Failed to find connection address");
-
-    try {
-      this.log("Connecting to upstream", address);
-      this.upstream = await Relay.connect(address);
-
-      this.upstream.onclose = () => {
-        this.log("Upstream connection closed");
-        this.upstream = undefined;
-      };
-    } catch (error) {
-      this.log("Failed to connect to upstream");
-      if (error instanceof Error) this.log(error);
-    }
-  }
-
-  async connect() {
-    if (this.upstream) return;
-    await this.connectUpstream();
-
-    setTimeout(() => {
-      this.syncMetadata();
-      this.syncDeletions();
-    }, 100);
-  }
-
-  handleEvent(event: NostrEvent) {
-    try {
-      switch (event.kind) {
-        case kinds.EventDeletion:
-          this.handleDeleteEvent(event);
-          break;
-        default:
-          this.eventStore.addEvent(event);
-          break;
-      }
-    } catch (error) {
-      this.log("Failed to handle event");
-      console.log(error);
-    }
-  }
-
-  handleDeleteEvent(deleteEvent: NostrEvent) {
-    const communityPubkey = this.definition.pubkey;
-
-    const ids = RelayActions.handleDeleteEvent(
-      this.eventStore,
-      deleteEvent,
-      deleteEvent.pubkey === communityPubkey ? () => true : undefined,
-    );
-
-    if (ids.length) this.log(`Deleted`, ids.length, "events");
-  }
-
-  syncMetadata() {
-    if (!this.upstream) return;
-
-    this.log("Opening subscription to sync metadata");
-    this.upstream.subscribe([{ kinds: [kinds.Metadata, kinds.RelayList, 12012, 39000, 39001, 39002] }], {
-      id: "metadata-sync",
-      onevent: (event) => this.handleEvent(event),
-      onclose: () => this.log("Closed metadata sync"),
-    });
-  }
-
-  syncDeletions() {
-    if (!this.upstream) return;
-
-    this.log("Opening subscription to sync deletions");
-
-    this.upstream.subscribe([{ kinds: [kinds.EventDeletion] }], {
-      id: "deletion-sync",
-      onevent: (event) => this.handleEvent(event),
-      onclose: () => this.log("Closed deletion sync"),
-    });
-  }
-
-  private syncChannelsFromFilters(filters: Filter[]) {
-    const channels = new Set<string>();
-    for (const filter of filters) {
-      if (filter["#h"]) filter["#h"].forEach((c) => channels.add(c));
-    }
-    for (const channel of channels) {
-      this.syncChannel(channel);
-    }
-  }
-
-  channelSubs = new Map<string, Subscription>();
-  syncChannel(channel: string) {
-    if (!this.upstream) return;
-    if (this.channelSubs.has(channel)) return;
-
-    this.log("Opening subscription to sync channel", channel);
-    const sub = this.upstream.subscribe([{ kinds: [9, 10, 11, 12], "#h": [channel] }], {
-      id: `channel-${channel}-sync`,
-      onevent: (event) => this.eventStore.addEvent(event),
-      onclose: () => {
-        this.channelSubs.delete(channel);
-      },
-    });
-
-    this.channelSubs.set(channel, sub);
-  }
-
-  stop() {
-    this.upstream?.close();
-  }
-}

src/modules/config-manager.ts

@@ -2,9 +2,9 @@ import { JSONFileSync } from "lowdb/node";
 import _throttle from "lodash.throttle";
 import { uniqueNamesGenerator, adjectives, colors, animals } from "unique-names-generator";
 import { PrivateNodeConfig } from "@satellite-earth/core/types/private-node-config.js";
-import { ReactiveJsonFileSync } from "@satellite-earth/core";
 
 import { logger } from "../logger.js";
+import { ReactiveJsonFileSync } from "../classes/json-file.js";
 
 export const defaultConfig: PrivateNodeConfig = {
   name: uniqueNamesGenerator({

src/modules/control/database-actions.ts

@@ -1,5 +1,4 @@
 import { WebSocket } from "ws";
-import os from "node:os";
 import { DatabaseMessage, DatabaseResponse, DatabaseStats } from "@satellite-earth/core/types/control-api/database.js";
 
 import App from "../../app/index.js";

src/modules/decryption-cache/decryption-cache.ts

@@ -1,11 +1,11 @@
-import { mapParams } from "@satellite-earth/core/helpers/sql.js";
+import { MigrationSet } from "../../sqlite/migrations.js";
-import { MigrationSet } from "@satellite-earth/core/sqlite";
 import { type Database } from "better-sqlite3";
 import { EventEmitter } from "events";
 
+import { EventRow, parseEventRow } from "../../sqlite/event-store.js";
 import { logger } from "../../logger.js";
-import { EventRow, parseEventRow } from "@satellite-earth/core/sqlite-event-store";
 import { NostrEvent } from "nostr-tools";
+import { mapParams } from "../../helpers/sql.js";
 
 const migrations = new MigrationSet("decryption-cache");
 

src/modules/direct-message-manager.ts

@@ -1,7 +1,6 @@
 import { filter, lastValueFrom, mergeMap, Subscription, tap, toArray } from "rxjs";
 import { NostrEvent, kinds } from "nostr-tools";
 import { createRxForwardReq } from "rx-nostr";
-import { getRelaysFromContactList } from "@satellite-earth/core/helpers/nostr/contacts.js";
 import { MailboxesQuery } from "applesauce-core/queries";
 import { EventEmitter } from "events";
 
@@ -12,6 +11,7 @@ import { rxNostr } from "../services/rx-nostr.js";
 import { eventStore, queryStore } from "../services/stores.js";
 import { COMMON_CONTACT_RELAYS } from "../env.js";
 import { bufferAudit } from "../helpers/rxjs.js";
+import { getRelaysFromContactList } from "../helpers/nostr/contacts.js";
 
 type EventMap = {
   open: [string, string];

src/modules/gossip.ts

@@ -1,11 +1,12 @@
 import { SimpleSigner } from "applesauce-signers/signers/simple-signer";
 import { EventTemplate, SimplePool } from "nostr-tools";
 import { getTagValue } from "applesauce-core/helpers";
-import { IEventStore, NostrRelay } from "@satellite-earth/core";
 import dayjs, { Dayjs } from "dayjs";
 
 import { logger } from "../logger.js";
 import InboundNetworkManager from "./network/inbound/index.js";
+import { NostrRelay } from "../relay/nostr-relay.js";
+import { SQLiteEventStore } from "../sqlite/event-store.js";
 
 function buildGossipTemplate(self: string, address: string, network: string): EventTemplate {
   return {
@@ -28,7 +29,7 @@ export default class Gossip {
   signer: SimpleSigner;
   pool: SimplePool;
   relay: NostrRelay;
-  eventStore: IEventStore;
+  eventStore: SQLiteEventStore;
 
   running = false;
   // default every 30 minutes
@@ -40,7 +41,7 @@ export default class Gossip {
     signer: SimpleSigner,
     pool: SimplePool,
     relay: NostrRelay,
-    eventStore: IEventStore,
+    eventStore: SQLiteEventStore,
   ) {
     this.network = network;
     this.signer = signer;

src/modules/labeled-event-store.ts

@@ -1,79 +0,0 @@
-import { Database } from "better-sqlite3";
-import { Filter, NostrEvent } from "nostr-tools";
-import { IEventStore, SQLiteEventStore } from "@satellite-earth/core";
-import { logger } from "../logger.js";
-import { MigrationSet } from "@satellite-earth/core/sqlite";
-
-export function mapParams(params: any[]) {
-  return `(${params.map(() => `?`).join(", ")})`;
-}
-
-const migrations = new MigrationSet("labeled-event-store");
-
-// Version 1
-migrations.addScript(1, async (db, log) => {
-  db.prepare(
-    `
-		CREATE TABLE IF NOT EXISTS event_labels (
-			id INTEGER PRIMARY KEY AUTOINCREMENT,
-			event TEXT(64) REFERENCES events(id),
-			label TEXT
-		)
-	`,
-  ).run();
-
-  db.prepare("CREATE INDEX IF NOT EXISTS event_labels_label ON event_labels(label)").run();
-  db.prepare("CREATE INDEX IF NOT EXISTS event_labels_event ON event_labels(event)").run();
-});
-
-/** An event store that is can only see a subset of events int the database */
-export class LabeledEventStore extends SQLiteEventStore implements IEventStore {
-  label: string;
-  readAll = false;
-
-  constructor(db: Database, label: string) {
-    super(db);
-    this.label = label;
-
-    this.log = logger.extend(`event-store:` + label);
-  }
-
-  async setup() {
-    await super.setup();
-    await migrations.run(this.db);
-  }
-
-  override buildConditionsForFilters(filter: Filter) {
-    const parts = super.buildConditionsForFilters(filter);
-
-    if (!this.readAll) {
-      parts.joins.push("INNER JOIN event_labels ON events.id = event_labels.event");
-      parts.conditions.push("event_labels.label = ?");
-      parts.parameters.push(this.label);
-      return parts;
-    }
-
-    return parts;
-  }
-
-  addEvent(event: NostrEvent) {
-    const inserted = super.addEvent(event);
-
-    const hasLabel = !!this.db
-      .prepare("SELECT * FROM event_labels WHERE event = ? AND label = ?")
-      .get(event.id, this.label);
-    if (!hasLabel) this.db.prepare(`INSERT INTO event_labels (event, label) VALUES (?, ?)`).run(event.id, this.label);
-
-    return inserted;
-  }
-
-  removeEvents(ids: string[]) {
-    this.db.prepare(`DELETE FROM event_labels WHERE event IN ${mapParams(ids)}`).run(...ids);
-    return super.removeEvents(ids);
-  }
-
-  removeEvent(id: string) {
-    this.db.prepare(`DELETE FROM event_labels WHERE event = ?`).run(id);
-    return super.removeEvent(id);
-  }
-}

src/modules/log-store/log-store.ts

@@ -1,10 +1,10 @@
 import { type Database as SQLDatabase } from "better-sqlite3";
-import { MigrationSet } from "@satellite-earth/core/sqlite";
 import EventEmitter from "events";
 import { nanoid } from "nanoid";
 import { Debugger } from "debug";
 
 import { logger } from "../../logger.js";
+import { MigrationSet } from "../../sqlite/migrations.js";
 
 type EventMap = {
   log: [LogEntry];
@@ -157,7 +157,7 @@ export default class LogStore extends EventEmitter<EventMap> {
     }
     if (conditions.length > 0) sql += ` WHERE ${conditions.join(" AND ")}`;
 
-    this.database.prepare<any[], DatabaseLogEntry>(sql).run(...parameters);
+    this.database.prepare<any[]>(sql).run(parameters);
     this.emit("clear", filter?.service);
   }
 }

src/modules/notifications/notifications-manager.ts

@@ -1,5 +1,4 @@
 import { NotificationChannel, WebPushNotification } from "@satellite-earth/core/types/control-api/notifications.js";
-import { getDMRecipient, getDMSender, getUserDisplayName } from "@satellite-earth/core/helpers/nostr";
 import { NostrEvent, kinds } from "nostr-tools";
 import { npubEncode } from "nostr-tools/nip19";
 import { getDisplayName, unixNow } from "applesauce-core/helpers";
@@ -10,6 +9,7 @@ import { logger } from "../../logger.js";
 import App from "../../app/index.js";
 import stateManager from "../../services/state.js";
 import config from "../../services/config.js";
+import { getDMRecipient, getDMSender } from "../../helpers/nostr/dms.js";
 
 export type NotificationsManagerState = {
   channels: NotificationChannel[];

src/modules/pubkey-batch-loader.ts

@@ -1,175 +0,0 @@
-import { Filter, NostrEvent, SimplePool } from "nostr-tools";
-import _throttle from "lodash.throttle";
-import { EventEmitter } from "events";
-import { getInboxes, getOutboxes } from "@satellite-earth/core/helpers/nostr/mailboxes.js";
-import SuperMap from "@satellite-earth/core/helpers/super-map.js";
-import { Deferred, createDefer } from "applesauce-core/promise";
-
-import { COMMON_CONTACT_RELAYS } from "../env.js";
-
-type EventMap = {
-  event: [NostrEvent];
-  batch: [number, number];
-};
-
-/** Loads 10002 events for pubkeys */
-export default class PubkeyBatchLoader extends EventEmitter<EventMap> {
-  extraRelays = COMMON_CONTACT_RELAYS;
-
-  kind: number;
-  pool: SimplePool;
-  loadFromCache?: (pubkey: string) => NostrEvent | undefined;
-
-  get queue() {
-    return this.next.size;
-  }
-
-  failed = new SuperMap<string, Set<string>>(() => new Set());
-
-  constructor(kind: number, pool: SimplePool, loadFromCache?: (pubkey: string) => NostrEvent | undefined) {
-    super();
-    this.kind = kind;
-    this.pool = pool;
-    this.loadFromCache = loadFromCache;
-  }
-
-  private cache = new Map<string, NostrEvent>();
-  getEvent(pubkey: string) {
-    if (this.cache.has(pubkey)) return this.cache.get(pubkey)!;
-
-    const event = this.loadFromCache?.(pubkey);
-    if (event) {
-      this.cache.set(pubkey, event);
-      return event;
-    }
-  }
-
-  getOutboxes(pubkey: string) {
-    const mailboxes = this.getEvent(pubkey);
-    return mailboxes && getOutboxes(mailboxes);
-  }
-
-  getInboxes(pubkey: string) {
-    const mailboxes = this.getEvent(pubkey);
-    return mailboxes && getInboxes(mailboxes);
-  }
-
-  handleEvent(event: NostrEvent) {
-    if (event.kind === this.kind) {
-      this.emit("event", event);
-      const current = this.cache.get(event.pubkey);
-      if (!current || event.created_at > current.created_at) this.cache.set(event.pubkey, event);
-    }
-  }
-
-  /** next queue */
-  private next = new Map<string, string[]>();
-  /** currently fetching */
-  private fetching = new Map<string, string[]>();
-  /** promises for next and fetching */
-  private pending = new Map<string, Deferred<NostrEvent | null>>();
-
-  private fetchEventsThrottle = _throttle(this.fetchEvents.bind(this), 1000);
-  private async fetchEvents() {
-    if (this.fetching.size > 0 || this.next.size === 0) return;
-
-    // copy all from next queue to fetching queue
-    for (const [pubkey, relays] of this.next) this.fetching.set(pubkey, relays);
-    this.next.clear();
-
-    if (this.fetching.size > 0) {
-      const filters: Record<string, Filter> = {};
-
-      for (const [pubkey, relays] of this.fetching) {
-        for (const relay of relays) {
-          filters[relay] = filters[relay] || { kinds: [this.kind], authors: [] };
-
-          if (!filters[relay].authors?.includes(pubkey)) {
-            filters[relay].authors?.push(pubkey);
-          }
-        }
-      }
-
-      const requests: Record<string, Filter[]> = {};
-      for (const [relay, filter] of Object.entries(filters)) requests[relay] = [filter];
-
-      await new Promise<void>((res) => {
-        const sub = this.pool.subscribeManyMap(requests, {
-          onevent: (event) => this.handleEvent(event),
-          oneose: () => {
-            sub.close();
-
-            // resolve all pending promises
-            let failed = 0;
-            let found = 0;
-            for (const [pubkey, relays] of this.fetching) {
-              const p = this.pending.get(pubkey);
-              if (p) {
-                const event = this.getEvent(pubkey) ?? null;
-                p.resolve(event);
-                if (!event) {
-                  failed++;
-                  for (const url of relays) this.failed.get(pubkey).add(url);
-                  p.reject();
-                } else found++;
-                this.pending.delete(pubkey);
-              }
-            }
-            this.fetching.clear();
-
-            this.emit("batch", found, failed);
-
-            res();
-          },
-        });
-      });
-
-      // if there are pending requests, make another request
-      if (this.next.size > 0) this.fetchEventsThrottle();
-    }
-  }
-
-  getOrLoadEvent(pubkey: string, relays: string[] = []): Promise<NostrEvent | null> {
-    // if its in the cache, return it
-    const event = this.getEvent(pubkey);
-    if (event) return Promise.resolve(event);
-
-    // if its already being fetched, return promise
-    const pending = this.pending.get(pubkey);
-    if (pending) return pending;
-
-    return this.loadEvent(pubkey, relays);
-  }
-
-  loadEvent(pubkey: string, relays: string[] = [], ignoreFailed = false): Promise<NostrEvent | null> {
-    const urls = new Set(this.next.get(pubkey));
-
-    // add relays
-    for (const url of relays) urls.add(url);
-
-    // add extra relays
-    for (const url of this.extraRelays) urls.add(url);
-
-    // filter out failed relays
-    if (!ignoreFailed) {
-      const failed = this.failed.get(pubkey);
-      for (const url of failed) urls.delete(url);
-    }
-
-    if (urls.size === 0) {
-      // nothing new to try return null
-      return Promise.resolve(null);
-    }
-
-    // create a promise
-    const defer = createDefer<NostrEvent | null>();
-    this.pending.set(pubkey, defer);
-
-    // add pubkey and relay to next queue
-    this.next.set(pubkey, Array.from(urls));
-
-    // trigger queue
-    this.fetchEventsThrottle();
-    return defer;
-  }
-}

src/modules/receiver/index.ts

@@ -1,6 +1,5 @@
 import EventEmitter from "events";
 import { NostrEvent, SimplePool } from "nostr-tools";
-import SuperMap from "@satellite-earth/core/helpers/super-map.js";
 import { Subscription } from "nostr-tools/abstract-relay";
 import { getRelaysFromContactsEvent } from "applesauce-core/helpers";
 
@@ -9,6 +8,7 @@ import { logger } from "../../logger.js";
 import App from "../../app/index.js";
 import { arrayFallback } from "../../helpers/array.js";
 import { requestLoader } from "../../services/loaders.js";
+import SuperMap from "../../helpers/super-map.js";
 
 type EventMap = {
   started: [Receiver];

src/modules/reports/manager.ts

@@ -1,8 +1,8 @@
-import SuperMap from "@satellite-earth/core/helpers/super-map.js";
 import { filter, Observable, shareReplay, Subscription } from "rxjs";
 import hash_sum from "hash-sum";
 
 import { Session } from "../../relay/session.js";
+import SuperMap from "../../helpers/super-map.js";
 
 // open query messages (id, type, args)
 export type QueryOpen<Args extends Record<string, any>> = ["QRY", "OPEN", string, string, Args];

src/modules/reports/reports/conversations.ts

@@ -1,9 +1,9 @@
 import { ReportArguments, ReportResults } from "@satellite-earth/core/types";
+import { getTagValue } from "applesauce-core/helpers";
 import { NostrEvent } from "nostr-tools";
-import { getTagValue } from "@satellite-earth/core/helpers/nostr";
-import SuperMap from "@satellite-earth/core/helpers/super-map.js";
 
 import Report from "../report.js";
+import SuperMap from "../../../helpers/super-map.js";
 
 export default class ConversationsReport extends Report<"CONVERSATIONS"> {
   readonly type = "CONVERSATIONS";

src/modules/reports/reports/events-summary.ts

@@ -1,6 +1,6 @@
 import { ReportArguments } from "@satellite-earth/core/types";
-import { EventRow, parseEventRow } from "@satellite-earth/core";
 import Report from "../report.js";
+import { EventRow, parseEventRow } from "../../../sqlite/event-store.js";
 
 export default class EventsSummaryReport extends Report<"EVENTS_SUMMARY"> {
   readonly type = "EVENTS_SUMMARY";

src/modules/scrapper/index.ts

@@ -1,4 +1,3 @@
-import SuperMap from "@satellite-earth/core/helpers/super-map.js";
 import { EventEmitter } from "events";
 import { NostrEvent } from "nostr-tools";
 import { Deferred, createDefer } from "applesauce-core/promise";
@@ -7,6 +6,7 @@ import App from "../../app/index.js";
 import { logger } from "../../logger.js";
 import PubkeyScrapper from "./pubkey-scrapper.js";
 import { requestLoader } from "../../services/loaders.js";
+import SuperMap from "../../helpers/super-map.js";
 
 const MAX_TASKS = 10;
 

src/modules/state/application-state-manager.ts

@@ -1,7 +1,7 @@
-import { MigrationSet } from "@satellite-earth/core/sqlite";
 import { Database } from "better-sqlite3";
 
 import { MutableState } from "./mutable-state.js";
+import { MigrationSet } from "../../sqlite/migrations.js";
 
 const migrations = new MigrationSet("application-state");
 

src/relay/actions/handle-delete-event.ts

@@ -0,0 +1,53 @@
+import { Filter, NostrEvent, kinds } from "nostr-tools";
+import { SQLiteEventStore } from "../../sqlite/event-store.js";
+
+/**
+ * handles kind 5 delete events
+ * @param doseMatch if this returns true the event will be deleted
+ */
+export function handleDeleteEvent(
+  eventStore: SQLiteEventStore,
+  deleteEvent: NostrEvent,
+  doseMatch?: (event: NostrEvent) => boolean,
+) {
+  if (deleteEvent.kind !== kinds.EventDeletion) return [];
+
+  const events = new Map<string, NostrEvent>();
+
+  const ids = deleteEvent.tags.filter((t) => t[0] === "e" && t[1]).map((t) => t[1]);
+  if (ids.length) {
+    const eventsFromIds = eventStore.getEventsForFilters([{ ids, until: deleteEvent.created_at }]);
+    for (const event of eventsFromIds) events.set(event.id, event);
+  }
+
+  const cords = deleteEvent.tags
+    .filter((t) => t[0] === "a" && t[1])
+    .map((t) => t[1].split(":"))
+    .filter((cord) => cord.length === 3);
+
+  if (cords.length) {
+    const eventsFromCords = eventStore.getEventsForFilters(
+      cords.map(([kind, pubkey, d]) => {
+        return {
+          "#d": [d],
+          kinds: [parseInt(kind)],
+          authors: [pubkey],
+          until: deleteEvent.created_at,
+        } satisfies Filter;
+      }),
+    );
+    for (const event of eventsFromCords) events.set(event.id, event);
+  }
+
+  const deleteIds: string[] = [];
+  for (const [id, event] of events) {
+    // delete the target event if the delete event was signed by the community or original author
+    if (doseMatch?.(event) || event.pubkey === deleteEvent.pubkey) {
+      deleteIds.push(event.id);
+    }
+  }
+
+  eventStore.addEvent(deleteEvent);
+  eventStore.removeEvents(deleteIds);
+  return deleteIds;
+}

src/relay/actions/index.ts

@@ -0,0 +1,5 @@
+import { handleDeleteEvent } from './handle-delete-event.js';
+
+export const RelayActions = {
+	handleDeleteEvent,
+};

src/relay/nostr-relay.ts

@@ -0,0 +1,424 @@
+import EventEmitter from "events";
+import crypto, { randomUUID } from "crypto";
+import { IncomingMessage } from "http";
+import { type WebSocket, RawData } from "ws";
+import { Filter, NostrEvent, verifyEvent, matchFilters } from "nostr-tools";
+
+import { logger } from "../logger.js";
+import { SQLiteEventStore } from "../sqlite/event-store.js";
+
+export type IncomingReqMessage = ["REQ", string, ...Filter[]];
+export type IncomingCountMessage = ["COUNT", string, ...Filter[]];
+export type IncomingEventMessage = ["EVENT", NostrEvent];
+export type IncomingAuthMessage = ["AUTH", NostrEvent];
+export type IncomingCloseMessage = ["CLOSE", string];
+
+export type Subscription = {
+  type: "REQ" | "COUNT";
+  ws: WebSocket;
+  id: string;
+  filters: Filter[];
+};
+
+export type HandlerNext = () => Promise<void>;
+
+export type HandlerContext = { event: NostrEvent; socket: WebSocket; relay: NostrRelay };
+export type EventHandler = (
+  ctx: HandlerContext,
+  next: HandlerNext,
+) => boolean | undefined | string | void | Promise<string | boolean | undefined | void>;
+
+export type SubscriptionFilterContext = { id: string; filters: Filter[]; socket: WebSocket; relay: NostrRelay };
+export type SubscriptionFilter = (
+  ctx: SubscriptionFilterContext,
+  next: HandlerNext,
+) => boolean | undefined | void | Promise<boolean | undefined | void>;
+
+type EventMap = {
+  "event:received": [NostrEvent, WebSocket];
+  "event:inserted": [NostrEvent, WebSocket];
+  "event:rejected": [NostrEvent, WebSocket];
+  "subscription:created": [Subscription, WebSocket];
+  "subscription:updated": [Subscription, WebSocket];
+  "subscription:closed": [Subscription, WebSocket];
+  "socket:connect": [WebSocket];
+  "socket:disconnect": [WebSocket];
+  "socket:auth": [WebSocket, NostrEvent];
+};
+
+export class NostrRelay extends EventEmitter<EventMap> {
+  static SUPPORTED_NIPS = [1, 4, 11, 45, 50, 70, 119];
+
+  log = logger.extend("relay");
+  eventStore: SQLiteEventStore;
+
+  connectionId = new WeakMap<WebSocket, string>();
+
+  // A map of subscriptions
+  subscriptions: Subscription[] = [];
+
+  // Create a map of connections
+  // in the form <connid> : <ws>
+  connections: Record<string, WebSocket> = {};
+
+  publicURL?: string;
+  requireRelayInAuth = true;
+  sendChallenge = false;
+  auth = new Map<WebSocket, { challenge: string; response?: NostrEvent }>();
+  checkAuth?: (ws: WebSocket, auth: NostrEvent) => boolean | string;
+
+  checkReadEvent?: (ws: WebSocket, event: NostrEvent, auth?: NostrEvent) => boolean;
+
+  constructor(eventStore: SQLiteEventStore) {
+    super();
+
+    this.eventStore = eventStore;
+
+    // listen for new events inserted into the store
+    this.eventStore.on("event:inserted", (event) => {
+      // make sure it wasn't the last event we inserted
+      if (event.id !== this.lastInserted) this.sendEventToSubscriptions(event);
+    });
+  }
+
+  async handleMessage(message: Buffer | string, ws: WebSocket) {
+    let data;
+
+    try {
+      // TODO enforce max size
+
+      // Parse JSON from the raw buffer
+      data = JSON.parse(typeof message === "string" ? message : message.toString("utf-8"));
+
+      if (!Array.isArray(data)) throw new Error("Message is not an array");
+
+      // Pass the data to appropriate handler
+      switch (data[0]) {
+        case "REQ":
+        case "COUNT":
+          await this.handleSubscriptionMessage(data as IncomingReqMessage | IncomingCountMessage, ws);
+          break;
+        case "EVENT":
+          await this.handleEventMessage(data as IncomingEventMessage, ws);
+          break;
+        case "AUTH":
+          await this.handleAuthMessage(data as IncomingAuthMessage, ws);
+          break;
+        case "CLOSE":
+          await this.handleCloseMessage(data as IncomingCloseMessage, ws);
+          break;
+      }
+    } catch (err) {
+      this.log("Failed to handle message", message.toString("utf-8"), err);
+    }
+
+    return data;
+  }
+
+  private socketCleanup = new Map<WebSocket, () => void>();
+  handleConnection(ws: WebSocket, req: IncomingMessage) {
+    let ip;
+
+    // Record the IP address of the client
+    if (typeof req.headers["x-forwarded-for"] === "string") {
+      ip = req.headers["x-forwarded-for"].split(",")[0].trim();
+    } else {
+      ip = req.socket.remoteAddress;
+    }
+
+    // listen for messages
+    const messageListener = (data: RawData, isBinary: boolean) => {
+      if (data instanceof Buffer) this.handleMessage(data, ws);
+    };
+    ws.on("message", messageListener);
+
+    const closeListener = () => this.handleDisconnect(ws);
+    ws.on("close", closeListener);
+
+    if (this.sendChallenge) {
+      const challenge = randomUUID();
+      this.auth.set(ws, { challenge });
+      ws.send(JSON.stringify(["AUTH", challenge]));
+    }
+
+    this.emit("socket:connect", ws);
+
+    // Generate a unique ID for ws connection
+    const id = crypto.randomUUID();
+
+    this.connectionId.set(ws, id);
+    this.connections[id] = ws;
+
+    this.socketCleanup.set(ws, () => {
+      delete this.connections[id];
+      ws.off("message", messageListener);
+      ws.off("close", closeListener);
+      this.connectionId.delete(ws);
+      this.auth.delete(ws);
+      this.emit("socket:disconnect", ws);
+    });
+  }
+
+  disconnectSocket(ws: WebSocket) {
+    this.socketCleanup.get(ws)?.();
+  }
+
+  handleDisconnect(ws: WebSocket) {
+    const id = this.connectionId.get(ws);
+    if (!id) return;
+
+    const openSubscriptions = this.subscriptions.filter((sub) => sub.ws === ws);
+
+    // remove all subscriptions
+    this.subscriptions = this.subscriptions.filter((sub) => sub.ws !== ws);
+
+    for (const sub of openSubscriptions) {
+      this.emit("subscription:closed", sub, ws);
+    }
+
+    this.connectionId.delete(ws);
+    delete this.connections[id];
+
+    this.emit("socket:disconnect", ws);
+  }
+
+  sendEventToSubscriptions(event: NostrEvent) {
+    for (const sub of this.subscriptions) {
+      if (sub.type === "REQ" && !sub.filters.some((f) => f.search) && matchFilters(sub.filters, event)) {
+        sub.ws.send(JSON.stringify(["EVENT", sub.id, event]));
+      }
+    }
+  }
+
+  /** Used to avoid infinite loop */
+  private lastInserted: string = "";
+
+  eventHandlers: EventHandler[] = [];
+  private async callEventHandler(ctx: HandlerContext, index = 0): Promise<string | boolean | undefined | void> {
+    const handler = this.eventHandlers[index];
+    if (!handler) return;
+
+    return await handler(ctx, async () => {
+      await this.callEventHandler(ctx, index + 1);
+    });
+  }
+
+  registerEventHandler(handler: EventHandler) {
+    this.eventHandlers.push(handler);
+
+    return () => this.unregisterEventHandler(handler);
+  }
+  unregisterEventHandler(handler: EventHandler) {
+    const idx = this.eventHandlers.indexOf(handler);
+    if (idx !== -1) this.eventHandlers.splice(idx, 1);
+  }
+
+  async handleEventMessage(data: IncomingEventMessage, ws: WebSocket) {
+    // Get the event data
+    const event = data[1] as NostrEvent;
+
+    try {
+      let inserted = false;
+
+      // Verify the event's signature
+      if (!verifyEvent(event)) throw new Error(`invalid: event failed to validate or verify`);
+
+      // NIP-70 protected events
+      const isProtected = event.tags.some((t) => t[0] === "-");
+      if (isProtected && this.auth.get(ws)?.response?.pubkey !== event.pubkey) {
+        throw new Error("auth-required: this event may only be published by its author");
+      }
+
+      const context: HandlerContext = { event, socket: ws, relay: this };
+      let persist = (await this.callEventHandler(context)) ?? true;
+
+      if (persist) {
+        try {
+          // Persist to database
+          this.lastInserted = event.id;
+          inserted = this.eventStore.addEvent(event);
+        } catch (err) {
+          console.log(err);
+          throw new Error(`error: server error`);
+        }
+
+        this.emit("event:received", event, ws);
+        if (inserted) {
+          this.emit("event:inserted", event, ws);
+          this.sendPublishOkMessage(ws, event, true, typeof persist === "string" ? persist : "");
+
+          this.sendEventToSubscriptions(event);
+        } else {
+          this.sendPublishOkMessage(ws, event, true, typeof persist === "string" ? persist : "Duplicate");
+        }
+      } else {
+        // reject with generic message
+        throw new Error("Rejected");
+      }
+    } catch (err) {
+      if (err instanceof Error) {
+        // error occurred, send back the OK message with false
+        this.emit("event:rejected", event, ws);
+        this.sendPublishOkMessage(ws, event, false, err.message);
+      }
+    }
+  }
+
+  // response helpers
+  makeAuthRequiredReason(reason: string) {
+    return "auth-required: " + reason;
+  }
+  sendPublishOkMessage(ws: WebSocket, event: NostrEvent, success: boolean, message?: string) {
+    ws.send(JSON.stringify(message ? ["OK", event.id, success, message] : ["OK", event.id, success]));
+  }
+  sendPublishAuthRequired(ws: WebSocket, event: NostrEvent, message: string) {
+    ws.send(JSON.stringify(["OK", event.id, false, this.makeAuthRequiredReason(message)]));
+  }
+
+  handleAuthMessage(data: IncomingAuthMessage, ws: WebSocket) {
+    try {
+      const event = data[1];
+      if (!verifyEvent(event)) {
+        return this.sendPublishOkMessage(ws, event, false, "Invalid event");
+      }
+
+      const relay = event.tags.find((t) => t[0] === "relay")?.[1];
+      if (this.requireRelayInAuth) {
+        if (!relay) {
+          return this.sendPublishOkMessage(ws, event, false, "Missing relay tag");
+        }
+        if (new URL(relay).toString() !== this.publicURL) {
+          return this.sendPublishOkMessage(ws, event, false, "Bad relay tag");
+        }
+      }
+
+      // check challenge
+      const challenge = this.auth.get(ws)?.challenge;
+      const challengeResponse = event.tags.find((t) => t[0] === "challenge")?.[1];
+
+      if (!challengeResponse || !challenge) {
+        return this.sendPublishOkMessage(ws, event, false, "Missing challenge tag");
+      }
+      if (challengeResponse !== challenge) {
+        return this.sendPublishOkMessage(ws, event, false, "Bad challenge");
+      }
+
+      if (this.checkAuth) {
+        const message = this.checkAuth(ws, event);
+        if (typeof message === "string") return this.sendPublishOkMessage(ws, event, false, message);
+        else if (message === false) return this.sendPublishOkMessage(ws, event, false, "Rejected auth");
+      }
+
+      this.auth.set(ws, { challenge, response: event });
+      this.emit("socket:auth", ws, event);
+      this.log("Authenticated", event.pubkey);
+      this.sendPublishOkMessage(ws, event, true, "Authenticated");
+    } catch (e) {}
+  }
+
+  protected runSubscription(sub: Subscription) {
+    const auth = this.getSocketAuth(sub.ws);
+
+    switch (sub.type) {
+      case "REQ":
+        const events = this.eventStore.getEventsForFilters(sub.filters);
+        for (let event of events) {
+          if (!this.checkReadEvent || this.checkReadEvent(sub.ws, event, auth)) {
+            sub.ws.send(JSON.stringify(["EVENT", sub.id, event]));
+          }
+        }
+        sub.ws.send(JSON.stringify(["EOSE", sub.id]));
+        break;
+      case "COUNT":
+        const count = this.eventStore.countEventsForFilters(sub.filters);
+        sub.ws.send(JSON.stringify(["COUNT", sub.id, { count }]));
+        break;
+    }
+  }
+
+  subscriptionFilters: SubscriptionFilter[] = [];
+  private async checkSubscriptionFilters(
+    ctx: SubscriptionFilterContext,
+    index = 0,
+  ): Promise<boolean | undefined | void> {
+    const handler = this.subscriptionFilters[index];
+    if (!handler) return;
+
+    return await handler(ctx, async () => {
+      await this.checkSubscriptionFilters(ctx, index + 1);
+    });
+  }
+  registerSubscriptionFilter(filter: SubscriptionFilter) {
+    this.subscriptionFilters.push(filter);
+
+    return () => this.unregisterSubscriptionFilter(filter);
+  }
+  unregisterSubscriptionFilter(filter: SubscriptionFilter) {
+    const idx = this.subscriptionFilters.indexOf(filter);
+    if (idx !== -1) this.subscriptionFilters.splice(idx, 1);
+  }
+
+  async handleSubscriptionMessage(data: IncomingReqMessage | IncomingCountMessage, ws: WebSocket) {
+    const [type, id, ...filters] = data;
+    if (typeof id !== "string" || filters.length === 0) return;
+
+    try {
+      const allow = (await this.checkSubscriptionFilters({ socket: ws, filters, id, relay: this })) ?? true;
+
+      if (allow === false) {
+        return this.closeSubscription(id, ws, "Rejected");
+      }
+
+      let subscription = this.subscriptions.find((s) => s.id === id) || { type, id: id, ws, filters: [] };
+
+      // override or set the filters
+      subscription.filters = filters;
+
+      // only save the subscription if its not a count
+      if (type !== "COUNT") {
+        if (!this.subscriptions.includes(subscription)) {
+          this.subscriptions.push(subscription);
+          this.emit("subscription:created", subscription, ws);
+        } else {
+          this.emit("subscription:updated", subscription, ws);
+        }
+      }
+
+      // Run the subscription
+      await this.runSubscription(subscription);
+    } catch (error) {
+      if (typeof error === "string") {
+        this.closeSubscription(id, ws, error);
+      } else if (error instanceof Error) {
+        this.closeSubscription(id, ws, error.message);
+      }
+    }
+  }
+
+  closeSubscription(id: string, ws?: WebSocket, reason?: string) {
+    const subscription = this.subscriptions.find((s) => s.id === id && (ws ? s.ws === ws : true));
+    if (subscription) {
+      this.subscriptions.splice(this.subscriptions.indexOf(subscription), 1);
+      this.emit("subscription:closed", subscription, subscription.ws);
+    }
+
+    if (reason) (subscription?.ws || ws)?.send(JSON.stringify(["CLOSED", id, reason]));
+  }
+  handleCloseMessage(data: IncomingCloseMessage, ws: WebSocket) {
+    if (typeof data[1] !== "string") return;
+    const id = data[1];
+
+    this.closeSubscription(id, ws);
+  }
+
+  getSocketAuth(ws: WebSocket) {
+    return this.auth.get(ws)?.response;
+  }
+
+  stop() {
+    for (const ws of Object.values(this.connections)) {
+      ws.close();
+    }
+    this.removeAllListeners();
+  }
+}

src/services/event-cache.ts

@@ -1,7 +1,7 @@
-import { SQLiteEventStore } from "@satellite-earth/core";
+import { SQLiteEventStore } from "../sqlite/event-store.js";
 import database from "./database.js";
 
-const eventCache = new SQLiteEventStore(database.db);
+const sqliteEventStore = new SQLiteEventStore(database.db);
-await eventCache.setup();
+await sqliteEventStore.setup();
 
-export default eventCache;
+export default sqliteEventStore;

src/services/loaders.ts

@@ -5,11 +5,11 @@ import { ReplaceableLoader, RequestLoader } from "applesauce-loaders/loaders";
 
 import { COMMON_CONTACT_RELAYS } from "../env.js";
 import { rxNostr } from "./rx-nostr.js";
-import eventCache from "./event-cache.js";
+import sqliteEventStore from "./event-cache.js";
 import { eventStore, queryStore } from "./stores.js";
 
 function cacheRequest(filters: Filter[]) {
-  const events = eventCache.getEventsForFilters(filters);
+  const events = sqliteEventStore.getEventsForFilters(filters);
   return from(events).pipe(tap(markFromCache));
 }
 
@@ -20,7 +20,7 @@ replaceableLoader.subscribe((packet) => {
   const event = eventStore.add(packet.event, packet.from);
 
   // save it to the cache if its new
-  if (!isFromCache(event)) eventCache.addEvent(event);
+  if (!isFromCache(event)) sqliteEventStore.addEvent(event);
 });
 
 export const requestLoader = new RequestLoader(queryStore);

src/sqlite/event-store.ts

@@ -0,0 +1,517 @@
+import { Database } from "better-sqlite3";
+import { Filter, NostrEvent, kinds } from "nostr-tools";
+import EventEmitter from "events";
+
+import { mapParams } from "../helpers/sql.js";
+import { logger } from "../logger.js";
+import { MigrationSet } from "../sqlite/migrations.js";
+
+const isFilterKeyIndexableTag = (key: string) => {
+  return key[0] === "#" && key.length === 2;
+};
+const isFilterKeyIndexableAndTag = (key: string) => {
+  return key[0] === "&" && key.length === 2;
+};
+
+export type EventRow = {
+  id: string;
+  kind: number;
+  pubkey: string;
+  content: string;
+  tags: string;
+  created_at: number;
+  sig: string;
+  d?: string;
+};
+
+export function parseEventRow(row: EventRow): NostrEvent {
+  return { ...row, tags: JSON.parse(row.tags) };
+}
+
+// search behavior
+const SEARCHABLE_TAGS = ["title", "description", "about", "summary", "alt"];
+const SEARCHABLE_KIND_BLACKLIST = [kinds.EncryptedDirectMessage];
+const SEARCHABLE_CONTENT_FORMATTERS: Record<number, (content: string) => string> = {
+  [kinds.Metadata]: (content) => {
+    const SEARCHABLE_PROFILE_FIELDS = [
+      "name",
+      "display_name",
+      "about",
+      "nip05",
+      "lud16",
+      "website",
+      // Deprecated fields
+      "displayName",
+      "username",
+    ];
+    try {
+      const lines: string[] = [];
+      const json = JSON.parse(content);
+
+      for (const field of SEARCHABLE_PROFILE_FIELDS) {
+        if (json[field]) lines.push(json[field]);
+      }
+
+      return lines.join("\n");
+    } catch (error) {
+      return content;
+    }
+  },
+};
+
+function convertEventToSearchRow(event: NostrEvent) {
+  const tags = event.tags
+    .filter((t) => SEARCHABLE_TAGS.includes(t[0]))
+    .map((t) => t[1])
+    .join(" ");
+
+  const content = SEARCHABLE_CONTENT_FORMATTERS[event.kind]
+    ? SEARCHABLE_CONTENT_FORMATTERS[event.kind](event.content)
+    : event.content;
+
+  return { id: event.id, content, tags };
+}
+
+const migrations = new MigrationSet("event-store");
+
+// Version 1
+migrations.addScript(1, async (db, log) => {
+  // Create events table
+  db.prepare(
+    `
+		CREATE TABLE IF NOT EXISTS events (
+			id TEXT(64) PRIMARY KEY,
+			created_at INTEGER,
+			pubkey TEXT(64),
+			sig TEXT(128),
+			kind INTEGER,
+			content TEXT,
+			tags TEXT
+		)
+		`,
+  ).run();
+
+  log("Setup events");
+
+  // Create tags table
+  db.prepare(
+    `
+		CREATE TABLE IF NOT EXISTS tags (
+			i INTEGER PRIMARY KEY AUTOINCREMENT,
+			e TEXT(64) REFERENCES events(id),
+			t TEXT(1),
+			v TEXT
+		)
+		`,
+  ).run();
+
+  log("Setup tags table");
+
+  // Create indices
+  const indices = [
+    db.prepare("CREATE INDEX IF NOT EXISTS events_created_at ON events(created_at)"),
+    db.prepare("CREATE INDEX IF NOT EXISTS events_pubkey ON events(pubkey)"),
+    db.prepare("CREATE INDEX IF NOT EXISTS events_kind ON events(kind)"),
+    db.prepare("CREATE INDEX IF NOT EXISTS tags_e ON tags(e)"),
+    db.prepare("CREATE INDEX IF NOT EXISTS tags_t ON tags(t)"),
+    db.prepare("CREATE INDEX IF NOT EXISTS tags_v ON tags(v)"),
+  ];
+
+  indices.forEach((statement) => statement.run());
+
+  log(`Setup ${indices.length} indices`);
+});
+
+// Version 2, search table
+migrations.addScript(2, async (db, log) => {
+  db.prepare(
+    `CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(id UNINDEXED, content, tags, tokenize='trigram')`,
+  ).run();
+  log("Created event search table");
+
+  const rows = db
+    .prepare<number[], EventRow>(`SELECT * FROM events WHERE kind NOT IN ${mapParams(SEARCHABLE_KIND_BLACKLIST)}`)
+    .all(...SEARCHABLE_KIND_BLACKLIST);
+
+  // insert search content into table
+  let changes = 0;
+  for (const row of rows) {
+    const search = convertEventToSearchRow(parseEventRow(row));
+
+    const result = db
+      .prepare<[string, string, string]>(`INSERT OR REPLACE INTO events_fts (id, content, tags) VALUES (?, ?, ?)`)
+      .run(search.id, search.content, search.tags);
+
+    changes += result.changes;
+  }
+  log(`Inserted ${changes} events into search table`);
+});
+
+// Version 3, indexed d tags
+migrations.addScript(3, async (db, log) => {
+  db.prepare(`ALTER TABLE events ADD COLUMN d TEXT`).run();
+  log("Created d column");
+
+  db.prepare("CREATE INDEX IF NOT EXISTS events_d ON events(d)").run();
+  log(`Created d index`);
+
+  log(`Adding d tags to events table`);
+  let updated = 0;
+  db.transaction(() => {
+    const events = db
+      .prepare<[], { id: string; d: string }>(
+        `
+				SELECT events.id as id, tags.v as d
+				FROM events
+				INNER JOIN tags ON tags.e = events.id AND tags.t = 'd'
+				WHERE events.kind >= 30000 AND events.kind < 40000
+			`,
+      )
+      .all();
+    const update = db.prepare<[string, string]>("UPDATE events SET d = ? WHERE id = ?");
+
+    for (const row of events) {
+      const { changes } = update.run(row.d, row.id);
+      if (changes > 0) updated++;
+    }
+  })();
+
+  log(`Updated ${updated} events`);
+});
+
+type EventMap = {
+  "event:inserted": [NostrEvent];
+  "event:removed": [string];
+};
+
+export class SQLiteEventStore extends EventEmitter<EventMap> {
+  db: Database;
+  log = logger.extend("sqlite-event-store");
+
+  preserveEphemeral = false;
+  preserveReplaceable = false;
+
+  constructor(db: Database) {
+    super();
+    this.db = db;
+  }
+
+  setup() {
+    return migrations.run(this.db);
+  }
+
+  addEvent(event: NostrEvent) {
+    // Don't store ephemeral events in db,
+    // just return the event directly
+    if (!this.preserveEphemeral && kinds.isEphemeralKind(event.kind)) return false;
+
+    const inserted = this.db.transaction(() => {
+      // TODO: Check if event is replaceable and if its newer
+      // before inserting it into the database
+
+      // get event d value so it can be indexed
+      const d = kinds.isParameterizedReplaceableKind(event.kind)
+        ? event.tags.find((t) => t[0] === "d" && t[1])?.[1]
+        : undefined;
+
+      const insert = this.db
+        .prepare(
+          `
+					INSERT OR IGNORE INTO events (id, created_at, pubkey, sig, kind, content, tags, d)
+					VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+				`,
+        )
+        .run(
+          event.id,
+          event.created_at,
+          event.pubkey,
+          event.sig,
+          event.kind,
+          event.content,
+          JSON.stringify(event.tags),
+          d,
+        );
+
+      // If event inserted, index tags, insert search
+      if (insert.changes) {
+        this.insertEventTags(event);
+
+        // Remove older replaceable events and all their associated tags
+        if (this.preserveReplaceable === false) {
+          let older: { id: string; created_at: number }[] = [];
+
+          if (kinds.isReplaceableKind(event.kind)) {
+            // Normal replaceable event
+            older = this.db
+              .prepare<[number, string], { id: string; created_at: number }>(
+                `
+								SELECT id, created_at FROM events WHERE kind = ? AND pubkey = ?
+								`,
+              )
+              .all(event.kind, event.pubkey);
+          } else if (kinds.isParameterizedReplaceableKind(event.kind)) {
+            // Parameterized Replaceable
+            const d = event.tags.find((t) => t[0] === "d")?.[1];
+
+            if (d) {
+              older = this.db
+                .prepare<[number, string, "d", string], { id: string; created_at: number }>(
+                  `
+									SELECT events.id, events.created_at FROM events
+									INNER JOIN tags ON events.id = tags.e
+									WHERE kind = ? AND pubkey = ? AND tags.t = ? AND tags.v = ?
+								`,
+                )
+                .all(event.kind, event.pubkey, "d", d);
+            }
+          }
+
+          // If found other events that may need to be replaced,
+          // sort the events according to timestamp descending,
+          // falling back to id lexical order ascending as per
+          // NIP-01. Remove all non-most-recent events and tags.
+          if (older.length > 1) {
+            const removeIds = older
+              .sort((a, b) => {
+                return a.created_at === b.created_at ? a.id.localeCompare(b.id) : b.created_at - a.created_at;
+              })
+              .slice(1)
+              .map((item) => item.id);
+
+            if (!removeIds.includes(event.id)) this.log("Removed", removeIds.length, "old replaceable events");
+
+            this.removeEvents(removeIds);
+
+            // If the event that was just inserted was one of
+            // the events that was removed, return null so to
+            // indicate that the event was in effect *not*
+            // upserted and thus, if using the DB for a nostr
+            // relay, does not need to be pushed to clients
+            if (removeIds.indexOf(event.id) !== -1) return false;
+          }
+        }
+      }
+
+      return insert.changes > 0;
+    })();
+
+    if (inserted) {
+      this.insertEventIntoSearch(event);
+      this.emit("event:inserted", event);
+    }
+
+    return inserted;
+  }
+
+  private insertEventTags(event: NostrEvent) {
+    for (let tag of event.tags) {
+      if (tag[0].length === 1) {
+        this.db.prepare(`INSERT INTO tags (e, t, v) VALUES (?, ?, ?)`).run(event.id, tag[0], tag[1]);
+      }
+    }
+  }
+
+  private insertEventIntoSearch(event: NostrEvent) {
+    const search = convertEventToSearchRow(event);
+
+    return this.db
+      .prepare<[string, string, string]>(`INSERT OR REPLACE INTO events_fts (id, content, tags) VALUES (?, ?, ?)`)
+      .run(search.id, search.content, search.tags);
+  }
+
+  removeEvents(ids: string[]) {
+    const results = this.db.transaction(() => {
+      this.db.prepare(`DELETE FROM tags WHERE e IN ${mapParams(ids)}`).run(...ids);
+      this.db.prepare(`DELETE FROM events_fts WHERE id IN ${mapParams(ids)}`).run(...ids);
+
+      return this.db.prepare(`DELETE FROM events WHERE events.id IN ${mapParams(ids)}`).run(...ids);
+    })();
+
+    if (results.changes > 0) {
+      for (const id of ids) {
+        this.emit("event:removed", id);
+      }
+    }
+  }
+
+  removeEvent(id: string) {
+    const results = this.db.transaction(() => {
+      this.db.prepare(`DELETE FROM tags WHERE e = ?`).run(id);
+      this.db.prepare(`DELETE FROM events_fts WHERE id = ?`).run(id);
+
+      return this.db.prepare(`DELETE FROM events WHERE events.id = ?`).run(id);
+    })();
+
+    if (results.changes > 0) this.emit("event:removed", id);
+
+    return results.changes > 0;
+  }
+
+  buildConditionsForFilters(filter: Filter) {
+    const joins: string[] = [];
+    const conditions: string[] = [];
+    const parameters: (string | number)[] = [];
+    const groupBy: string[] = [];
+    const having: string[] = [];
+
+    // get AND tag filters
+    const andTagQueries = Object.keys(filter).filter(isFilterKeyIndexableAndTag);
+    // get OR tag filters and remove any ones that appear in the AND
+    const orTagQueries = Object.keys(filter)
+      .filter(isFilterKeyIndexableTag)
+      .filter((t) => !andTagQueries.includes(t));
+
+    if (orTagQueries.length > 0) {
+      joins.push("INNER JOIN tags as or_tags ON events.id = or_tags.e");
+    }
+    if (andTagQueries.length > 0) {
+      joins.push("INNER JOIN tags as and_tags ON events.id = and_tags.e");
+    }
+    if (filter.search) {
+      joins.push("INNER JOIN events_fts ON events_fts.id = events.id");
+
+      conditions.push(`events_fts MATCH ?`);
+      parameters.push('"' + filter.search.replace(/"/g, '""') + '"');
+    }
+
+    if (typeof filter.since === "number") {
+      conditions.push(`events.created_at >= ?`);
+      parameters.push(filter.since);
+    }
+
+    if (typeof filter.until === "number") {
+      conditions.push(`events.created_at < ?`);
+      parameters.push(filter.until);
+    }
+
+    if (filter.ids) {
+      conditions.push(`events.id IN ${mapParams(filter.ids)}`);
+      parameters.push(...filter.ids);
+    }
+
+    if (filter.kinds) {
+      conditions.push(`events.kind IN ${mapParams(filter.kinds)}`);
+      parameters.push(...filter.kinds);
+    }
+
+    if (filter.authors) {
+      conditions.push(`events.pubkey IN ${mapParams(filter.authors)}`);
+      parameters.push(...filter.authors);
+    }
+
+    // add AND tag filters
+    for (const t of andTagQueries) {
+      conditions.push(`and_tags.t = ?`);
+      parameters.push(t.slice(1));
+
+      // @ts-expect-error
+      const v = filter[t] as string[];
+      conditions.push(`and_tags.v IN ${mapParams(v)}`);
+      parameters.push(...v);
+    }
+
+    // add OR tag filters
+    for (let t of orTagQueries) {
+      conditions.push(`or_tags.t = ?`);
+      parameters.push(t.slice(1));
+
+      // @ts-expect-error
+      const v = filter[t] as string[];
+      conditions.push(`or_tags.v IN ${mapParams(v)}`);
+      parameters.push(...v);
+    }
+
+    // if there is an AND tag filter set GROUP BY so that HAVING can be used
+    if (andTagQueries.length > 0) {
+      groupBy.push("events.id");
+      having.push("COUNT(and_tags.i) = ?");
+
+      // @ts-expect-error
+      parameters.push(andTagQueries.reduce((t, k) => t + (filter[k] as string[]).length, 0));
+    }
+
+    return { conditions, parameters, joins, groupBy, having };
+  }
+
+  protected buildSQLQueryForFilters(filters: Filter[], select = "events.*") {
+    let sql = `SELECT ${select} FROM events `;
+
+    const orConditions: string[] = [];
+    const parameters: any[] = [];
+    const groupBy = new Set<string>();
+    const having = new Set<string>();
+
+    let joins = new Set<string>();
+    for (const filter of filters) {
+      const parts = this.buildConditionsForFilters(filter);
+
+      if (parts.conditions.length > 0) {
+        orConditions.push(`(${parts.conditions.join(" AND ")})`);
+        parameters.push(...parts.parameters);
+
+        for (const join of parts.joins) joins.add(join);
+        for (const group of parts.groupBy) groupBy.add(group);
+        for (const have of parts.having) having.add(have);
+      }
+    }
+
+    sql += Array.from(joins).join(" ");
+
+    if (orConditions.length > 0) {
+      sql += ` WHERE ${orConditions.join(" OR ")}`;
+    }
+
+    if (groupBy.size > 0) {
+      sql += " GROUP BY " + Array.from(groupBy).join(",");
+    }
+    if (having.size > 0) {
+      sql += " HAVING " + Array.from(having).join(" AND ");
+    }
+
+    // @ts-expect-error
+    const order = filters.find((f) => f.order)?.order;
+    if (filters.some((f) => f.search) && (order === "rank" || order === undefined)) {
+      sql = sql + " ORDER BY rank";
+    } else {
+      sql = sql + " ORDER BY created_at DESC";
+    }
+
+    let minLimit = Infinity;
+    for (const filter of filters) {
+      if (filter.limit) minLimit = Math.min(minLimit, filter.limit);
+    }
+    if (minLimit !== Infinity) {
+      sql += " LIMIT ?";
+      parameters.push(minLimit);
+    }
+
+    return { sql, parameters };
+  }
+
+  getEventsForFilters(filters: Filter[]) {
+    const { sql, parameters } = this.buildSQLQueryForFilters(filters);
+
+    return this.db.prepare<any[], EventRow>(sql).all(parameters).map(parseEventRow);
+  }
+
+  *iterateEventsForFilters(filters: Filter[]): IterableIterator<NostrEvent> {
+    const { sql, parameters } = this.buildSQLQueryForFilters(filters);
+    const iterator = this.db.prepare<any[], EventRow>(sql).iterate(parameters);
+
+    while (true) {
+      const { value: row, done } = iterator.next();
+      if (done) break;
+
+      yield parseEventRow(row);
+    }
+  }
+
+  countEventsForFilters(filters: Filter[]) {
+    const { sql, parameters } = this.buildSQLQueryForFilters(filters);
+
+    const results = this.db
+      .prepare<any[], { count: number }>(`SELECT COUNT(*) as count FROM ( ${sql} )`)
+      .get(parameters) as { count: number } | undefined;
+    return results?.count ?? 0;
+  }
+}

src/sqlite/migrations.ts

@@ -0,0 +1,94 @@
+import { unixNow } from "applesauce-core/helpers";
+import { Database } from "better-sqlite3";
+
+type ScriptFunction = (database: Database, log: (message: string) => void) => Promise<void>;
+type MigrationScript = { version: number; migrate: ScriptFunction };
+
+export class MigrationSet {
+  scripts: MigrationScript[] = [];
+
+  name: string;
+  database?: Database;
+  setupMigrationTables = true;
+
+  constructor(name: string, database?: Database) {
+    this.database = database;
+    this.name = name;
+  }
+
+  private ensureMigrations(database: Database | undefined = this.database) {
+    if (!database) throw new Error("database required");
+
+    database
+      .prepare(
+        `
+			CREATE TABLE IF NOT EXISTS "migrations" (
+				"id"	INTEGER NOT NULL,
+				"name"	TEXT NOT NULL,
+				"version"	INTEGER NOT NULL,
+				"date"	INTEGER NOT NULL,
+				PRIMARY KEY("id" AUTOINCREMENT)
+			);
+			`,
+      )
+      .run();
+    database
+      .prepare(
+        `
+			CREATE TABLE IF NOT EXISTS "migration_logs" (
+				"id"	INTEGER NOT NULL,
+				"migration"	INTEGER NOT NULL,
+				"message"	TEXT NOT NULL,
+				FOREIGN KEY("migration") REFERENCES "migrations",
+				PRIMARY KEY("id" AUTOINCREMENT)
+			);
+			`,
+      )
+      .run();
+  }
+
+  addScript(version: number, migrate: ScriptFunction) {
+    this.scripts.push({ version, migrate });
+  }
+
+  async run(database: Database | undefined = this.database) {
+    if (!database) throw new Error("database required");
+
+    // ensure migration tables are setup
+    await this.ensureMigrations(database);
+
+    const prev = database
+      .prepare<[string], { name: string; version: number }>(`SELECT * FROM migrations WHERE name=?`)
+      .all(this.name);
+    const lastVersion = prev.reduce((v, m) => Math.max(m.version, v), 0);
+
+    const sorted = Array.from(this.scripts).sort((a, b) => a.version - b.version);
+
+    let version = lastVersion;
+    for (const script of sorted) {
+      if (version < script.version) {
+        let logs: string[] = [];
+        await database.transaction(() => {
+          return script.migrate(database, (message) => logs.push(message));
+        })();
+
+        version = script.version;
+
+        // save the migration
+        database.transaction(() => {
+          const result = database
+            .prepare<[string, number, number]>(`INSERT INTO migrations (name, version, date) VALUES (?1, ?2, ?3)`)
+            .run(this.name, script.version, unixNow());
+
+          const insertLog = database.prepare<[number | bigint, string]>(
+            `INSERT INTO migration_logs (migration, message) VALUES (?, ?)`,
+          );
+
+          for (const message of logs) {
+            insertLog.run(result.lastInsertRowid, message);
+          }
+        })();
+      }
+    }
+  }
+}