aljaz/bakery
1
0
Fork 0
mirror of https://github.com/aljazceru/bakery.git synced 2025-12-17 04:35:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

large refactor

Browse Source
This commit is contained in:
hzrd149
2025-03-08 15:52:12 +00:00
parent 5ccfdee953
commit 6a07ce7888
45 changed files with 3085 additions and 770 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1155
bun.lock Normal file
View File

File diff suppressed because it is too large Load Diff

1
package.json
View File

@@ -64,6 +64,7 @@
"@swc-node/register": "^1.10.9",
"@swc/core": "^1.10.18",
"@types/better-sqlite3": "^7.6.12",
"@types/bun": "^1.2.4",
"@types/cors": "^2.8.17",
"@types/debug": "^4.1.12",
"@types/express": "^4.17.21",

519
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff

8
src/app/database.ts
View File

@@ -34,14 +34,14 @@ export default class LocalDatabase extends EventEmitter {
// Detect architecture to pass the correct native sqlite module
this.db = new Database(this.path.main);
if (this.config.wal) this.db.pragma("journal_mode = WAL");
if (this.config.wal) this.db.exec("PRAGMA journal_mode = WAL");
}
hasTable(table: string) {
const result = this.db
.prepare(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='table' AND name=?`)
.get([table]) as { count: number };
return result.count > 0;
.prepare<[string], { count: number }>(`SELECT COUNT(*) as count FROM sqlite_master WHERE type='table' AND name=?`)
.get(table);
return !!result && result.count > 0;
}
// Delete all events in the database

15
src/app/index.ts
View File

@@ -1,8 +1,6 @@
import { WebSocketServer } from "ws";
import { createServer, Server } from "http";
import { Server } from "http";
import { SimpleSigner } from "applesauce-signers/signers/simple-signer";
import { IEventStore, NostrRelay, SQLiteEventStore } from "@satellite-earth/core";
import { getDMRecipient } from "@satellite-earth/core/helpers/nostr";
import { kinds } from "nostr-tools";
import { AbstractRelay } from "nostr-tools/abstract-relay";
import express, { Express } from "express";
@@ -48,9 +46,12 @@ import secrets from "../services/secrets.js";
import config from "../services/config.js";
import logStore from "../services/log-store.js";
import stateManager from "../services/state.js";
import eventCache from "../services/event-cache.js";
import sqliteEventStore from "../services/event-cache.js";
import { inboundNetwork, outboundNetwork } from "../services/network.js";
import { server } from "../services/server.js";
import { SQLiteEventStore } from "../sqlite/event-store.js";
import { NostrRelay } from "../relay/nostr-relay.js";
import { getDMRecipient } from "../helpers/nostr/dms.js";
type EventMap = {
listening: [];
@@ -71,7 +72,7 @@ export default class App extends EventEmitter<EventMap> {
outboundNetwork: OutboundNetworkManager;
database: Database;
eventStore: IEventStore;
eventStore: SQLiteEventStore;
logStore: LogStore;
relay: NostrRelay;
receiver: Receiver;
@@ -114,7 +115,7 @@ export default class App extends EventEmitter<EventMap> {
// create http and ws server interface
this.server = server;
this.inboundNetwork = inboundNetwork
this.inboundNetwork = inboundNetwork;
this.outboundNetwork = outboundNetwork;
/** make the outbound network reflect the app config */
@@ -153,7 +154,7 @@ export default class App extends EventEmitter<EventMap> {
});
// Initialize the event store
this.eventStore = eventCache;
this.eventStore = sqliteEventStore;
// setup decryption cache
this.decryptionCache = new DecryptionCache(this.database.db);

100
src/classes/json-file.ts Normal file
View File

@@ -0,0 +1,100 @@
import { EventEmitter } from "events";
import { Adapter, Low, LowSync, SyncAdapter } from "lowdb";
type EventMap<T> = {
/** fires when file is loaded */
loaded: [T];
/** fires when a field is set */
changed: [T, string, any];
/** fires when file is loaded or changed */
updated: [T];
saved: [T];
};
export class ReactiveJsonFile<T extends object> extends EventEmitter<EventMap<T>> implements Low<T> {
protected db: Low<T>;
adapter: Adapter<T>;
data: T;
constructor(adapter: Adapter<T>, defaultData: T) {
super();
this.adapter = adapter;
this.db = new Low<T>(adapter, defaultData);
this.data = this.createProxy();
}
private createProxy() {
return (this.data = new Proxy(this.db.data, {
get(target, prop, receiver) {
return Reflect.get(target, prop, receiver);
},
set: (target, p, newValue, receiver) => {
Reflect.set(target, p, newValue, receiver);
this.emit("changed", target as T, String(p), newValue);
this.emit("updated", target as T);
return true;
},
}));
}
async read() {
await this.db.read();
this.emit("loaded", this.db.data);
this.emit("updated", this.db.data);
this.createProxy();
}
async write() {
await this.db.write();
this.emit("saved", this.db.data);
}
update(fn: (data: T) => unknown) {
return this.db.update(fn);
}
}
export class ReactiveJsonFileSync<T extends object> extends EventEmitter<EventMap<T>> implements LowSync<T> {
protected db: LowSync<T>;
adapter: SyncAdapter<T>;
data: T;
constructor(adapter: SyncAdapter<T>, defaultData: T) {
super();
this.adapter = adapter;
this.db = new LowSync<T>(adapter, defaultData);
this.data = this.createProxy();
}
private createProxy() {
return (this.data = new Proxy(this.db.data, {
get(target, prop, receiver) {
return Reflect.get(target, prop, receiver);
},
set: (target, p, newValue, receiver) => {
Reflect.set(target, p, newValue, receiver);
this.emit("changed", target as T, String(p), newValue);
this.emit("updated", target as T);
return true;
},
}));
}
read() {
this.db.read();
this.emit("loaded", this.db.data);
this.emit("updated", this.db.data);
this.createProxy();
}
write() {
this.db.write();
this.emit("saved", this.db.data);
}
update(fn: (data: T) => unknown) {
return this.db.update(fn);
}
}

21
src/helpers/nostr/channel.ts Normal file
View File

@@ -0,0 +1,21 @@
import { NostrEvent } from 'nostr-tools';
import { getTagValue } from './event.js';
import { COMMUNITY_CHANNEL_KIND } from './kinds.js';
export const CHANNEL_KIND = COMMUNITY_CHANNEL_KIND;
export function getChannelId(channel: NostrEvent) {
const id = getTagValue(channel, 'd');
if (!id) throw new Error('Channel missing id');
return id;
}
export function getChannelName(channel: NostrEvent) {
return getTagValue(channel, 'name');
}
export function getChannelAbout(channel: NostrEvent) {
return getTagValue(channel, 'about');
}
export function getChannelPicture(channel: NostrEvent) {
return getTagValue(channel, 'picture');
}

18
src/helpers/nostr/communities.ts Normal file
View File

@@ -0,0 +1,18 @@
import { NostrEvent } from 'nostr-tools';
const tagValue = (e: NostrEvent, k: string) => e.tags.find((t) => t[0] === k)?.[1];
export function getCommunityName(definition: NostrEvent) {
return tagValue(definition, 'name');
}
export function getCommunityBanner(definition: NostrEvent) {
return tagValue(definition, 'banner');
}
export function getCommunityImage(definition: NostrEvent) {
return tagValue(definition, 'image');
}
export function getCommunityRelay(definition: NostrEvent) {
return tagValue(definition, 'r');
}
export function getCommunityCDN(definition: NostrEvent) {
return tagValue(definition, 'cdn');
}

20
src/helpers/nostr/contacts.ts Normal file
View File

@@ -0,0 +1,20 @@
import { NostrEvent } from 'nostr-tools';
import { safeRelayUrl } from './relays.js';
export function getRelaysFromContactList(event: NostrEvent) {
try {
const json = JSON.parse(event.content) as Record<string, { write?: boolean; read?: boolean }>;
const relays: { url: string; write?: boolean; read?: boolean }[] = [];
for (const [url, value] of Object.entries(json)) {
const safeUrl = safeRelayUrl(url);
if (safeUrl) {
relays.push({ url: safeUrl, write: value.write, read: value.read });
}
}
return relays;
} catch (error) {
return null;
}
}

10
src/helpers/nostr/dms.ts Normal file
View File

@@ -0,0 +1,10 @@
import { NostrEvent } from 'nostr-tools';
export function getDMSender(event: NostrEvent) {
return event.pubkey;
}
export function getDMRecipient(event: NostrEvent) {
const pubkey = event.tags.find((t) => t[0] === 'p')?.[1];
if (!pubkey) throw new Error('Missing recipient pubkey');
return pubkey;
}

63
src/helpers/nostr/event.ts Normal file
View File

@@ -0,0 +1,63 @@
import { NostrEvent, kinds, nip19 } from 'nostr-tools';
export function isReplaceable(kind: number) {
return kinds.isReplaceableKind(kind) || kinds.isParameterizedReplaceableKind(kind);
}
export function sortByDate(a: NostrEvent, b: NostrEvent) {
return b.created_at - a.created_at;
}
export function getEventCoordinate(event: NostrEvent) {
const d = event.tags.find((t) => t[0] === 'd')?.[1];
return d ? `${event.kind}:${event.pubkey}:${d}` : `${event.kind}:${event.pubkey}`;
}
export function getTagValue(event: NostrEvent, tag: string) {
return event.tags.find((t) => t[0] === tag)?.[1];
}
export function doesEventMatchCoordinate(event: NostrEvent, coordinate: string) {
const [kind, pubkey, d] = coordinate.split(':');
if (!kind || !pubkey || !d) return false;
return (
event.kind === parseInt(kind) && event.pubkey === event.pubkey && event.tags.find((t) => t[0] === 'd')?.[1] === d
);
}
export type CustomAddressPointer = Omit<nip19.AddressPointer, 'identifier'> & {
identifier?: string;
};
export function parseCoordinate(a: string): CustomAddressPointer | null;
export function parseCoordinate(a: string, requireD: false): CustomAddressPointer | null;
export function parseCoordinate(a: string, requireD: true): nip19.AddressPointer | null;
export function parseCoordinate(a: string, requireD: false, silent: false): CustomAddressPointer;
export function parseCoordinate(a: string, requireD: true, silent: false): nip19.AddressPointer;
export function parseCoordinate(a: string, requireD: true, silent: true): nip19.AddressPointer | null;
export function parseCoordinate(a: string, requireD: false, silent: true): CustomAddressPointer | null;
export function parseCoordinate(a: string, requireD = false, silent = true): CustomAddressPointer | null {
const parts = a.split(':') as (string | undefined)[];
const kind = parts[0] && parseInt(parts[0]);
const pubkey = parts[1];
const d = parts[2];
if (!kind) {
if (silent) return null;
else throw new Error('Missing kind');
}
if (!pubkey) {
if (silent) return null;
else throw new Error('Missing pubkey');
}
if (requireD && d === undefined) {
if (silent) return null;
else throw new Error('Missing identifier');
}
return {
kind,
pubkey,
identifier: d,
};
}

7
src/helpers/nostr/index.ts Normal file
View File

@@ -0,0 +1,7 @@
export * from './channel.js';
export * from './communities.js';
export * from './dms.js';
export * from './event.js';
export * from './kinds.js';
export * from './profile.js';
export * from './nip19.js';

2
src/helpers/nostr/kinds.ts Normal file
View File

@@ -0,0 +1,2 @@
export const COMMUNITY_CHANNEL_KIND = 39000;
export const COMMUNITY_CHAT_MESSAGE = 9;

104
src/helpers/nostr/lists.ts Normal file
View File

@@ -0,0 +1,104 @@
import { EventTemplate, nip19, NostrEvent } from 'nostr-tools';
import { parseCoordinate } from './event.js';
function unixNow() {
return Math.round(Date.now() / 1000);
}
export function getPubkeysFromList(event: NostrEvent | EventTemplate) {
return event.tags.filter((t) => t[0] === 'p' && t[1]).map((t) => ({ pubkey: t[1], relay: t[2], petname: t[3] }));
}
export function getEventPointersFromList(event: NostrEvent | EventTemplate): nip19.EventPointer[] {
return event.tags
.filter((t) => t[0] === 'e' && t[1])
.map((t) => (t[2] ? { id: t[1], relays: [t[2]] } : { id: t[1] }));
}
export function getCoordinatesFromList(event: NostrEvent | EventTemplate) {
return event.tags.filter((t) => t[0] === 'a' && t[1] && t[2]).map((t) => ({ coordinate: t[1], relay: t[2] }));
}
export function getAddressPointersFromList(event: NostrEvent | EventTemplate): nip19.AddressPointer[] {
const pointers: nip19.AddressPointer[] = [];
for (const tag of event.tags) {
if (!tag[1]) continue;
const relay = tag[2];
const parsed = parseCoordinate(tag[1]);
if (!parsed?.identifier) continue;
pointers.push({ ...parsed, identifier: parsed?.identifier, relays: relay ? [relay] : undefined });
}
return pointers;
}
export function isPubkeyInList(list?: NostrEvent, pubkey?: string) {
if (!pubkey || !list) return false;
return list.tags.some((t) => t[0] === 'p' && t[1] === pubkey);
}
export function listAddPerson(
list: NostrEvent | EventTemplate,
pubkey: string,
relay?: string,
petname?: string,
): EventTemplate {
if (list.tags.some((t) => t[0] === 'p' && t[1] === pubkey)) throw new Error('Person already in list');
const pTag = ['p', pubkey, relay ?? '', petname ?? ''];
while (pTag[pTag.length - 1] === '') pTag.pop();
return {
created_at: unixNow(),
kind: list.kind,
content: list.content,
tags: [...list.tags, pTag],
};
}
export function listRemovePerson(list: NostrEvent | EventTemplate, pubkey: string): EventTemplate {
return {
created_at: unixNow(),
kind: list.kind,
content: list.content,
tags: list.tags.filter((t) => !(t[0] === 'p' && t[1] === pubkey)),
};
}
export function listAddEvent(list: NostrEvent | EventTemplate, event: string, relay?: string): EventTemplate {
if (list.tags.some((t) => t[0] === 'e' && t[1] === event)) throw new Error('Event already in list');
return {
created_at: unixNow(),
kind: list.kind,
content: list.content,
tags: [...list.tags, relay ? ['e', event, relay] : ['e', event]],
};
}
export function listRemoveEvent(list: NostrEvent | EventTemplate, event: string): EventTemplate {
return {
created_at: unixNow(),
kind: list.kind,
content: list.content,
tags: list.tags.filter((t) => !(t[0] === 'e' && t[1] === event)),
};
}
export function listAddCoordinate(list: NostrEvent | EventTemplate, coordinate: string, relay?: string): EventTemplate {
if (list.tags.some((t) => t[0] === 'a' && t[1] === coordinate)) throw new Error('Event already in list');
return {
created_at: unixNow(),
kind: list.kind,
content: list.content,
tags: [...list.tags, relay ? ['a', coordinate, relay] : ['a', coordinate]],
};
}
export function listRemoveCoordinate(list: NostrEvent | EventTemplate, coordinate: string): EventTemplate {
return {
created_at: unixNow(),
kind: list.kind,
content: list.content,
tags: list.tags.filter((t) => !(t[0] === 'a' && t[1] === coordinate)),
};
}

15
src/helpers/nostr/long-form.ts Normal file
View File

@@ -0,0 +1,15 @@
import { NostrEvent } from 'nostr-tools';
export function getArticleTitle(event: NostrEvent) {
return event.tags.find((t) => t[0] === 'title')?.[1];
}
export function getArticleSummary(event: NostrEvent) {
return event.tags.find((t) => t[0] === 'summary')?.[1];
}
export function getArticleImage(event: NostrEvent) {
return event.tags.find((t) => t[0] === 'image')?.[1];
}
export function getArticlePublishDate(event: NostrEvent) {
const timestamp = event.tags.find((t) => t[0] === 'published_at')?.[1];
return timestamp ? parseInt(timestamp) : undefined;
}

17
src/helpers/nostr/mailboxes.ts Normal file
View File

@@ -0,0 +1,17 @@
import { NostrEvent } from 'nostr-tools';
import { safeRelayUrls } from './relays.js';
// inbox relays can be ["r", <url>, "read"] or ["r", <url>]
export function getInboxes(event?: NostrEvent | null, fallback?: string[]) {
const tags = event ? event.tags.filter((t) => (t[0] === 'r' && t[2] === 'read') || t[2] === undefined) : [];
const urls = safeRelayUrls(tags.map((t) => t[1]));
if (fallback && urls.length === 0) return fallback;
return urls;
}
export function getOutboxes(event?: NostrEvent | null, fallback?: string[]) {
const tags = event ? event.tags.filter((t) => (t[0] === 'r' && t[2] === 'write') || t[2] === undefined) : [];
const urls = safeRelayUrls(tags.map((t) => t[1]));
if (fallback && urls.length === 0) return fallback;
return urls;
}

22
src/helpers/nostr/nip19.ts Normal file
View File

@@ -0,0 +1,22 @@
import { nip19, NostrEvent } from 'nostr-tools';
import { isReplaceable } from './event.js';
export function getSharableEventAddress(event: NostrEvent, relays?: Iterable<string>) {
if (isReplaceable(event.kind)) {
const d = event.tags.find((t) => t[0] === 'd' && t[1])?.[1];
if (!d) return null;
return nip19.naddrEncode({
kind: event.kind,
identifier: d,
pubkey: event.pubkey,
relays: relays && Array.from(relays),
});
} else {
return nip19.neventEncode({
id: event.id,
kind: event.kind,
relays: relays && Array.from(relays),
author: event.pubkey,
});
}
}

51
src/helpers/nostr/profile.ts Normal file
View File

@@ -0,0 +1,51 @@
import { NostrEvent, nip19 } from 'nostr-tools';
export type Kind0ParsedContent = {
pubkey?: string;
name?: string;
display_name?: string;
displayName?: string;
about?: string;
/** @deprecated */
image?: string;
picture?: string;
banner?: string;
website?: string;
lud16?: string;
lud06?: string;
nip05?: string;
};
export function parseKind0Event(event: NostrEvent): Kind0ParsedContent {
if (event.kind !== 0) throw new Error('expected a kind 0 event');
try {
const metadata = JSON.parse(event.content) as Kind0ParsedContent;
metadata.pubkey = event.pubkey;
// ensure nip05 is a string
if (metadata.nip05 && typeof metadata.nip05 !== 'string') metadata.nip05 = String(metadata.nip05);
// fix user website
if (metadata.website) metadata.website = fixWebsiteUrl(metadata.website);
return metadata;
} catch (e) {}
return {};
}
export function getSearchNames(metadata: Kind0ParsedContent) {
if (!metadata) return [];
return [metadata.displayName, metadata.display_name, metadata.name].filter(Boolean) as string[];
}
export function getUserDisplayName(metadata: Kind0ParsedContent | undefined, pubkey: string) {
return metadata?.displayName || metadata?.display_name || metadata?.name || nip19.npubEncode(pubkey).slice(0, 8);
}
export function fixWebsiteUrl(website: string) {
if (website.match(/^http?s:\/\//)) {
return website;
}
return 'https://' + website;
}

24
src/helpers/nostr/relays.ts Normal file
View File

@@ -0,0 +1,24 @@
export function validateRelayURL(relay: string | URL) {
if (typeof relay === 'string' && relay.includes(',ws')) throw new Error('Can not have multiple relays in one string');
const url = typeof relay === 'string' ? new URL(relay) : relay;
if (url.protocol !== 'wss:' && url.protocol !== 'ws:') throw new Error('Incorrect protocol');
return url;
}
export function isValidRelayURL(relay: string | URL) {
try {
validateRelayURL(relay);
return true;
} catch (e) {
return false;
}
}
export function safeRelayUrl(relayUrl: string | URL) {
try {
return validateRelayURL(relayUrl).toString();
} catch (e) {
return null;
}
}
export function safeRelayUrls(urls: Iterable<string>): string[] {
return Array.from(urls).map(safeRelayUrl).filter(Boolean) as string[];
}

3
src/helpers/sql.ts Normal file
View File

@@ -0,0 +1,3 @@
export function mapParams(params: any[]) {
return `(${params.map(() => "?").join(", ")})`;
}

17
src/helpers/super-map.ts Normal file
View File

@@ -0,0 +1,17 @@
export default class SuperMap<Key, Value> extends Map<Key, Value> {
newValue: (key: Key) => Value;
constructor(newValue: (key: Key) => Value) {
super();
this.newValue = newValue;
}
get(key: Key) {
let value = super.get(key);
if (value === undefined) {
value = this.newValue(key);
this.set(key, value);
}
return value;
}
}

89
src/modules/community-multiplexer.ts
View File

@@ -1,89 +0,0 @@
import { type Database } from "better-sqlite3";
import { WebSocket, WebSocketServer } from "ws";
import { type IncomingMessage } from "http";
import { randomBytes } from "crypto";
import { NostrEvent, SimplePool } from "nostr-tools";
import { HyperConnectionManager } from "./hyper-connection-manager.js";
import { logger } from "../logger.js";
import { CommunityProxy } from "./community-proxy.js";
import { IEventStore } from "@satellite-earth/core";
export class CommunityMultiplexer {
log = logger.extend("community-multiplexer");
db: Database;
eventStore: IEventStore;
pool: SimplePool;
connectionManager: HyperConnectionManager;
communities = new Map<string, CommunityProxy>();
constructor(db: Database, eventStore: IEventStore) {
this.db = db;
this.eventStore = eventStore;
this.pool = new SimplePool();
this.connectionManager = new HyperConnectionManager(randomBytes(32).toString("hex"));
this.syncCommunityDefinitions();
}
attachToServer(wss: WebSocketServer) {
wss.on("connection", this.handleConnection.bind(this));
}
handleConnection(ws: WebSocket, req: IncomingMessage) {
if (!req.url) return false;
const url = new URL(req.url, `http://${req.headers.host}`);
const pubkey = url.pathname.split("/")[1] as string | undefined;
if (!pubkey || pubkey.length !== 64) return false;
try {
let community = this.communities.get(pubkey);
if (!community) community = this.getCommunityProxy(pubkey);
// connect the socket to the relay
community.relay.handleConnection(ws, req);
return true;
} catch (error) {
this.log("Failed handle ws connection to", pubkey);
console.log(error);
return false;
}
}
syncCommunityDefinitions() {
this.log("Syncing community definitions");
const sub = this.pool.subscribeMany(["wss://nostrue.com"], [{ kinds: [12012] }], {
onevent: (event) => this.eventStore.addEvent(event),
oneose: () => sub.close(),
});
}
getCommunityProxy(pubkey: string) {
this.log("Looking for community definition", pubkey);
let definition: NostrEvent | undefined = undefined;
const local = this.eventStore.getEventsForFilters([{ kinds: [12012], authors: [pubkey] }]);
if (local[0]) definition = local[0];
if (!definition) throw new Error("Failed to find community definition");
this.log("Creating community proxy", pubkey);
const community = new CommunityProxy(this.db, definition, this.connectionManager);
community.connect();
this.communities.set(pubkey, community);
return community;
}
stop() {
for (const [pubkey, community] of this.communities) {
community.stop();
}
this.communities.clear();
this.connectionManager.stop();
}
}

175
src/modules/community-proxy.ts
View File

@@ -1,175 +0,0 @@
import { type Database } from "better-sqlite3";
import { Debugger } from "debug";
import { Filter, NostrEvent, Relay, kinds } from "nostr-tools";
import { NostrRelay, RelayActions } from "@satellite-earth/core";
import { Subscription } from "nostr-tools/abstract-relay";
import { LabeledEventStore } from "./labeled-event-store.js";
import { HyperConnectionManager } from "./hyper-connection-manager.js";
import { logger } from "../logger.js";
/** Used to connect to and sync with remote communities */
export class CommunityProxy {
log: Debugger;
database: Database;
connectionManager: HyperConnectionManager;
definition: NostrEvent;
upstream?: Relay;
eventStore: LabeledEventStore;
relay: NostrRelay;
get addresses() {
return this.definition.tags.filter((t) => t[0] === "r" && t[1]).map((t) => t[1]);
}
constructor(database: Database, communityDefinition: NostrEvent, connectionManager: HyperConnectionManager) {
this.database = database;
this.connectionManager = connectionManager;
this.definition = communityDefinition;
this.log = logger.extend("community-proxy:" + communityDefinition.pubkey);
this.eventStore = new LabeledEventStore(this.database, communityDefinition.pubkey);
this.eventStore.setup();
this.relay = new NostrRelay(this.eventStore);
// handle incoming events and pass them to the upstream relay
this.relay.registerEventHandler(async (ctx, next) => {
// send event to upstream relay
if (this.upstream) {
const result = this.upstream.publish(ctx.event);
this.log("Sent event to upstream", ctx.event.id);
return result;
} else throw new Error("Not connected to upstream");
});
this.relay.on("subscription:created", (subscription, ws) => {
this.syncChannelsFromFilters(subscription.filters);
});
this.relay.on("subscription:updated", (subscription, ws) => {
this.syncChannelsFromFilters(subscription.filters);
});
}
protected async connectUpstream() {
if (this.upstream) {
if (this.upstream.connected) this.upstream.close();
this.upstream = undefined;
}
const hyperAddress = this.definition.tags.find((t) => t[0] === "r" && t[1] && t[2] === "hyper")?.[1];
let address = this.definition.tags.find((t) => t[0] === "r" && t[1].startsWith("ws"))?.[1];
if (hyperAddress) {
const serverInfo = await this.connectionManager.getLocalAddress(hyperAddress);
address = new URL(`ws://${serverInfo.address}:${serverInfo.port}`).toString();
}
if (!address) throw new Error("Failed to find connection address");
try {
this.log("Connecting to upstream", address);
this.upstream = await Relay.connect(address);
this.upstream.onclose = () => {
this.log("Upstream connection closed");
this.upstream = undefined;
};
} catch (error) {
this.log("Failed to connect to upstream");
if (error instanceof Error) this.log(error);
}
}
async connect() {
if (this.upstream) return;
await this.connectUpstream();
setTimeout(() => {
this.syncMetadata();
this.syncDeletions();
}, 100);
}
handleEvent(event: NostrEvent) {
try {
switch (event.kind) {
case kinds.EventDeletion:
this.handleDeleteEvent(event);
break;
default:
this.eventStore.addEvent(event);
break;
}
} catch (error) {
this.log("Failed to handle event");
console.log(error);
}
}
handleDeleteEvent(deleteEvent: NostrEvent) {
const communityPubkey = this.definition.pubkey;
const ids = RelayActions.handleDeleteEvent(
this.eventStore,
deleteEvent,
deleteEvent.pubkey === communityPubkey ? () => true : undefined,
);
if (ids.length) this.log(`Deleted`, ids.length, "events");
}
syncMetadata() {
if (!this.upstream) return;
this.log("Opening subscription to sync metadata");
this.upstream.subscribe([{ kinds: [kinds.Metadata, kinds.RelayList, 12012, 39000, 39001, 39002] }], {
id: "metadata-sync",
onevent: (event) => this.handleEvent(event),
onclose: () => this.log("Closed metadata sync"),
});
}
syncDeletions() {
if (!this.upstream) return;
this.log("Opening subscription to sync deletions");
this.upstream.subscribe([{ kinds: [kinds.EventDeletion] }], {
id: "deletion-sync",
onevent: (event) => this.handleEvent(event),
onclose: () => this.log("Closed deletion sync"),
});
}
private syncChannelsFromFilters(filters: Filter[]) {
const channels = new Set<string>();
for (const filter of filters) {
if (filter["#h"]) filter["#h"].forEach((c) => channels.add(c));
}
for (const channel of channels) {
this.syncChannel(channel);
}
}
channelSubs = new Map<string, Subscription>();
syncChannel(channel: string) {
if (!this.upstream) return;
if (this.channelSubs.has(channel)) return;
this.log("Opening subscription to sync channel", channel);
const sub = this.upstream.subscribe([{ kinds: [9, 10, 11, 12], "#h": [channel] }], {
id: `channel-${channel}-sync`,
onevent: (event) => this.eventStore.addEvent(event),
onclose: () => {
this.channelSubs.delete(channel);
},
});
this.channelSubs.set(channel, sub);
}
stop() {
this.upstream?.close();
}
}

2
src/modules/config-manager.ts
View File

@@ -2,9 +2,9 @@ import { JSONFileSync } from "lowdb/node";
import _throttle from "lodash.throttle";
import { uniqueNamesGenerator, adjectives, colors, animals } from "unique-names-generator";
import { PrivateNodeConfig } from "@satellite-earth/core/types/private-node-config.js";
import { ReactiveJsonFileSync } from "@satellite-earth/core";
import { logger } from "../logger.js";
import { ReactiveJsonFileSync } from "../classes/json-file.js";
export const defaultConfig: PrivateNodeConfig = {
name: uniqueNamesGenerator({

1
src/modules/control/database-actions.ts
View File

@@ -1,5 +1,4 @@
import { WebSocket } from "ws";
import os from "node:os";
import { DatabaseMessage, DatabaseResponse, DatabaseStats } from "@satellite-earth/core/types/control-api/database.js";
import App from "../../app/index.js";

6
src/modules/decryption-cache/decryption-cache.ts
View File

@@ -1,11 +1,11 @@
import { mapParams } from "@satellite-earth/core/helpers/sql.js";
import { MigrationSet } from "@satellite-earth/core/sqlite";
import { MigrationSet } from "../../sqlite/migrations.js";
import { type Database } from "better-sqlite3";
import { EventEmitter } from "events";
import { EventRow, parseEventRow } from "../../sqlite/event-store.js";
import { logger } from "../../logger.js";
import { EventRow, parseEventRow } from "@satellite-earth/core/sqlite-event-store";
import { NostrEvent } from "nostr-tools";
import { mapParams } from "../../helpers/sql.js";
const migrations = new MigrationSet("decryption-cache");

2
src/modules/direct-message-manager.ts
View File

@@ -1,7 +1,6 @@
import { filter, lastValueFrom, mergeMap, Subscription, tap, toArray } from "rxjs";
import { NostrEvent, kinds } from "nostr-tools";
import { createRxForwardReq } from "rx-nostr";
import { getRelaysFromContactList } from "@satellite-earth/core/helpers/nostr/contacts.js";
import { MailboxesQuery } from "applesauce-core/queries";
import { EventEmitter } from "events";
@@ -12,6 +11,7 @@ import { rxNostr } from "../services/rx-nostr.js";
import { eventStore, queryStore } from "../services/stores.js";
import { COMMON_CONTACT_RELAYS } from "../env.js";
import { bufferAudit } from "../helpers/rxjs.js";
import { getRelaysFromContactList } from "../helpers/nostr/contacts.js";
type EventMap = {
open: [string, string];

7
src/modules/gossip.ts
View File

@@ -1,11 +1,12 @@
import { SimpleSigner } from "applesauce-signers/signers/simple-signer";
import { EventTemplate, SimplePool } from "nostr-tools";
import { getTagValue } from "applesauce-core/helpers";
import { IEventStore, NostrRelay } from "@satellite-earth/core";
import dayjs, { Dayjs } from "dayjs";
import { logger } from "../logger.js";
import InboundNetworkManager from "./network/inbound/index.js";
import { NostrRelay } from "../relay/nostr-relay.js";
import { SQLiteEventStore } from "../sqlite/event-store.js";
function buildGossipTemplate(self: string, address: string, network: string): EventTemplate {
return {
@@ -28,7 +29,7 @@ export default class Gossip {
signer: SimpleSigner;
pool: SimplePool;
relay: NostrRelay;
eventStore: IEventStore;
eventStore: SQLiteEventStore;
running = false;
// default every 30 minutes
@@ -40,7 +41,7 @@ export default class Gossip {
signer: SimpleSigner,
pool: SimplePool,
relay: NostrRelay,
eventStore: IEventStore,
eventStore: SQLiteEventStore,
) {
this.network = network;
this.signer = signer;

79
src/modules/labeled-event-store.ts
View File

@@ -1,79 +0,0 @@
import { Database } from "better-sqlite3";
import { Filter, NostrEvent } from "nostr-tools";
import { IEventStore, SQLiteEventStore } from "@satellite-earth/core";
import { logger } from "../logger.js";
import { MigrationSet } from "@satellite-earth/core/sqlite";
export function mapParams(params: any[]) {
return `(${params.map(() => `?`).join(", ")})`;
}
const migrations = new MigrationSet("labeled-event-store");
// Version 1
migrations.addScript(1, async (db, log) => {
db.prepare(
`
CREATE TABLE IF NOT EXISTS event_labels (
id INTEGER PRIMARY KEY AUTOINCREMENT,
event TEXT(64) REFERENCES events(id),
label TEXT
)
`,
).run();
db.prepare("CREATE INDEX IF NOT EXISTS event_labels_label ON event_labels(label)").run();
db.prepare("CREATE INDEX IF NOT EXISTS event_labels_event ON event_labels(event)").run();
});
/** An event store that is can only see a subset of events int the database */
export class LabeledEventStore extends SQLiteEventStore implements IEventStore {
label: string;
readAll = false;
constructor(db: Database, label: string) {
super(db);
this.label = label;
this.log = logger.extend(`event-store:` + label);
}
async setup() {
await super.setup();
await migrations.run(this.db);
}
override buildConditionsForFilters(filter: Filter) {
const parts = super.buildConditionsForFilters(filter);
if (!this.readAll) {
parts.joins.push("INNER JOIN event_labels ON events.id = event_labels.event");
parts.conditions.push("event_labels.label = ?");
parts.parameters.push(this.label);
return parts;
}
return parts;
}
addEvent(event: NostrEvent) {
const inserted = super.addEvent(event);
const hasLabel = !!this.db
.prepare("SELECT * FROM event_labels WHERE event = ? AND label = ?")
.get(event.id, this.label);
if (!hasLabel) this.db.prepare(`INSERT INTO event_labels (event, label) VALUES (?, ?)`).run(event.id, this.label);
return inserted;
}
removeEvents(ids: string[]) {
this.db.prepare(`DELETE FROM event_labels WHERE event IN ${mapParams(ids)}`).run(...ids);
return super.removeEvents(ids);
}
removeEvent(id: string) {
this.db.prepare(`DELETE FROM event_labels WHERE event = ?`).run(id);
return super.removeEvent(id);
}
}

4
src/modules/log-store/log-store.ts
View File

@@ -1,10 +1,10 @@
import { type Database as SQLDatabase } from "better-sqlite3";
import { MigrationSet } from "@satellite-earth/core/sqlite";
import EventEmitter from "events";
import { nanoid } from "nanoid";
import { Debugger } from "debug";
import { logger } from "../../logger.js";
import { MigrationSet } from "../../sqlite/migrations.js";
type EventMap = {
log: [LogEntry];
@@ -157,7 +157,7 @@ export default class LogStore extends EventEmitter<EventMap> {
}
if (conditions.length > 0) sql += ` WHERE ${conditions.join(" AND ")}`;
this.database.prepare<any[], DatabaseLogEntry>(sql).run(...parameters);
this.database.prepare<any[]>(sql).run(parameters);
this.emit("clear", filter?.service);
}
}

2
src/modules/notifications/notifications-manager.ts
View File

@@ -1,5 +1,4 @@
import { NotificationChannel, WebPushNotification } from "@satellite-earth/core/types/control-api/notifications.js";
import { getDMRecipient, getDMSender, getUserDisplayName } from "@satellite-earth/core/helpers/nostr";
import { NostrEvent, kinds } from "nostr-tools";
import { npubEncode } from "nostr-tools/nip19";
import { getDisplayName, unixNow } from "applesauce-core/helpers";
@@ -10,6 +9,7 @@ import { logger } from "../../logger.js";
import App from "../../app/index.js";
import stateManager from "../../services/state.js";
import config from "../../services/config.js";
import { getDMRecipient, getDMSender } from "../../helpers/nostr/dms.js";
export type NotificationsManagerState = {
channels: NotificationChannel[];

175
src/modules/pubkey-batch-loader.ts
View File

@@ -1,175 +0,0 @@
import { Filter, NostrEvent, SimplePool } from "nostr-tools";
import _throttle from "lodash.throttle";
import { EventEmitter } from "events";
import { getInboxes, getOutboxes } from "@satellite-earth/core/helpers/nostr/mailboxes.js";
import SuperMap from "@satellite-earth/core/helpers/super-map.js";
import { Deferred, createDefer } from "applesauce-core/promise";
import { COMMON_CONTACT_RELAYS } from "../env.js";
type EventMap = {
event: [NostrEvent];
batch: [number, number];
};
/** Loads 10002 events for pubkeys */
export default class PubkeyBatchLoader extends EventEmitter<EventMap> {
extraRelays = COMMON_CONTACT_RELAYS;
kind: number;
pool: SimplePool;
loadFromCache?: (pubkey: string) => NostrEvent | undefined;
get queue() {
return this.next.size;
}
failed = new SuperMap<string, Set<string>>(() => new Set());
constructor(kind: number, pool: SimplePool, loadFromCache?: (pubkey: string) => NostrEvent | undefined) {
super();
this.kind = kind;
this.pool = pool;
this.loadFromCache = loadFromCache;
}
private cache = new Map<string, NostrEvent>();
getEvent(pubkey: string) {
if (this.cache.has(pubkey)) return this.cache.get(pubkey)!;
const event = this.loadFromCache?.(pubkey);
if (event) {
this.cache.set(pubkey, event);
return event;
}
}
getOutboxes(pubkey: string) {
const mailboxes = this.getEvent(pubkey);
return mailboxes && getOutboxes(mailboxes);
}
getInboxes(pubkey: string) {
const mailboxes = this.getEvent(pubkey);
return mailboxes && getInboxes(mailboxes);
}
handleEvent(event: NostrEvent) {
if (event.kind === this.kind) {
this.emit("event", event);
const current = this.cache.get(event.pubkey);
if (!current || event.created_at > current.created_at) this.cache.set(event.pubkey, event);
}
}
/** next queue */
private next = new Map<string, string[]>();
/** currently fetching */
private fetching = new Map<string, string[]>();
/** promises for next and fetching */
private pending = new Map<string, Deferred<NostrEvent | null>>();
private fetchEventsThrottle = _throttle(this.fetchEvents.bind(this), 1000);
private async fetchEvents() {
if (this.fetching.size > 0 || this.next.size === 0) return;
// copy all from next queue to fetching queue
for (const [pubkey, relays] of this.next) this.fetching.set(pubkey, relays);
this.next.clear();
if (this.fetching.size > 0) {
const filters: Record<string, Filter> = {};
for (const [pubkey, relays] of this.fetching) {
for (const relay of relays) {
filters[relay] = filters[relay] || { kinds: [this.kind], authors: [] };
if (!filters[relay].authors?.includes(pubkey)) {
filters[relay].authors?.push(pubkey);
}
}
}
const requests: Record<string, Filter[]> = {};
for (const [relay, filter] of Object.entries(filters)) requests[relay] = [filter];
await new Promise<void>((res) => {
const sub = this.pool.subscribeManyMap(requests, {
onevent: (event) => this.handleEvent(event),
oneose: () => {
sub.close();
// resolve all pending promises
let failed = 0;
let found = 0;
for (const [pubkey, relays] of this.fetching) {
const p = this.pending.get(pubkey);
if (p) {
const event = this.getEvent(pubkey) ?? null;
p.resolve(event);
if (!event) {
failed++;
for (const url of relays) this.failed.get(pubkey).add(url);
p.reject();
} else found++;
this.pending.delete(pubkey);
}
}
this.fetching.clear();
this.emit("batch", found, failed);
res();
},
});
});
// if there are pending requests, make another request
if (this.next.size > 0) this.fetchEventsThrottle();
}
}
getOrLoadEvent(pubkey: string, relays: string[] = []): Promise<NostrEvent | null> {
// if its in the cache, return it
const event = this.getEvent(pubkey);
if (event) return Promise.resolve(event);
// if its already being fetched, return promise
const pending = this.pending.get(pubkey);
if (pending) return pending;
return this.loadEvent(pubkey, relays);
}
loadEvent(pubkey: string, relays: string[] = [], ignoreFailed = false): Promise<NostrEvent | null> {
const urls = new Set(this.next.get(pubkey));
// add relays
for (const url of relays) urls.add(url);
// add extra relays
for (const url of this.extraRelays) urls.add(url);
// filter out failed relays
if (!ignoreFailed) {
const failed = this.failed.get(pubkey);
for (const url of failed) urls.delete(url);
}
if (urls.size === 0) {
// nothing new to try return null
return Promise.resolve(null);
}
// create a promise
const defer = createDefer<NostrEvent | null>();
this.pending.set(pubkey, defer);
// add pubkey and relay to next queue
this.next.set(pubkey, Array.from(urls));
// trigger queue
this.fetchEventsThrottle();
return defer;
}
}

2
src/modules/receiver/index.ts
View File

@@ -1,6 +1,5 @@
import EventEmitter from "events";
import { NostrEvent, SimplePool } from "nostr-tools";
import SuperMap from "@satellite-earth/core/helpers/super-map.js";
import { Subscription } from "nostr-tools/abstract-relay";
import { getRelaysFromContactsEvent } from "applesauce-core/helpers";
@@ -9,6 +8,7 @@ import { logger } from "../../logger.js";
import App from "../../app/index.js";
import { arrayFallback } from "../../helpers/array.js";
import { requestLoader } from "../../services/loaders.js";
import SuperMap from "../../helpers/super-map.js";
type EventMap = {
started: [Receiver];

2
src/modules/reports/manager.ts
View File

@@ -1,8 +1,8 @@
import SuperMap from "@satellite-earth/core/helpers/super-map.js";
import { filter, Observable, shareReplay, Subscription } from "rxjs";
import hash_sum from "hash-sum";
import { Session } from "../../relay/session.js";
import SuperMap from "../../helpers/super-map.js";
// open query messages (id, type, args)
export type QueryOpen<Args extends Record<string, any>> = ["QRY", "OPEN", string, string, Args];

4
src/modules/reports/reports/conversations.ts
View File

@@ -1,9 +1,9 @@
import { ReportArguments, ReportResults } from "@satellite-earth/core/types";
import { getTagValue } from "applesauce-core/helpers";
import { NostrEvent } from "nostr-tools";
import { getTagValue } from "@satellite-earth/core/helpers/nostr";
import SuperMap from "@satellite-earth/core/helpers/super-map.js";
import Report from "../report.js";
import SuperMap from "../../../helpers/super-map.js";
export default class ConversationsReport extends Report<"CONVERSATIONS"> {
readonly type = "CONVERSATIONS";

2
src/modules/reports/reports/events-summary.ts
View File

@@ -1,6 +1,6 @@
import { ReportArguments } from "@satellite-earth/core/types";
import { EventRow, parseEventRow } from "@satellite-earth/core";
import Report from "../report.js";
import { EventRow, parseEventRow } from "../../../sqlite/event-store.js";
export default class EventsSummaryReport extends Report<"EVENTS_SUMMARY"> {
readonly type = "EVENTS_SUMMARY";

2
src/modules/scrapper/index.ts
View File

@@ -1,4 +1,3 @@
import SuperMap from "@satellite-earth/core/helpers/super-map.js";
import { EventEmitter } from "events";
import { NostrEvent } from "nostr-tools";
import { Deferred, createDefer } from "applesauce-core/promise";
@@ -7,6 +6,7 @@ import App from "../../app/index.js";
import { logger } from "../../logger.js";
import PubkeyScrapper from "./pubkey-scrapper.js";
import { requestLoader } from "../../services/loaders.js";
import SuperMap from "../../helpers/super-map.js";
const MAX_TASKS = 10;

2
src/modules/state/application-state-manager.ts
View File

@@ -1,7 +1,7 @@
import { MigrationSet } from "@satellite-earth/core/sqlite";
import { Database } from "better-sqlite3";
import { MutableState } from "./mutable-state.js";
import { MigrationSet } from "../../sqlite/migrations.js";
const migrations = new MigrationSet("application-state");

53
src/relay/actions/handle-delete-event.ts Normal file
View File

@@ -0,0 +1,53 @@
import { Filter, NostrEvent, kinds } from "nostr-tools";
import { SQLiteEventStore } from "../../sqlite/event-store.js";
/**
* handles kind 5 delete events
* @param doseMatch if this returns true the event will be deleted
*/
export function handleDeleteEvent(
eventStore: SQLiteEventStore,
deleteEvent: NostrEvent,
doseMatch?: (event: NostrEvent) => boolean,
) {
if (deleteEvent.kind !== kinds.EventDeletion) return [];
const events = new Map<string, NostrEvent>();
const ids = deleteEvent.tags.filter((t) => t[0] === "e" && t[1]).map((t) => t[1]);
if (ids.length) {
const eventsFromIds = eventStore.getEventsForFilters([{ ids, until: deleteEvent.created_at }]);
for (const event of eventsFromIds) events.set(event.id, event);
}
const cords = deleteEvent.tags
.filter((t) => t[0] === "a" && t[1])
.map((t) => t[1].split(":"))
.filter((cord) => cord.length === 3);
if (cords.length) {
const eventsFromCords = eventStore.getEventsForFilters(
cords.map(([kind, pubkey, d]) => {
return {
"#d": [d],
kinds: [parseInt(kind)],
authors: [pubkey],
until: deleteEvent.created_at,
} satisfies Filter;
}),
);
for (const event of eventsFromCords) events.set(event.id, event);
}
const deleteIds: string[] = [];
for (const [id, event] of events) {
// delete the target event if the delete event was signed by the community or original author
if (doseMatch?.(event) || event.pubkey === deleteEvent.pubkey) {
deleteIds.push(event.id);
}
}
eventStore.addEvent(deleteEvent);
eventStore.removeEvents(deleteIds);
return deleteIds;
}

5
src/relay/actions/index.ts Normal file
View File

@@ -0,0 +1,5 @@
import { handleDeleteEvent } from './handle-delete-event.js';
export const RelayActions = {
handleDeleteEvent,
};

424
src/relay/nostr-relay.ts Normal file
View File

@@ -0,0 +1,424 @@
import EventEmitter from "events";
import crypto, { randomUUID } from "crypto";
import { IncomingMessage } from "http";
import { type WebSocket, RawData } from "ws";
import { Filter, NostrEvent, verifyEvent, matchFilters } from "nostr-tools";
import { logger } from "../logger.js";
import { SQLiteEventStore } from "../sqlite/event-store.js";
export type IncomingReqMessage = ["REQ", string, ...Filter[]];
export type IncomingCountMessage = ["COUNT", string, ...Filter[]];
export type IncomingEventMessage = ["EVENT", NostrEvent];
export type IncomingAuthMessage = ["AUTH", NostrEvent];
export type IncomingCloseMessage = ["CLOSE", string];
export type Subscription = {
type: "REQ" | "COUNT";
ws: WebSocket;
id: string;
filters: Filter[];
};
export type HandlerNext = () => Promise<void>;
export type HandlerContext = { event: NostrEvent; socket: WebSocket; relay: NostrRelay };
export type EventHandler = (
ctx: HandlerContext,
next: HandlerNext,
) => boolean | undefined | string | void | Promise<string | boolean | undefined | void>;
export type SubscriptionFilterContext = { id: string; filters: Filter[]; socket: WebSocket; relay: NostrRelay };
export type SubscriptionFilter = (
ctx: SubscriptionFilterContext,
next: HandlerNext,
) => boolean | undefined | void | Promise<boolean | undefined | void>;
type EventMap = {
"event:received": [NostrEvent, WebSocket];
"event:inserted": [NostrEvent, WebSocket];
"event:rejected": [NostrEvent, WebSocket];
"subscription:created": [Subscription, WebSocket];
"subscription:updated": [Subscription, WebSocket];
"subscription:closed": [Subscription, WebSocket];
"socket:connect": [WebSocket];
"socket:disconnect": [WebSocket];
"socket:auth": [WebSocket, NostrEvent];
};
export class NostrRelay extends EventEmitter<EventMap> {
static SUPPORTED_NIPS = [1, 4, 11, 45, 50, 70, 119];
log = logger.extend("relay");
eventStore: SQLiteEventStore;
connectionId = new WeakMap<WebSocket, string>();
// A map of subscriptions
subscriptions: Subscription[] = [];
// Create a map of connections
// in the form <connid> : <ws>
connections: Record<string, WebSocket> = {};
publicURL?: string;
requireRelayInAuth = true;
sendChallenge = false;
auth = new Map<WebSocket, { challenge: string; response?: NostrEvent }>();
checkAuth?: (ws: WebSocket, auth: NostrEvent) => boolean | string;
checkReadEvent?: (ws: WebSocket, event: NostrEvent, auth?: NostrEvent) => boolean;
constructor(eventStore: SQLiteEventStore) {
super();
this.eventStore = eventStore;
// listen for new events inserted into the store
this.eventStore.on("event:inserted", (event) => {
// make sure it wasn't the last event we inserted
if (event.id !== this.lastInserted) this.sendEventToSubscriptions(event);
});
}
async handleMessage(message: Buffer | string, ws: WebSocket) {
let data;
try {
// TODO enforce max size
// Parse JSON from the raw buffer
data = JSON.parse(typeof message === "string" ? message : message.toString("utf-8"));
if (!Array.isArray(data)) throw new Error("Message is not an array");
// Pass the data to appropriate handler
switch (data[0]) {
case "REQ":
case "COUNT":
await this.handleSubscriptionMessage(data as IncomingReqMessage | IncomingCountMessage, ws);
break;
case "EVENT":
await this.handleEventMessage(data as IncomingEventMessage, ws);
break;
case "AUTH":
await this.handleAuthMessage(data as IncomingAuthMessage, ws);
break;
case "CLOSE":
await this.handleCloseMessage(data as IncomingCloseMessage, ws);
break;
}
} catch (err) {
this.log("Failed to handle message", message.toString("utf-8"), err);
}
return data;
}
private socketCleanup = new Map<WebSocket, () => void>();
handleConnection(ws: WebSocket, req: IncomingMessage) {
let ip;
// Record the IP address of the client
if (typeof req.headers["x-forwarded-for"] === "string") {
ip = req.headers["x-forwarded-for"].split(",")[0].trim();
} else {
ip = req.socket.remoteAddress;
}
// listen for messages
const messageListener = (data: RawData, isBinary: boolean) => {
if (data instanceof Buffer) this.handleMessage(data, ws);
};
ws.on("message", messageListener);
const closeListener = () => this.handleDisconnect(ws);
ws.on("close", closeListener);
if (this.sendChallenge) {
const challenge = randomUUID();
this.auth.set(ws, { challenge });
ws.send(JSON.stringify(["AUTH", challenge]));
}
this.emit("socket:connect", ws);
// Generate a unique ID for ws connection
const id = crypto.randomUUID();
this.connectionId.set(ws, id);
this.connections[id] = ws;
this.socketCleanup.set(ws, () => {
delete this.connections[id];
ws.off("message", messageListener);
ws.off("close", closeListener);
this.connectionId.delete(ws);
this.auth.delete(ws);
this.emit("socket:disconnect", ws);
});
}
disconnectSocket(ws: WebSocket) {
this.socketCleanup.get(ws)?.();
}
handleDisconnect(ws: WebSocket) {
const id = this.connectionId.get(ws);
if (!id) return;
const openSubscriptions = this.subscriptions.filter((sub) => sub.ws === ws);
// remove all subscriptions
this.subscriptions = this.subscriptions.filter((sub) => sub.ws !== ws);
for (const sub of openSubscriptions) {
this.emit("subscription:closed", sub, ws);
}
this.connectionId.delete(ws);
delete this.connections[id];
this.emit("socket:disconnect", ws);
}
sendEventToSubscriptions(event: NostrEvent) {
for (const sub of this.subscriptions) {
if (sub.type === "REQ" && !sub.filters.some((f) => f.search) && matchFilters(sub.filters, event)) {
sub.ws.send(JSON.stringify(["EVENT", sub.id, event]));
}
}
}
/** Used to avoid infinite loop */
private lastInserted: string = "";
eventHandlers: EventHandler[] = [];
private async callEventHandler(ctx: HandlerContext, index = 0): Promise<string | boolean | undefined | void> {
const handler = this.eventHandlers[index];
if (!handler) return;
return await handler(ctx, async () => {
await this.callEventHandler(ctx, index + 1);
});
}
registerEventHandler(handler: EventHandler) {
this.eventHandlers.push(handler);
return () => this.unregisterEventHandler(handler);
}
unregisterEventHandler(handler: EventHandler) {
const idx = this.eventHandlers.indexOf(handler);
if (idx !== -1) this.eventHandlers.splice(idx, 1);
}
async handleEventMessage(data: IncomingEventMessage, ws: WebSocket) {
// Get the event data
const event = data[1] as NostrEvent;
try {
let inserted = false;
// Verify the event's signature
if (!verifyEvent(event)) throw new Error(`invalid: event failed to validate or verify`);
// NIP-70 protected events
const isProtected = event.tags.some((t) => t[0] === "-");
if (isProtected && this.auth.get(ws)?.response?.pubkey !== event.pubkey) {
throw new Error("auth-required: this event may only be published by its author");
}
const context: HandlerContext = { event, socket: ws, relay: this };
let persist = (await this.callEventHandler(context)) ?? true;
if (persist) {
try {
// Persist to database
this.lastInserted = event.id;
inserted = this.eventStore.addEvent(event);
} catch (err) {
console.log(err);
throw new Error(`error: server error`);
}
this.emit("event:received", event, ws);
if (inserted) {
this.emit("event:inserted", event, ws);
this.sendPublishOkMessage(ws, event, true, typeof persist === "string" ? persist : "");
this.sendEventToSubscriptions(event);
} else {
this.sendPublishOkMessage(ws, event, true, typeof persist === "string" ? persist : "Duplicate");
}
} else {
// reject with generic message
throw new Error("Rejected");
}
} catch (err) {
if (err instanceof Error) {
// error occurred, send back the OK message with false
this.emit("event:rejected", event, ws);
this.sendPublishOkMessage(ws, event, false, err.message);
}
}
}
// response helpers
makeAuthRequiredReason(reason: string) {
return "auth-required: " + reason;
}
sendPublishOkMessage(ws: WebSocket, event: NostrEvent, success: boolean, message?: string) {
ws.send(JSON.stringify(message ? ["OK", event.id, success, message] : ["OK", event.id, success]));
}
sendPublishAuthRequired(ws: WebSocket, event: NostrEvent, message: string) {
ws.send(JSON.stringify(["OK", event.id, false, this.makeAuthRequiredReason(message)]));
}
handleAuthMessage(data: IncomingAuthMessage, ws: WebSocket) {
try {
const event = data[1];
if (!verifyEvent(event)) {
return this.sendPublishOkMessage(ws, event, false, "Invalid event");
}
const relay = event.tags.find((t) => t[0] === "relay")?.[1];
if (this.requireRelayInAuth) {
if (!relay) {
return this.sendPublishOkMessage(ws, event, false, "Missing relay tag");
}
if (new URL(relay).toString() !== this.publicURL) {
return this.sendPublishOkMessage(ws, event, false, "Bad relay tag");
}
}
// check challenge
const challenge = this.auth.get(ws)?.challenge;
const challengeResponse = event.tags.find((t) => t[0] === "challenge")?.[1];
if (!challengeResponse || !challenge) {
return this.sendPublishOkMessage(ws, event, false, "Missing challenge tag");
}
if (challengeResponse !== challenge) {
return this.sendPublishOkMessage(ws, event, false, "Bad challenge");
}
if (this.checkAuth) {
const message = this.checkAuth(ws, event);
if (typeof message === "string") return this.sendPublishOkMessage(ws, event, false, message);
else if (message === false) return this.sendPublishOkMessage(ws, event, false, "Rejected auth");
}
this.auth.set(ws, { challenge, response: event });
this.emit("socket:auth", ws, event);
this.log("Authenticated", event.pubkey);
this.sendPublishOkMessage(ws, event, true, "Authenticated");
} catch (e) {}
}
protected runSubscription(sub: Subscription) {
const auth = this.getSocketAuth(sub.ws);
switch (sub.type) {
case "REQ":
const events = this.eventStore.getEventsForFilters(sub.filters);
for (let event of events) {
if (!this.checkReadEvent || this.checkReadEvent(sub.ws, event, auth)) {
sub.ws.send(JSON.stringify(["EVENT", sub.id, event]));
}
}
sub.ws.send(JSON.stringify(["EOSE", sub.id]));
break;
case "COUNT":
const count = this.eventStore.countEventsForFilters(sub.filters);
sub.ws.send(JSON.stringify(["COUNT", sub.id, { count }]));
break;
}
}
subscriptionFilters: SubscriptionFilter[] = [];
private async checkSubscriptionFilters(
ctx: SubscriptionFilterContext,
index = 0,
): Promise<boolean | undefined | void> {
const handler = this.subscriptionFilters[index];
if (!handler) return;
return await handler(ctx, async () => {
await this.checkSubscriptionFilters(ctx, index + 1);
});
}
registerSubscriptionFilter(filter: SubscriptionFilter) {
this.subscriptionFilters.push(filter);
return () => this.unregisterSubscriptionFilter(filter);
}
unregisterSubscriptionFilter(filter: SubscriptionFilter) {
const idx = this.subscriptionFilters.indexOf(filter);
if (idx !== -1) this.subscriptionFilters.splice(idx, 1);
}
async handleSubscriptionMessage(data: IncomingReqMessage | IncomingCountMessage, ws: WebSocket) {
const [type, id, ...filters] = data;
if (typeof id !== "string" || filters.length === 0) return;
try {
const allow = (await this.checkSubscriptionFilters({ socket: ws, filters, id, relay: this })) ?? true;
if (allow === false) {
return this.closeSubscription(id, ws, "Rejected");
}
let subscription = this.subscriptions.find((s) => s.id === id) || { type, id: id, ws, filters: [] };
// override or set the filters
subscription.filters = filters;
// only save the subscription if its not a count
if (type !== "COUNT") {
if (!this.subscriptions.includes(subscription)) {
this.subscriptions.push(subscription);
this.emit("subscription:created", subscription, ws);
} else {
this.emit("subscription:updated", subscription, ws);
}
}
// Run the subscription
await this.runSubscription(subscription);
} catch (error) {
if (typeof error === "string") {
this.closeSubscription(id, ws, error);
} else if (error instanceof Error) {
this.closeSubscription(id, ws, error.message);
}
}
}
closeSubscription(id: string, ws?: WebSocket, reason?: string) {
const subscription = this.subscriptions.find((s) => s.id === id && (ws ? s.ws === ws : true));
if (subscription) {
this.subscriptions.splice(this.subscriptions.indexOf(subscription), 1);
this.emit("subscription:closed", subscription, subscription.ws);
}
if (reason) (subscription?.ws || ws)?.send(JSON.stringify(["CLOSED", id, reason]));
}
handleCloseMessage(data: IncomingCloseMessage, ws: WebSocket) {
if (typeof data[1] !== "string") return;
const id = data[1];
this.closeSubscription(id, ws);
}
getSocketAuth(ws: WebSocket) {
return this.auth.get(ws)?.response;
}
stop() {
for (const ws of Object.values(this.connections)) {
ws.close();
}
this.removeAllListeners();
}
}

8
src/services/event-cache.ts
View File

@@ -1,7 +1,7 @@
import { SQLiteEventStore } from "@satellite-earth/core";
import { SQLiteEventStore } from "../sqlite/event-store.js";
import database from "./database.js";
const eventCache = new SQLiteEventStore(database.db);
await eventCache.setup();
const sqliteEventStore = new SQLiteEventStore(database.db);
await sqliteEventStore.setup();
export default eventCache;
export default sqliteEventStore;

6
src/services/loaders.ts
View File

@@ -5,11 +5,11 @@ import { ReplaceableLoader, RequestLoader } from "applesauce-loaders/loaders";
import { COMMON_CONTACT_RELAYS } from "../env.js";
import { rxNostr } from "./rx-nostr.js";
import eventCache from "./event-cache.js";
import sqliteEventStore from "./event-cache.js";
import { eventStore, queryStore } from "./stores.js";
function cacheRequest(filters: Filter[]) {
const events = eventCache.getEventsForFilters(filters);
const events = sqliteEventStore.getEventsForFilters(filters);
return from(events).pipe(tap(markFromCache));
}
@@ -20,7 +20,7 @@ replaceableLoader.subscribe((packet) => {
const event = eventStore.add(packet.event, packet.from);
// save it to the cache if its new
if (!isFromCache(event)) eventCache.addEvent(event);
if (!isFromCache(event)) sqliteEventStore.addEvent(event);
});
export const requestLoader = new RequestLoader(queryStore);

517
src/sqlite/event-store.ts Normal file
View File

@@ -0,0 +1,517 @@
import { Database } from "better-sqlite3";
import { Filter, NostrEvent, kinds } from "nostr-tools";
import EventEmitter from "events";
import { mapParams } from "../helpers/sql.js";
import { logger } from "../logger.js";
import { MigrationSet } from "../sqlite/migrations.js";
const isFilterKeyIndexableTag = (key: string) => {
return key[0] === "#" && key.length === 2;
};
const isFilterKeyIndexableAndTag = (key: string) => {
return key[0] === "&" && key.length === 2;
};
export type EventRow = {
id: string;
kind: number;
pubkey: string;
content: string;
tags: string;
created_at: number;
sig: string;
d?: string;
};
export function parseEventRow(row: EventRow): NostrEvent {
return { ...row, tags: JSON.parse(row.tags) };
}
// search behavior
const SEARCHABLE_TAGS = ["title", "description", "about", "summary", "alt"];
const SEARCHABLE_KIND_BLACKLIST = [kinds.EncryptedDirectMessage];
const SEARCHABLE_CONTENT_FORMATTERS: Record<number, (content: string) => string> = {
[kinds.Metadata]: (content) => {
const SEARCHABLE_PROFILE_FIELDS = [
"name",
"display_name",
"about",
"nip05",
"lud16",
"website",
// Deprecated fields
"displayName",
"username",
];
try {
const lines: string[] = [];
const json = JSON.parse(content);
for (const field of SEARCHABLE_PROFILE_FIELDS) {
if (json[field]) lines.push(json[field]);
}
return lines.join("\n");
} catch (error) {
return content;
}
},
};
function convertEventToSearchRow(event: NostrEvent) {
const tags = event.tags
.filter((t) => SEARCHABLE_TAGS.includes(t[0]))
.map((t) => t[1])
.join(" ");
const content = SEARCHABLE_CONTENT_FORMATTERS[event.kind]
? SEARCHABLE_CONTENT_FORMATTERS[event.kind](event.content)
: event.content;
return { id: event.id, content, tags };
}
const migrations = new MigrationSet("event-store");
// Version 1
migrations.addScript(1, async (db, log) => {
// Create events table
db.prepare(
`
CREATE TABLE IF NOT EXISTS events (
id TEXT(64) PRIMARY KEY,
created_at INTEGER,
pubkey TEXT(64),
sig TEXT(128),
kind INTEGER,
content TEXT,
tags TEXT
)
`,
).run();
log("Setup events");
// Create tags table
db.prepare(
`
CREATE TABLE IF NOT EXISTS tags (
i INTEGER PRIMARY KEY AUTOINCREMENT,
e TEXT(64) REFERENCES events(id),
t TEXT(1),
v TEXT
)
`,
).run();
log("Setup tags table");
// Create indices
const indices = [
db.prepare("CREATE INDEX IF NOT EXISTS events_created_at ON events(created_at)"),
db.prepare("CREATE INDEX IF NOT EXISTS events_pubkey ON events(pubkey)"),
db.prepare("CREATE INDEX IF NOT EXISTS events_kind ON events(kind)"),
db.prepare("CREATE INDEX IF NOT EXISTS tags_e ON tags(e)"),
db.prepare("CREATE INDEX IF NOT EXISTS tags_t ON tags(t)"),
db.prepare("CREATE INDEX IF NOT EXISTS tags_v ON tags(v)"),
];
indices.forEach((statement) => statement.run());
log(`Setup ${indices.length} indices`);
});
// Version 2, search table
migrations.addScript(2, async (db, log) => {
db.prepare(
`CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(id UNINDEXED, content, tags, tokenize='trigram')`,
).run();
log("Created event search table");
const rows = db
.prepare<number[], EventRow>(`SELECT * FROM events WHERE kind NOT IN ${mapParams(SEARCHABLE_KIND_BLACKLIST)}`)
.all(...SEARCHABLE_KIND_BLACKLIST);
// insert search content into table
let changes = 0;
for (const row of rows) {
const search = convertEventToSearchRow(parseEventRow(row));
const result = db
.prepare<[string, string, string]>(`INSERT OR REPLACE INTO events_fts (id, content, tags) VALUES (?, ?, ?)`)
.run(search.id, search.content, search.tags);
changes += result.changes;
}
log(`Inserted ${changes} events into search table`);
});
// Version 3, indexed d tags
migrations.addScript(3, async (db, log) => {
db.prepare(`ALTER TABLE events ADD COLUMN d TEXT`).run();
log("Created d column");
db.prepare("CREATE INDEX IF NOT EXISTS events_d ON events(d)").run();
log(`Created d index`);
log(`Adding d tags to events table`);
let updated = 0;
db.transaction(() => {
const events = db
.prepare<[], { id: string; d: string }>(
`
SELECT events.id as id, tags.v as d
FROM events
INNER JOIN tags ON tags.e = events.id AND tags.t = 'd'
WHERE events.kind >= 30000 AND events.kind < 40000
`,
)
.all();
const update = db.prepare<[string, string]>("UPDATE events SET d = ? WHERE id = ?");
for (const row of events) {
const { changes } = update.run(row.d, row.id);
if (changes > 0) updated++;
}
})();
log(`Updated ${updated} events`);
});
type EventMap = {
"event:inserted": [NostrEvent];
"event:removed": [string];
};
export class SQLiteEventStore extends EventEmitter<EventMap> {
db: Database;
log = logger.extend("sqlite-event-store");
preserveEphemeral = false;
preserveReplaceable = false;
constructor(db: Database) {
super();
this.db = db;
}
setup() {
return migrations.run(this.db);
}
addEvent(event: NostrEvent) {
// Don't store ephemeral events in db,
// just return the event directly
if (!this.preserveEphemeral && kinds.isEphemeralKind(event.kind)) return false;
const inserted = this.db.transaction(() => {
// TODO: Check if event is replaceable and if its newer
// before inserting it into the database
// get event d value so it can be indexed
const d = kinds.isParameterizedReplaceableKind(event.kind)
? event.tags.find((t) => t[0] === "d" && t[1])?.[1]
: undefined;
const insert = this.db
.prepare(
`
INSERT OR IGNORE INTO events (id, created_at, pubkey, sig, kind, content, tags, d)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`,
)
.run(
event.id,
event.created_at,
event.pubkey,
event.sig,
event.kind,
event.content,
JSON.stringify(event.tags),
d,
);
// If event inserted, index tags, insert search
if (insert.changes) {
this.insertEventTags(event);
// Remove older replaceable events and all their associated tags
if (this.preserveReplaceable === false) {
let older: { id: string; created_at: number }[] = [];
if (kinds.isReplaceableKind(event.kind)) {
// Normal replaceable event
older = this.db
.prepare<[number, string], { id: string; created_at: number }>(
`
SELECT id, created_at FROM events WHERE kind = ? AND pubkey = ?
`,
)
.all(event.kind, event.pubkey);
} else if (kinds.isParameterizedReplaceableKind(event.kind)) {
// Parameterized Replaceable
const d = event.tags.find((t) => t[0] === "d")?.[1];
if (d) {
older = this.db
.prepare<[number, string, "d", string], { id: string; created_at: number }>(
`
SELECT events.id, events.created_at FROM events
INNER JOIN tags ON events.id = tags.e
WHERE kind = ? AND pubkey = ? AND tags.t = ? AND tags.v = ?
`,
)
.all(event.kind, event.pubkey, "d", d);
}
}
// If found other events that may need to be replaced,
// sort the events according to timestamp descending,
// falling back to id lexical order ascending as per
// NIP-01. Remove all non-most-recent events and tags.
if (older.length > 1) {
const removeIds = older
.sort((a, b) => {
return a.created_at === b.created_at ? a.id.localeCompare(b.id) : b.created_at - a.created_at;
})
.slice(1)
.map((item) => item.id);
if (!removeIds.includes(event.id)) this.log("Removed", removeIds.length, "old replaceable events");
this.removeEvents(removeIds);
// If the event that was just inserted was one of
// the events that was removed, return null so to
// indicate that the event was in effect *not*
// upserted and thus, if using the DB for a nostr
// relay, does not need to be pushed to clients
if (removeIds.indexOf(event.id) !== -1) return false;
}
}
}
return insert.changes > 0;
})();
if (inserted) {
this.insertEventIntoSearch(event);
this.emit("event:inserted", event);
}
return inserted;
}
private insertEventTags(event: NostrEvent) {
for (let tag of event.tags) {
if (tag[0].length === 1) {
this.db.prepare(`INSERT INTO tags (e, t, v) VALUES (?, ?, ?)`).run(event.id, tag[0], tag[1]);
}
}
}
private insertEventIntoSearch(event: NostrEvent) {
const search = convertEventToSearchRow(event);
return this.db
.prepare<[string, string, string]>(`INSERT OR REPLACE INTO events_fts (id, content, tags) VALUES (?, ?, ?)`)
.run(search.id, search.content, search.tags);
}
removeEvents(ids: string[]) {
const results = this.db.transaction(() => {
this.db.prepare(`DELETE FROM tags WHERE e IN ${mapParams(ids)}`).run(...ids);
this.db.prepare(`DELETE FROM events_fts WHERE id IN ${mapParams(ids)}`).run(...ids);
return this.db.prepare(`DELETE FROM events WHERE events.id IN ${mapParams(ids)}`).run(...ids);
})();
if (results.changes > 0) {
for (const id of ids) {
this.emit("event:removed", id);
}
}
}
removeEvent(id: string) {
const results = this.db.transaction(() => {
this.db.prepare(`DELETE FROM tags WHERE e = ?`).run(id);
this.db.prepare(`DELETE FROM events_fts WHERE id = ?`).run(id);
return this.db.prepare(`DELETE FROM events WHERE events.id = ?`).run(id);
})();
if (results.changes > 0) this.emit("event:removed", id);
return results.changes > 0;
}
buildConditionsForFilters(filter: Filter) {
const joins: string[] = [];
const conditions: string[] = [];
const parameters: (string | number)[] = [];
const groupBy: string[] = [];
const having: string[] = [];
// get AND tag filters
const andTagQueries = Object.keys(filter).filter(isFilterKeyIndexableAndTag);
// get OR tag filters and remove any ones that appear in the AND
const orTagQueries = Object.keys(filter)
.filter(isFilterKeyIndexableTag)
.filter((t) => !andTagQueries.includes(t));
if (orTagQueries.length > 0) {
joins.push("INNER JOIN tags as or_tags ON events.id = or_tags.e");
}
if (andTagQueries.length > 0) {
joins.push("INNER JOIN tags as and_tags ON events.id = and_tags.e");
}
if (filter.search) {
joins.push("INNER JOIN events_fts ON events_fts.id = events.id");
conditions.push(`events_fts MATCH ?`);
parameters.push('"' + filter.search.replace(/"/g, '""') + '"');
}
if (typeof filter.since === "number") {
conditions.push(`events.created_at >= ?`);
parameters.push(filter.since);
}
if (typeof filter.until === "number") {
conditions.push(`events.created_at < ?`);
parameters.push(filter.until);
}
if (filter.ids) {
conditions.push(`events.id IN ${mapParams(filter.ids)}`);
parameters.push(...filter.ids);
}
if (filter.kinds) {
conditions.push(`events.kind IN ${mapParams(filter.kinds)}`);
parameters.push(...filter.kinds);
}
if (filter.authors) {
conditions.push(`events.pubkey IN ${mapParams(filter.authors)}`);
parameters.push(...filter.authors);
}
// add AND tag filters
for (const t of andTagQueries) {
conditions.push(`and_tags.t = ?`);
parameters.push(t.slice(1));
// @ts-expect-error
const v = filter[t] as string[];
conditions.push(`and_tags.v IN ${mapParams(v)}`);
parameters.push(...v);
}
// add OR tag filters
for (let t of orTagQueries) {
conditions.push(`or_tags.t = ?`);
parameters.push(t.slice(1));
// @ts-expect-error
const v = filter[t] as string[];
conditions.push(`or_tags.v IN ${mapParams(v)}`);
parameters.push(...v);
}
// if there is an AND tag filter set GROUP BY so that HAVING can be used
if (andTagQueries.length > 0) {
groupBy.push("events.id");
having.push("COUNT(and_tags.i) = ?");
// @ts-expect-error
parameters.push(andTagQueries.reduce((t, k) => t + (filter[k] as string[]).length, 0));
}
return { conditions, parameters, joins, groupBy, having };
}
protected buildSQLQueryForFilters(filters: Filter[], select = "events.*") {
let sql = `SELECT ${select} FROM events `;
const orConditions: string[] = [];
const parameters: any[] = [];
const groupBy = new Set<string>();
const having = new Set<string>();
let joins = new Set<string>();
for (const filter of filters) {
const parts = this.buildConditionsForFilters(filter);
if (parts.conditions.length > 0) {
orConditions.push(`(${parts.conditions.join(" AND ")})`);
parameters.push(...parts.parameters);
for (const join of parts.joins) joins.add(join);
for (const group of parts.groupBy) groupBy.add(group);
for (const have of parts.having) having.add(have);
}
}
sql += Array.from(joins).join(" ");
if (orConditions.length > 0) {
sql += ` WHERE ${orConditions.join(" OR ")}`;
}
if (groupBy.size > 0) {
sql += " GROUP BY " + Array.from(groupBy).join(",");
}
if (having.size > 0) {
sql += " HAVING " + Array.from(having).join(" AND ");
}
// @ts-expect-error
const order = filters.find((f) => f.order)?.order;
if (filters.some((f) => f.search) && (order === "rank" || order === undefined)) {
sql = sql + " ORDER BY rank";
} else {
sql = sql + " ORDER BY created_at DESC";
}
let minLimit = Infinity;
for (const filter of filters) {
if (filter.limit) minLimit = Math.min(minLimit, filter.limit);
}
if (minLimit !== Infinity) {
sql += " LIMIT ?";
parameters.push(minLimit);
}
return { sql, parameters };
}
getEventsForFilters(filters: Filter[]) {
const { sql, parameters } = this.buildSQLQueryForFilters(filters);
return this.db.prepare<any[], EventRow>(sql).all(parameters).map(parseEventRow);
}
*iterateEventsForFilters(filters: Filter[]): IterableIterator<NostrEvent> {
const { sql, parameters } = this.buildSQLQueryForFilters(filters);
const iterator = this.db.prepare<any[], EventRow>(sql).iterate(parameters);
while (true) {
const { value: row, done } = iterator.next();
if (done) break;
yield parseEventRow(row);
}
}
countEventsForFilters(filters: Filter[]) {
const { sql, parameters } = this.buildSQLQueryForFilters(filters);
const results = this.db
.prepare<any[], { count: number }>(`SELECT COUNT(*) as count FROM ( ${sql} )`)
.get(parameters) as { count: number } | undefined;
return results?.count ?? 0;
}
}

94
src/sqlite/migrations.ts Normal file
View File

@@ -0,0 +1,94 @@
import { unixNow } from "applesauce-core/helpers";
import { Database } from "better-sqlite3";
type ScriptFunction = (database: Database, log: (message: string) => void) => Promise<void>;
type MigrationScript = { version: number; migrate: ScriptFunction };
export class MigrationSet {
scripts: MigrationScript[] = [];
name: string;
database?: Database;
setupMigrationTables = true;
constructor(name: string, database?: Database) {
this.database = database;
this.name = name;
}
private ensureMigrations(database: Database | undefined = this.database) {
if (!database) throw new Error("database required");
database
.prepare(
`
CREATE TABLE IF NOT EXISTS "migrations" (
"id" INTEGER NOT NULL,
"name" TEXT NOT NULL,
"version" INTEGER NOT NULL,
"date" INTEGER NOT NULL,
PRIMARY KEY("id" AUTOINCREMENT)
);
`,
)
.run();
database
.prepare(
`
CREATE TABLE IF NOT EXISTS "migration_logs" (
"id" INTEGER NOT NULL,
"migration" INTEGER NOT NULL,
"message" TEXT NOT NULL,
FOREIGN KEY("migration") REFERENCES "migrations",
PRIMARY KEY("id" AUTOINCREMENT)
);
`,
)
.run();
}
addScript(version: number, migrate: ScriptFunction) {
this.scripts.push({ version, migrate });
}
async run(database: Database | undefined = this.database) {
if (!database) throw new Error("database required");
// ensure migration tables are setup
await this.ensureMigrations(database);
const prev = database
.prepare<[string], { name: string; version: number }>(`SELECT * FROM migrations WHERE name=?`)
.all(this.name);
const lastVersion = prev.reduce((v, m) => Math.max(m.version, v), 0);
const sorted = Array.from(this.scripts).sort((a, b) => a.version - b.version);
let version = lastVersion;
for (const script of sorted) {
if (version < script.version) {
let logs: string[] = [];
await database.transaction(() => {
return script.migrate(database, (message) => logs.push(message));
})();
version = script.version;
// save the migration
database.transaction(() => {
const result = database
.prepare<[string, number, number]>(`INSERT INTO migrations (name, version, date) VALUES (?1, ?2, ?3)`)
.run(this.name, script.version, unixNow());
const insertLog = database.prepare<[number | bigint, string]>(
`INSERT INTO migration_logs (migration, message) VALUES (?, ?)`,
);
for (const message of logs) {
insertLog.run(result.lastInsertRowid, message);
}
})();
}
}
}
}