From a004e96eca7dc19d71a335258c972a84b55ff45c Mon Sep 17 00:00:00 2001 From: Gigi Date: Fri, 17 Oct 2025 22:47:20 +0200 Subject: [PATCH] feat: extract bookmark streaming helpers and centralize loading Created bookmarkStream.ts with shared helpers: - getEventKey: deduplication logic - hasEncryptedContent: encryption detection - loadBookmarksStream: streaming with non-blocking decryption Refactored bookmarkService.ts to use shared helpers: - Uses loadBookmarksStream for consistent behavior with Debug page - Maintains progressive loading via callbacks - Added accountManager parameter to fetchBookmarks Updated App.tsx to pass accountManager to fetchBookmarks: - Progressive loading indicators via onProgressUpdate callback All bookmark loading now uses the same battle-tested streaming logic as Debug page. --- src/App.tsx | 13 +- src/components/Debug.tsx | 238 ++++++++++++++++++++++++++++---- src/services/bookmarkService.ts | 104 +++----------- src/services/bookmarkStream.ts | 171 +++++++++++++++++++++++ 4 files changed, 420 insertions(+), 106 deletions(-) create mode 100644 src/services/bookmarkStream.ts diff --git a/src/App.tsx b/src/App.tsx index 202c64eb..80e219cf 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -51,7 +51,18 @@ function AppRoutes({ console.log('[app] 🔍 Loading bookmarks for', activeAccount.pubkey.slice(0, 8)) const fullAccount = accountManager.getActive() - await fetchBookmarks(relayPool, fullAccount || activeAccount, setBookmarks) + // Progressive updates via onProgressUpdate callback + await fetchBookmarks( + relayPool, + fullAccount || activeAccount, + accountManager, + setBookmarks, + undefined, // settings + () => { + // Trigger re-render on each event/decrypt (progressive loading) + setBookmarksLoading(true) + } + ) console.log('[app] ✅ Bookmarks loaded') } catch (error) { diff --git a/src/components/Debug.tsx b/src/components/Debug.tsx index 25dc2c38..63ed6a4b 100644 --- a/src/components/Debug.tsx +++ b/src/components/Debug.tsx @@ -7,9 +7,14 @@ import { useEventStore } from 'applesauce-react/hooks' import { Accounts } from 'applesauce-accounts' import { NostrConnectSigner } from 'applesauce-signers' import { RelayPool } from 'applesauce-relay' +import { Helpers } from 'applesauce-core' import { getDefaultBunkerPermissions } from '../services/nostrConnect' import { DebugBus, type DebugLogEntry } from '../utils/debugBus' import ThreePaneLayout from './ThreePaneLayout' +import { queryEvents } from '../services/dataFetch' +import { KINDS } from '../config/kinds' +import { collectBookmarksFromEvents } from '../services/bookmarkProcessing' +import type { NostrEvent } from '../services/bookmarkHelpers' import { Bookmark } from '../types/bookmarks' import { useBookmarksUI } from '../hooks/useBookmarksUI' import { useSettings } from '../hooks/useSettings' @@ -67,8 +72,15 @@ const Debug: React.FC = ({ const [isBunkerLoading, setIsBunkerLoading] = useState(false) const [bunkerError, setBunkerError] = useState(null) - // Bookmark loading timing (actual loading uses centralized function) + // Bookmark loading state + const [bookmarkEvents, setBookmarkEvents] = useState([]) + const [isLoadingBookmarks, setIsLoadingBookmarks] = useState(false) + const [bookmarkStats, setBookmarkStats] = useState<{ public: number; private: number } | null>(null) const [tLoadBookmarks, setTLoadBookmarks] = useState(null) + const [tDecryptBookmarks, setTDecryptBookmarks] = useState(null) + + // Individual event decryption results + const [decryptedEvents, setDecryptedEvents] = useState>(new Map()) // Live timing state const [liveTiming, setLiveTiming] = useState<{ @@ -97,6 +109,63 @@ const Debug: React.FC = ({ const hasNip04 = typeof (signer as { nip04?: { encrypt?: unknown; decrypt?: unknown } } | undefined)?.nip04?.encrypt === 'function' const hasNip44 = typeof (signer as { nip44?: { encrypt?: unknown; decrypt?: unknown } } | undefined)?.nip44?.encrypt === 'function' + const getKindName = (kind: number): string => { + switch (kind) { + case KINDS.ListSimple: return 'Simple List (10003)' + case KINDS.ListReplaceable: return 'Replaceable List (30003)' + case KINDS.List: return 'List (30001)' + case KINDS.WebBookmark: return 'Web Bookmark (39701)' + default: return `Kind ${kind}` + } + } + + const getEventSize = (evt: NostrEvent): number => { + const content = evt.content || '' + const tags = JSON.stringify(evt.tags || []) + return content.length + tags.length + } + + const hasEncryptedContent = (evt: NostrEvent): boolean => { + // Check for NIP-44 encrypted content (detected by Helpers) + if (Helpers.hasHiddenContent(evt)) return true + + // Check for NIP-04 encrypted content (base64 with ?iv= suffix) + if (evt.content && evt.content.includes('?iv=')) return true + + // Check for encrypted tags + if (Helpers.hasHiddenTags(evt) && !Helpers.isHiddenTagsUnlocked(evt)) return true + + return false + } + + const getBookmarkCount = (evt: NostrEvent): { public: number; private: number } => { + const publicTags = (evt.tags || []).filter((t: string[]) => t[0] === 'e' || t[0] === 'a') + const hasEncrypted = hasEncryptedContent(evt) + return { + public: publicTags.length, + private: hasEncrypted ? 1 : 0 // Can't know exact count until decrypted + } + } + + const formatBytes = (bytes: number): string => { + if (bytes < 1024) return `${bytes} B` + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB` + return `${(bytes / (1024 * 1024)).toFixed(2)} MB` + } + + const getEventKey = (evt: NostrEvent): string => { + if (evt.kind === 30003 || evt.kind === 30001) { + // Replaceable: kind:pubkey:dtag + const dTag = evt.tags?.find((t: string[]) => t[0] === 'd')?.[1] || '' + return `${evt.kind}:${evt.pubkey}:${dTag}` + } else if (evt.kind === 10003) { + // Simple list: kind:pubkey + return `${evt.kind}:${evt.pubkey}` + } + // Web bookmarks: use event id (no deduplication) + return evt.id + } + const doEncrypt = async (mode: 'nip44' | 'nip04') => { if (!signer || !pubkey) return try { @@ -166,20 +235,103 @@ const Debug: React.FC = ({ } const handleLoadBookmarks = async () => { - // Use the centralized bookmark loading (same as refresh button in sidebar) - const start = performance.now() - setLiveTiming(prev => ({ ...prev, loadBookmarks: { startTime: start } })) - - await onRefreshBookmarks() - - const ms = Math.round(performance.now() - start) - setLiveTiming(prev => ({ ...prev, loadBookmarks: undefined })) - setTLoadBookmarks(ms) + if (!relayPool || !activeAccount) { + DebugBus.warn('debug', 'Cannot load bookmarks: missing relayPool or activeAccount') + return + } + + try { + setIsLoadingBookmarks(true) + setBookmarkStats(null) + setBookmarkEvents([]) // Clear existing events + DebugBus.info('debug', 'Loading bookmark events...') + + // Start timing + const start = performance.now() + setLiveTiming(prev => ({ ...prev, loadBookmarks: { startTime: start } })) + + // Get signer for auto-decryption + const fullAccount = accountManager.getActive() + const signerCandidate = fullAccount || activeAccount + + // Use onEvent callback to stream events as they arrive + // Trust EOSE - completes when relays finish, no artificial timeouts + const rawEvents = await queryEvents( + relayPool, + { kinds: [KINDS.ListSimple, KINDS.ListReplaceable, KINDS.List, KINDS.WebBookmark], authors: [activeAccount.pubkey] }, + { + onEvent: async (evt) => { + // Add event immediately with live deduplication + setBookmarkEvents(prev => { + // Create unique key for deduplication + const key = getEventKey(evt) + + // Find existing event with same key + const existingIdx = prev.findIndex(e => getEventKey(e) === key) + + if (existingIdx >= 0) { + // Replace if newer + const existing = prev[existingIdx] + if ((evt.created_at || 0) > (existing.created_at || 0)) { + const newEvents = [...prev] + newEvents[existingIdx] = evt + return newEvents + } + return prev // Keep existing (it's newer) + } + + // Add new event + return [...prev, evt] + }) + + // Auto-decrypt if event has encrypted content + if (hasEncryptedContent(evt)) { + console.log('[bunker] 🔓 Auto-decrypting event', evt.id.slice(0, 8)) + try { + const { publicItemsAll, privateItemsAll } = await collectBookmarksFromEvents( + [evt], + activeAccount, + signerCandidate + ) + setDecryptedEvents(prev => new Map(prev).set(evt.id, { + public: publicItemsAll.length, + private: privateItemsAll.length + })) + console.log('[bunker] ✅ Auto-decrypted:', evt.id.slice(0, 8), { + public: publicItemsAll.length, + private: privateItemsAll.length + }) + } catch (error) { + console.error('[bunker] ❌ Auto-decrypt failed:', evt.id.slice(0, 8), error) + } + } + } + } + ) + + const ms = Math.round(performance.now() - start) + setLiveTiming(prev => ({ ...prev, loadBookmarks: undefined })) + setTLoadBookmarks(ms) + + DebugBus.info('debug', `Loaded ${rawEvents.length} bookmark events`, { + kinds: rawEvents.map(e => e.kind).join(', '), + ms + }) + } catch (error) { + setLiveTiming(prev => ({ ...prev, loadBookmarks: undefined })) + DebugBus.error('debug', 'Failed to load bookmarks', error instanceof Error ? error.message : String(error)) + } finally { + setIsLoadingBookmarks(false) + } } const handleClearBookmarks = () => { + setBookmarkEvents([]) + setBookmarkStats(null) setTLoadBookmarks(null) - DebugBus.info('debug', 'Cleared bookmark timing data') + setTDecryptBookmarks(null) + setDecryptedEvents(new Map()) + DebugBus.info('debug', 'Cleared bookmark data') } const handleBunkerLogin = async () => { @@ -436,19 +588,15 @@ const Debug: React.FC = ({ {/* Bookmark Loading Section */}

Bookmark Loading

-
- Uses centralized bookmark loading (same as refresh button in sidebar) -
- Bookmarks: {bookmarks.length > 0 ? `${bookmarks[0]?.individualBookmarks?.length || 0} items` : '0 items'} -
+
Test bookmark loading with auto-decryption (kinds: 10003, 30003, 30001, 39701)
+
-
- ℹī¸ This button calls the same centralized bookmark loading function as the refresh button in the sidebar. - Check the sidebar to see the loaded bookmarks, or check the console for [app] logs. -
+ {bookmarkStats && ( +
+
Decrypted Bookmarks:
+
+
Public: {bookmarkStats.public}
+
Private: {bookmarkStats.private}
+
Total: {bookmarkStats.public + bookmarkStats.private}
+
+
+ )} + + {bookmarkEvents.length > 0 && ( +
+
Loaded Events ({bookmarkEvents.length}):
+
+ {bookmarkEvents.map((evt, idx) => { + const dTag = evt.tags?.find((t: string[]) => t[0] === 'd')?.[1] + const titleTag = evt.tags?.find((t: string[]) => t[0] === 'title')?.[1] + const size = getEventSize(evt) + const counts = getBookmarkCount(evt) + const hasEncrypted = hasEncryptedContent(evt) + const decryptResult = decryptedEvents.get(evt.id) + + return ( +
+
{getKindName(evt.kind)}
+ {dTag &&
d-tag: {dTag}
} + {titleTag &&
title: {titleTag}
} +
+
Size: {formatBytes(size)}
+
Public: {counts.public}
+ {hasEncrypted &&
🔒 Has encrypted content
} +
+ {decryptResult && ( +
+
✓ Decrypted: {decryptResult.public} public, {decryptResult.private} private
+
+ )} +
ID: {evt.id}
+
+ ) + })} +
+
+ )}
{/* Debug Logs Section */} diff --git a/src/services/bookmarkService.ts b/src/services/bookmarkService.ts index cb5d3a3a..fc93c91b 100644 --- a/src/services/bookmarkService.ts +++ b/src/services/bookmarkService.ts @@ -1,5 +1,4 @@ import { RelayPool } from 'applesauce-relay' -import { Helpers } from 'applesauce-core' import { AccountWithExtension, NostrEvent, @@ -13,56 +12,22 @@ import { collectBookmarksFromEvents } from './bookmarkProcessing.ts' import { UserSettings } from './settingsService' import { rebroadcastEvents } from './rebroadcastService' import { queryEvents } from './dataFetch' -import { KINDS } from '../config/kinds' - -// Helper to check if event has encrypted content -const hasEncryptedContent = (evt: NostrEvent): boolean => { - // Check for NIP-44 encrypted content (detected by Helpers) - if (Helpers.hasHiddenContent(evt)) return true - - // Check for NIP-04 encrypted content (base64 with ?iv= suffix) - if (evt.content && evt.content.includes('?iv=')) return true - - // Check for encrypted tags - if (Helpers.hasHiddenTags(evt) && !Helpers.isHiddenTagsUnlocked(evt)) return true - - return false -} - -// Helper to deduplicate events by key -const getEventKey = (evt: NostrEvent): string => { - if (evt.kind === 30003 || evt.kind === 30001) { - // Replaceable: kind:pubkey:dtag - const dTag = evt.tags?.find((t: string[]) => t[0] === 'd')?.[1] || '' - return `${evt.kind}:${evt.pubkey}:${dTag}` - } else if (evt.kind === 10003) { - // Simple list: kind:pubkey - return `${evt.kind}:${evt.pubkey}` - } - // Web bookmarks: use event id (no deduplication) - return evt.id -} +import { loadBookmarksStream } from './bookmarkStream' export const fetchBookmarks = async ( relayPool: RelayPool, activeAccount: unknown, + accountManager: { getActive: () => unknown }, setBookmarks: (bookmarks: Bookmark[]) => void, - settings?: UserSettings + settings?: UserSettings, + onProgressUpdate?: () => void ) => { try { if (!isAccountWithExtension(activeAccount)) { throw new Error('Invalid account object provided') } - console.log('[app] 🔍 Fetching bookmark events with streaming') - - // Track events with deduplication as they arrive - const eventMap = new Map() - let processedCount = 0 - - console.log('[app] Account:', activeAccount.pubkey.slice(0, 8)) - - // Get signer for auto-decryption + // Get signer for bookmark processing const maybeAccount = activeAccount as AccountWithExtension let signerCandidate: unknown = maybeAccount const hasNip04Prop = (signerCandidate as { nip04?: unknown })?.nip04 !== undefined @@ -186,58 +151,35 @@ export const fetchBookmarks = async ( setBookmarks([bookmark]) } - // Stream events (just collect, decrypt after) - const rawEvents = await queryEvents( + // Use shared streaming helper for consistent behavior with Debug page + // Progressive updates via callbacks (non-blocking) + const { events: dedupedEvents } = await loadBookmarksStream({ relayPool, - { kinds: [KINDS.ListSimple, KINDS.ListReplaceable, KINDS.List, KINDS.WebBookmark], authors: [activeAccount.pubkey] }, - { - onEvent: (evt) => { - // Deduplicate by key - const key = getEventKey(evt) - const existing = eventMap.get(key) - - if (existing && (existing.created_at || 0) >= (evt.created_at || 0)) { - return // Keep existing (it's newer or same) - } - - // Add/update event - eventMap.set(key, evt) - processedCount++ - - console.log(`[app] 📨 Event ${processedCount}: kind=${evt.kind}, id=${evt.id.slice(0, 8)}, hasEncrypted=${hasEncryptedContent(evt)}`) + activeAccount: maybeAccount, + accountManager, + onEvent: () => { + // Signal that an event arrived (for loading indicator updates) + if (onProgressUpdate) { + onProgressUpdate() + } + }, + onDecryptComplete: () => { + // Signal that a decrypt completed (for loading indicator updates) + if (onProgressUpdate) { + onProgressUpdate() } } - ) + }) - console.log('[app] 📊 Query complete, raw events fetched:', rawEvents.length, 'events') - // Rebroadcast bookmark events to local/all relays based on settings - await rebroadcastEvents(rawEvents, relayPool, settings) + await rebroadcastEvents(dedupedEvents, relayPool, settings) - const dedupedEvents = Array.from(eventMap.values()) - console.log('[app] 📋 After deduplication:', dedupedEvents.length, 'bookmark events') - if (dedupedEvents.length === 0) { console.log('[app] ⚠ī¸ No bookmark events found') - setBookmarks([]) // Clear bookmarks if none found + setBookmarks([]) return } - // Auto-decrypt events with encrypted content (batch processing) - const encryptedEvents = dedupedEvents.filter(evt => hasEncryptedContent(evt)) - if (encryptedEvents.length > 0) { - console.log('[app] 🔓 Auto-decrypting', encryptedEvents.length, 'encrypted events') - for (const evt of encryptedEvents) { - try { - // Trigger decryption - this unlocks the content for the main collection pass - await collectBookmarksFromEvents([evt], activeAccount, signerCandidate) - console.log('[app] ✅ Auto-decrypted:', evt.id.slice(0, 8)) - } catch (error) { - console.error('[app] ❌ Auto-decrypt failed:', evt.id.slice(0, 8), error) - } - } - } - // Final update with all events (now with decrypted content) console.log('[app] 🔄 Final bookmark processing with', dedupedEvents.length, 'events') await updateBookmarks(dedupedEvents) diff --git a/src/services/bookmarkStream.ts b/src/services/bookmarkStream.ts new file mode 100644 index 00000000..c04a6a82 --- /dev/null +++ b/src/services/bookmarkStream.ts @@ -0,0 +1,171 @@ +import { RelayPool } from 'applesauce-relay' +import { Helpers } from 'applesauce-core' +import { NostrEvent } from 'nostr-tools' +import { queryEvents } from './dataFetch' +import { KINDS } from '../config/kinds' +import { collectBookmarksFromEvents } from './bookmarkProcessing' + +/** + * Get unique key for event deduplication + * Replaceable events (30001, 30003) use kind:pubkey:dtag + * Simple lists (10003) use kind:pubkey + * Web bookmarks (39701) use event id + */ +export function getEventKey(evt: NostrEvent): string { + if (evt.kind === 30003 || evt.kind === 30001) { + // Replaceable: kind:pubkey:dtag + const dTag = evt.tags?.find((t: string[]) => t[0] === 'd')?.[1] || '' + return `${evt.kind}:${evt.pubkey}:${dTag}` + } else if (evt.kind === 10003) { + // Simple list: kind:pubkey + return `${evt.kind}:${evt.pubkey}` + } + // Web bookmarks: use event id (no deduplication) + return evt.id +} + +/** + * Check if event has encrypted content + * Detects NIP-44 (via Helpers), NIP-04 (?iv= suffix), and encrypted tags + */ +export function hasEncryptedContent(evt: NostrEvent): boolean { + // Check for NIP-44 encrypted content (detected by Helpers) + if (Helpers.hasHiddenContent(evt)) return true + + // Check for NIP-04 encrypted content (base64 with ?iv= suffix) + if (evt.content && evt.content.includes('?iv=')) return true + + // Check for encrypted tags + if (Helpers.hasHiddenTags(evt) && !Helpers.isHiddenTagsUnlocked(evt)) return true + + return false +} + +interface LoadBookmarksStreamOptions { + relayPool: RelayPool + activeAccount: { pubkey: string; [key: string]: unknown } + accountManager: { getActive: () => unknown } + onEvent?: (event: NostrEvent) => void + onDecryptStart?: (eventId: string) => void + onDecryptComplete?: (eventId: string, success: boolean) => void +} + +interface LoadBookmarksStreamResult { + events: NostrEvent[] + decryptedCount: number +} + +/** + * Load bookmark events with streaming and non-blocking decryption + * - Streams events via onEvent callback as they arrive + * - Deduplicates by getEventKey + * - Decrypts encrypted events AFTER query completes (non-blocking UI) + * - Trusts EOSE signal to complete + */ +export async function loadBookmarksStream( + options: LoadBookmarksStreamOptions +): Promise { + const { + relayPool, + activeAccount, + accountManager, + onEvent, + onDecryptStart, + onDecryptComplete + } = options + + console.log('[app] 🔍 Fetching bookmark events with streaming') + console.log('[app] Account:', activeAccount.pubkey.slice(0, 8)) + + // Track events with deduplication as they arrive + const eventMap = new Map() + let processedCount = 0 + + // Get signer for auto-decryption + const fullAccount = accountManager.getActive() as { + pubkey: string + signer?: unknown + nip04?: unknown + nip44?: unknown + [key: string]: unknown + } | null + const maybeAccount = fullAccount || activeAccount + let signerCandidate: unknown = maybeAccount + const hasNip04Prop = (signerCandidate as { nip04?: unknown })?.nip04 !== undefined + const hasNip44Prop = (signerCandidate as { nip44?: unknown })?.nip44 !== undefined + if (signerCandidate && !hasNip04Prop && !hasNip44Prop && maybeAccount?.signer) { + signerCandidate = maybeAccount.signer + } + + // Stream events (just collect, decrypt after) + const rawEvents = await queryEvents( + relayPool, + { kinds: [KINDS.ListSimple, KINDS.ListReplaceable, KINDS.List, KINDS.WebBookmark], authors: [activeAccount.pubkey] }, + { + onEvent: (evt) => { + // Deduplicate by key + const key = getEventKey(evt) + const existing = eventMap.get(key) + + if (existing && (existing.created_at || 0) >= (evt.created_at || 0)) { + return // Keep existing (it's newer or same) + } + + // Add/update event + eventMap.set(key, evt) + processedCount++ + + console.log(`[app] 📨 Event ${processedCount}: kind=${evt.kind}, id=${evt.id.slice(0, 8)}, hasEncrypted=${hasEncryptedContent(evt)}`) + + // Call optional callback for progressive UI updates + if (onEvent) { + onEvent(evt) + } + } + } + ) + + console.log('[app] 📊 Query complete, raw events fetched:', rawEvents.length, 'events') + + const dedupedEvents = Array.from(eventMap.values()) + console.log('[app] 📋 After deduplication:', dedupedEvents.length, 'bookmark events') + + if (dedupedEvents.length === 0) { + console.log('[app] ⚠ī¸ No bookmark events found') + return { events: [], decryptedCount: 0 } + } + + // Auto-decrypt events with encrypted content (batch processing after EOSE) + const encryptedEvents = dedupedEvents.filter(evt => hasEncryptedContent(evt)) + let decryptedCount = 0 + + if (encryptedEvents.length > 0) { + console.log('[app] 🔓 Auto-decrypting', encryptedEvents.length, 'encrypted events') + for (const evt of encryptedEvents) { + try { + if (onDecryptStart) { + onDecryptStart(evt.id) + } + + // Trigger decryption - this unlocks the content for the bookmark collection + await collectBookmarksFromEvents([evt], activeAccount, signerCandidate) + decryptedCount++ + console.log('[app] ✅ Auto-decrypted:', evt.id.slice(0, 8)) + + if (onDecryptComplete) { + onDecryptComplete(evt.id, true) + } + } catch (error) { + console.error('[app] ❌ Auto-decrypt failed:', evt.id.slice(0, 8), error) + if (onDecryptComplete) { + onDecryptComplete(evt.id, false) + } + } + } + } + + console.log('[app] ✅ Bookmark streaming complete:', dedupedEvents.length, 'events,', decryptedCount, 'decrypted') + + return { events: dedupedEvents, decryptedCount } +} +