feat: extract bookmark streaming helpers and centralize loading

Created bookmarkStream.ts with shared helpers:
- getEventKey: deduplication logic
- hasEncryptedContent: encryption detection
- loadBookmarksStream: streaming with non-blocking decryption

Refactored bookmarkService.ts to use shared helpers:
- Uses loadBookmarksStream for consistent behavior with Debug page
- Maintains progressive loading via callbacks
- Added accountManager parameter to fetchBookmarks

Updated App.tsx to pass accountManager to fetchBookmarks:
- Progressive loading indicators via onProgressUpdate callback

All bookmark loading now uses the same battle-tested streaming logic as Debug page.
This commit is contained in:
Gigi
2025-10-17 22:47:20 +02:00
parent ce2432632c
commit a004e96eca
4 changed files with 420 additions and 106 deletions

View File

@@ -51,7 +51,18 @@ function AppRoutes({
console.log('[app] 🔍 Loading bookmarks for', activeAccount.pubkey.slice(0, 8))
const fullAccount = accountManager.getActive()
await fetchBookmarks(relayPool, fullAccount || activeAccount, setBookmarks)
// Progressive updates via onProgressUpdate callback
await fetchBookmarks(
relayPool,
fullAccount || activeAccount,
accountManager,
setBookmarks,
undefined, // settings
() => {
// Trigger re-render on each event/decrypt (progressive loading)
setBookmarksLoading(true)
}
)
console.log('[app] ✅ Bookmarks loaded')
} catch (error) {

View File

@@ -7,9 +7,14 @@ import { useEventStore } from 'applesauce-react/hooks'
import { Accounts } from 'applesauce-accounts'
import { NostrConnectSigner } from 'applesauce-signers'
import { RelayPool } from 'applesauce-relay'
import { Helpers } from 'applesauce-core'
import { getDefaultBunkerPermissions } from '../services/nostrConnect'
import { DebugBus, type DebugLogEntry } from '../utils/debugBus'
import ThreePaneLayout from './ThreePaneLayout'
import { queryEvents } from '../services/dataFetch'
import { KINDS } from '../config/kinds'
import { collectBookmarksFromEvents } from '../services/bookmarkProcessing'
import type { NostrEvent } from '../services/bookmarkHelpers'
import { Bookmark } from '../types/bookmarks'
import { useBookmarksUI } from '../hooks/useBookmarksUI'
import { useSettings } from '../hooks/useSettings'
@@ -67,8 +72,15 @@ const Debug: React.FC<DebugProps> = ({
const [isBunkerLoading, setIsBunkerLoading] = useState<boolean>(false)
const [bunkerError, setBunkerError] = useState<string | null>(null)
// Bookmark loading timing (actual loading uses centralized function)
// Bookmark loading state
const [bookmarkEvents, setBookmarkEvents] = useState<NostrEvent[]>([])
const [isLoadingBookmarks, setIsLoadingBookmarks] = useState(false)
const [bookmarkStats, setBookmarkStats] = useState<{ public: number; private: number } | null>(null)
const [tLoadBookmarks, setTLoadBookmarks] = useState<number | null>(null)
const [tDecryptBookmarks, setTDecryptBookmarks] = useState<number | null>(null)
// Individual event decryption results
const [decryptedEvents, setDecryptedEvents] = useState<Map<string, { public: number; private: number }>>(new Map())
// Live timing state
const [liveTiming, setLiveTiming] = useState<{
@@ -97,6 +109,63 @@ const Debug: React.FC<DebugProps> = ({
const hasNip04 = typeof (signer as { nip04?: { encrypt?: unknown; decrypt?: unknown } } | undefined)?.nip04?.encrypt === 'function'
const hasNip44 = typeof (signer as { nip44?: { encrypt?: unknown; decrypt?: unknown } } | undefined)?.nip44?.encrypt === 'function'
const getKindName = (kind: number): string => {
switch (kind) {
case KINDS.ListSimple: return 'Simple List (10003)'
case KINDS.ListReplaceable: return 'Replaceable List (30003)'
case KINDS.List: return 'List (30001)'
case KINDS.WebBookmark: return 'Web Bookmark (39701)'
default: return `Kind ${kind}`
}
}
const getEventSize = (evt: NostrEvent): number => {
const content = evt.content || ''
const tags = JSON.stringify(evt.tags || [])
return content.length + tags.length
}
const hasEncryptedContent = (evt: NostrEvent): boolean => {
// Check for NIP-44 encrypted content (detected by Helpers)
if (Helpers.hasHiddenContent(evt)) return true
// Check for NIP-04 encrypted content (base64 with ?iv= suffix)
if (evt.content && evt.content.includes('?iv=')) return true
// Check for encrypted tags
if (Helpers.hasHiddenTags(evt) && !Helpers.isHiddenTagsUnlocked(evt)) return true
return false
}
const getBookmarkCount = (evt: NostrEvent): { public: number; private: number } => {
const publicTags = (evt.tags || []).filter((t: string[]) => t[0] === 'e' || t[0] === 'a')
const hasEncrypted = hasEncryptedContent(evt)
return {
public: publicTags.length,
private: hasEncrypted ? 1 : 0 // Can't know exact count until decrypted
}
}
const formatBytes = (bytes: number): string => {
if (bytes < 1024) return `${bytes} B`
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
return `${(bytes / (1024 * 1024)).toFixed(2)} MB`
}
const getEventKey = (evt: NostrEvent): string => {
if (evt.kind === 30003 || evt.kind === 30001) {
// Replaceable: kind:pubkey:dtag
const dTag = evt.tags?.find((t: string[]) => t[0] === 'd')?.[1] || ''
return `${evt.kind}:${evt.pubkey}:${dTag}`
} else if (evt.kind === 10003) {
// Simple list: kind:pubkey
return `${evt.kind}:${evt.pubkey}`
}
// Web bookmarks: use event id (no deduplication)
return evt.id
}
const doEncrypt = async (mode: 'nip44' | 'nip04') => {
if (!signer || !pubkey) return
try {
@@ -166,20 +235,103 @@ const Debug: React.FC<DebugProps> = ({
}
const handleLoadBookmarks = async () => {
// Use the centralized bookmark loading (same as refresh button in sidebar)
const start = performance.now()
setLiveTiming(prev => ({ ...prev, loadBookmarks: { startTime: start } }))
await onRefreshBookmarks()
const ms = Math.round(performance.now() - start)
setLiveTiming(prev => ({ ...prev, loadBookmarks: undefined }))
setTLoadBookmarks(ms)
if (!relayPool || !activeAccount) {
DebugBus.warn('debug', 'Cannot load bookmarks: missing relayPool or activeAccount')
return
}
try {
setIsLoadingBookmarks(true)
setBookmarkStats(null)
setBookmarkEvents([]) // Clear existing events
DebugBus.info('debug', 'Loading bookmark events...')
// Start timing
const start = performance.now()
setLiveTiming(prev => ({ ...prev, loadBookmarks: { startTime: start } }))
// Get signer for auto-decryption
const fullAccount = accountManager.getActive()
const signerCandidate = fullAccount || activeAccount
// Use onEvent callback to stream events as they arrive
// Trust EOSE - completes when relays finish, no artificial timeouts
const rawEvents = await queryEvents(
relayPool,
{ kinds: [KINDS.ListSimple, KINDS.ListReplaceable, KINDS.List, KINDS.WebBookmark], authors: [activeAccount.pubkey] },
{
onEvent: async (evt) => {
// Add event immediately with live deduplication
setBookmarkEvents(prev => {
// Create unique key for deduplication
const key = getEventKey(evt)
// Find existing event with same key
const existingIdx = prev.findIndex(e => getEventKey(e) === key)
if (existingIdx >= 0) {
// Replace if newer
const existing = prev[existingIdx]
if ((evt.created_at || 0) > (existing.created_at || 0)) {
const newEvents = [...prev]
newEvents[existingIdx] = evt
return newEvents
}
return prev // Keep existing (it's newer)
}
// Add new event
return [...prev, evt]
})
// Auto-decrypt if event has encrypted content
if (hasEncryptedContent(evt)) {
console.log('[bunker] 🔓 Auto-decrypting event', evt.id.slice(0, 8))
try {
const { publicItemsAll, privateItemsAll } = await collectBookmarksFromEvents(
[evt],
activeAccount,
signerCandidate
)
setDecryptedEvents(prev => new Map(prev).set(evt.id, {
public: publicItemsAll.length,
private: privateItemsAll.length
}))
console.log('[bunker] ✅ Auto-decrypted:', evt.id.slice(0, 8), {
public: publicItemsAll.length,
private: privateItemsAll.length
})
} catch (error) {
console.error('[bunker] ❌ Auto-decrypt failed:', evt.id.slice(0, 8), error)
}
}
}
}
)
const ms = Math.round(performance.now() - start)
setLiveTiming(prev => ({ ...prev, loadBookmarks: undefined }))
setTLoadBookmarks(ms)
DebugBus.info('debug', `Loaded ${rawEvents.length} bookmark events`, {
kinds: rawEvents.map(e => e.kind).join(', '),
ms
})
} catch (error) {
setLiveTiming(prev => ({ ...prev, loadBookmarks: undefined }))
DebugBus.error('debug', 'Failed to load bookmarks', error instanceof Error ? error.message : String(error))
} finally {
setIsLoadingBookmarks(false)
}
}
const handleClearBookmarks = () => {
setBookmarkEvents([])
setBookmarkStats(null)
setTLoadBookmarks(null)
DebugBus.info('debug', 'Cleared bookmark timing data')
setTDecryptBookmarks(null)
setDecryptedEvents(new Map())
DebugBus.info('debug', 'Cleared bookmark data')
}
const handleBunkerLogin = async () => {
@@ -436,19 +588,15 @@ const Debug: React.FC<DebugProps> = ({
{/* Bookmark Loading Section */}
<div className="settings-section">
<h3 className="section-title">Bookmark Loading</h3>
<div className="text-sm opacity-70 mb-3">
Uses centralized bookmark loading (same as refresh button in sidebar)
<br />
Bookmarks: {bookmarks.length > 0 ? `${bookmarks[0]?.individualBookmarks?.length || 0} items` : '0 items'}
</div>
<div className="text-sm opacity-70 mb-3">Test bookmark loading with auto-decryption (kinds: 10003, 30003, 30001, 39701)</div>
<div className="flex gap-2 mb-3 items-center">
<button
className="btn btn-primary"
onClick={handleLoadBookmarks}
disabled={bookmarksLoading || !relayPool || !activeAccount}
disabled={isLoadingBookmarks || !relayPool || !activeAccount}
>
{bookmarksLoading ? (
{isLoadingBookmarks ? (
<>
<FontAwesomeIcon icon={faSpinner} className="animate-spin mr-2" />
Loading...
@@ -460,20 +608,62 @@ const Debug: React.FC<DebugProps> = ({
<button
className="btn btn-secondary ml-auto"
onClick={handleClearBookmarks}
disabled={!tLoadBookmarks}
disabled={bookmarkEvents.length === 0 && !bookmarkStats}
>
Clear Timing
Clear
</button>
</div>
<div className="mb-3 flex gap-2 flex-wrap">
<Stat label="load" value={tLoadBookmarks} bookmarkOp="loadBookmarks" />
<Stat label="decrypt" value={tDecryptBookmarks} bookmarkOp="decryptBookmarks" />
</div>
<div className="text-xs opacity-70 mt-3 p-2 bg-gray-100 dark:bg-gray-800 rounded">
This button calls the same centralized bookmark loading function as the refresh button in the sidebar.
Check the sidebar to see the loaded bookmarks, or check the console for [app] logs.
</div>
{bookmarkStats && (
<div className="mb-3">
<div className="text-sm opacity-70 mb-2">Decrypted Bookmarks:</div>
<div className="font-mono text-xs p-2 bg-gray-100 dark:bg-gray-800 rounded">
<div>Public: {bookmarkStats.public}</div>
<div>Private: {bookmarkStats.private}</div>
<div className="font-semibold mt-1">Total: {bookmarkStats.public + bookmarkStats.private}</div>
</div>
</div>
)}
{bookmarkEvents.length > 0 && (
<div className="mb-3">
<div className="text-sm opacity-70 mb-2">Loaded Events ({bookmarkEvents.length}):</div>
<div className="space-y-2">
{bookmarkEvents.map((evt, idx) => {
const dTag = evt.tags?.find((t: string[]) => t[0] === 'd')?.[1]
const titleTag = evt.tags?.find((t: string[]) => t[0] === 'title')?.[1]
const size = getEventSize(evt)
const counts = getBookmarkCount(evt)
const hasEncrypted = hasEncryptedContent(evt)
const decryptResult = decryptedEvents.get(evt.id)
return (
<div key={idx} className="font-mono text-xs p-2 bg-gray-100 dark:bg-gray-800 rounded">
<div className="font-semibold mb-1">{getKindName(evt.kind)}</div>
{dTag && <div className="opacity-70">d-tag: {dTag}</div>}
{titleTag && <div className="opacity-70">title: {titleTag}</div>}
<div className="mt-1">
<div>Size: {formatBytes(size)}</div>
<div>Public: {counts.public}</div>
{hasEncrypted && <div>🔒 Has encrypted content</div>}
</div>
{decryptResult && (
<div className="mt-1 text-[11px] opacity-80">
<div> Decrypted: {decryptResult.public} public, {decryptResult.private} private</div>
</div>
)}
<div className="opacity-50 mt-1 text-[10px] break-all">ID: {evt.id}</div>
</div>
)
})}
</div>
</div>
)}
</div>
{/* Debug Logs Section */}

View File

@@ -1,5 +1,4 @@
import { RelayPool } from 'applesauce-relay'
import { Helpers } from 'applesauce-core'
import {
AccountWithExtension,
NostrEvent,
@@ -13,56 +12,22 @@ import { collectBookmarksFromEvents } from './bookmarkProcessing.ts'
import { UserSettings } from './settingsService'
import { rebroadcastEvents } from './rebroadcastService'
import { queryEvents } from './dataFetch'
import { KINDS } from '../config/kinds'
// Helper to check if event has encrypted content
const hasEncryptedContent = (evt: NostrEvent): boolean => {
// Check for NIP-44 encrypted content (detected by Helpers)
if (Helpers.hasHiddenContent(evt)) return true
// Check for NIP-04 encrypted content (base64 with ?iv= suffix)
if (evt.content && evt.content.includes('?iv=')) return true
// Check for encrypted tags
if (Helpers.hasHiddenTags(evt) && !Helpers.isHiddenTagsUnlocked(evt)) return true
return false
}
// Helper to deduplicate events by key
const getEventKey = (evt: NostrEvent): string => {
if (evt.kind === 30003 || evt.kind === 30001) {
// Replaceable: kind:pubkey:dtag
const dTag = evt.tags?.find((t: string[]) => t[0] === 'd')?.[1] || ''
return `${evt.kind}:${evt.pubkey}:${dTag}`
} else if (evt.kind === 10003) {
// Simple list: kind:pubkey
return `${evt.kind}:${evt.pubkey}`
}
// Web bookmarks: use event id (no deduplication)
return evt.id
}
import { loadBookmarksStream } from './bookmarkStream'
export const fetchBookmarks = async (
relayPool: RelayPool,
activeAccount: unknown,
accountManager: { getActive: () => unknown },
setBookmarks: (bookmarks: Bookmark[]) => void,
settings?: UserSettings
settings?: UserSettings,
onProgressUpdate?: () => void
) => {
try {
if (!isAccountWithExtension(activeAccount)) {
throw new Error('Invalid account object provided')
}
console.log('[app] 🔍 Fetching bookmark events with streaming')
// Track events with deduplication as they arrive
const eventMap = new Map<string, NostrEvent>()
let processedCount = 0
console.log('[app] Account:', activeAccount.pubkey.slice(0, 8))
// Get signer for auto-decryption
// Get signer for bookmark processing
const maybeAccount = activeAccount as AccountWithExtension
let signerCandidate: unknown = maybeAccount
const hasNip04Prop = (signerCandidate as { nip04?: unknown })?.nip04 !== undefined
@@ -186,58 +151,35 @@ export const fetchBookmarks = async (
setBookmarks([bookmark])
}
// Stream events (just collect, decrypt after)
const rawEvents = await queryEvents(
// Use shared streaming helper for consistent behavior with Debug page
// Progressive updates via callbacks (non-blocking)
const { events: dedupedEvents } = await loadBookmarksStream({
relayPool,
{ kinds: [KINDS.ListSimple, KINDS.ListReplaceable, KINDS.List, KINDS.WebBookmark], authors: [activeAccount.pubkey] },
{
onEvent: (evt) => {
// Deduplicate by key
const key = getEventKey(evt)
const existing = eventMap.get(key)
if (existing && (existing.created_at || 0) >= (evt.created_at || 0)) {
return // Keep existing (it's newer or same)
}
// Add/update event
eventMap.set(key, evt)
processedCount++
console.log(`[app] 📨 Event ${processedCount}: kind=${evt.kind}, id=${evt.id.slice(0, 8)}, hasEncrypted=${hasEncryptedContent(evt)}`)
activeAccount: maybeAccount,
accountManager,
onEvent: () => {
// Signal that an event arrived (for loading indicator updates)
if (onProgressUpdate) {
onProgressUpdate()
}
},
onDecryptComplete: () => {
// Signal that a decrypt completed (for loading indicator updates)
if (onProgressUpdate) {
onProgressUpdate()
}
}
)
})
console.log('[app] 📊 Query complete, raw events fetched:', rawEvents.length, 'events')
// Rebroadcast bookmark events to local/all relays based on settings
await rebroadcastEvents(rawEvents, relayPool, settings)
await rebroadcastEvents(dedupedEvents, relayPool, settings)
const dedupedEvents = Array.from(eventMap.values())
console.log('[app] 📋 After deduplication:', dedupedEvents.length, 'bookmark events')
if (dedupedEvents.length === 0) {
console.log('[app] ⚠️ No bookmark events found')
setBookmarks([]) // Clear bookmarks if none found
setBookmarks([])
return
}
// Auto-decrypt events with encrypted content (batch processing)
const encryptedEvents = dedupedEvents.filter(evt => hasEncryptedContent(evt))
if (encryptedEvents.length > 0) {
console.log('[app] 🔓 Auto-decrypting', encryptedEvents.length, 'encrypted events')
for (const evt of encryptedEvents) {
try {
// Trigger decryption - this unlocks the content for the main collection pass
await collectBookmarksFromEvents([evt], activeAccount, signerCandidate)
console.log('[app] ✅ Auto-decrypted:', evt.id.slice(0, 8))
} catch (error) {
console.error('[app] ❌ Auto-decrypt failed:', evt.id.slice(0, 8), error)
}
}
}
// Final update with all events (now with decrypted content)
console.log('[app] 🔄 Final bookmark processing with', dedupedEvents.length, 'events')
await updateBookmarks(dedupedEvents)

View File

@@ -0,0 +1,171 @@
import { RelayPool } from 'applesauce-relay'
import { Helpers } from 'applesauce-core'
import { NostrEvent } from 'nostr-tools'
import { queryEvents } from './dataFetch'
import { KINDS } from '../config/kinds'
import { collectBookmarksFromEvents } from './bookmarkProcessing'
/**
* Get unique key for event deduplication
* Replaceable events (30001, 30003) use kind:pubkey:dtag
* Simple lists (10003) use kind:pubkey
* Web bookmarks (39701) use event id
*/
export function getEventKey(evt: NostrEvent): string {
if (evt.kind === 30003 || evt.kind === 30001) {
// Replaceable: kind:pubkey:dtag
const dTag = evt.tags?.find((t: string[]) => t[0] === 'd')?.[1] || ''
return `${evt.kind}:${evt.pubkey}:${dTag}`
} else if (evt.kind === 10003) {
// Simple list: kind:pubkey
return `${evt.kind}:${evt.pubkey}`
}
// Web bookmarks: use event id (no deduplication)
return evt.id
}
/**
* Check if event has encrypted content
* Detects NIP-44 (via Helpers), NIP-04 (?iv= suffix), and encrypted tags
*/
export function hasEncryptedContent(evt: NostrEvent): boolean {
// Check for NIP-44 encrypted content (detected by Helpers)
if (Helpers.hasHiddenContent(evt)) return true
// Check for NIP-04 encrypted content (base64 with ?iv= suffix)
if (evt.content && evt.content.includes('?iv=')) return true
// Check for encrypted tags
if (Helpers.hasHiddenTags(evt) && !Helpers.isHiddenTagsUnlocked(evt)) return true
return false
}
interface LoadBookmarksStreamOptions {
relayPool: RelayPool
activeAccount: { pubkey: string; [key: string]: unknown }
accountManager: { getActive: () => unknown }
onEvent?: (event: NostrEvent) => void
onDecryptStart?: (eventId: string) => void
onDecryptComplete?: (eventId: string, success: boolean) => void
}
interface LoadBookmarksStreamResult {
events: NostrEvent[]
decryptedCount: number
}
/**
* Load bookmark events with streaming and non-blocking decryption
* - Streams events via onEvent callback as they arrive
* - Deduplicates by getEventKey
* - Decrypts encrypted events AFTER query completes (non-blocking UI)
* - Trusts EOSE signal to complete
*/
export async function loadBookmarksStream(
options: LoadBookmarksStreamOptions
): Promise<LoadBookmarksStreamResult> {
const {
relayPool,
activeAccount,
accountManager,
onEvent,
onDecryptStart,
onDecryptComplete
} = options
console.log('[app] 🔍 Fetching bookmark events with streaming')
console.log('[app] Account:', activeAccount.pubkey.slice(0, 8))
// Track events with deduplication as they arrive
const eventMap = new Map<string, NostrEvent>()
let processedCount = 0
// Get signer for auto-decryption
const fullAccount = accountManager.getActive() as {
pubkey: string
signer?: unknown
nip04?: unknown
nip44?: unknown
[key: string]: unknown
} | null
const maybeAccount = fullAccount || activeAccount
let signerCandidate: unknown = maybeAccount
const hasNip04Prop = (signerCandidate as { nip04?: unknown })?.nip04 !== undefined
const hasNip44Prop = (signerCandidate as { nip44?: unknown })?.nip44 !== undefined
if (signerCandidate && !hasNip04Prop && !hasNip44Prop && maybeAccount?.signer) {
signerCandidate = maybeAccount.signer
}
// Stream events (just collect, decrypt after)
const rawEvents = await queryEvents(
relayPool,
{ kinds: [KINDS.ListSimple, KINDS.ListReplaceable, KINDS.List, KINDS.WebBookmark], authors: [activeAccount.pubkey] },
{
onEvent: (evt) => {
// Deduplicate by key
const key = getEventKey(evt)
const existing = eventMap.get(key)
if (existing && (existing.created_at || 0) >= (evt.created_at || 0)) {
return // Keep existing (it's newer or same)
}
// Add/update event
eventMap.set(key, evt)
processedCount++
console.log(`[app] 📨 Event ${processedCount}: kind=${evt.kind}, id=${evt.id.slice(0, 8)}, hasEncrypted=${hasEncryptedContent(evt)}`)
// Call optional callback for progressive UI updates
if (onEvent) {
onEvent(evt)
}
}
}
)
console.log('[app] 📊 Query complete, raw events fetched:', rawEvents.length, 'events')
const dedupedEvents = Array.from(eventMap.values())
console.log('[app] 📋 After deduplication:', dedupedEvents.length, 'bookmark events')
if (dedupedEvents.length === 0) {
console.log('[app] ⚠️ No bookmark events found')
return { events: [], decryptedCount: 0 }
}
// Auto-decrypt events with encrypted content (batch processing after EOSE)
const encryptedEvents = dedupedEvents.filter(evt => hasEncryptedContent(evt))
let decryptedCount = 0
if (encryptedEvents.length > 0) {
console.log('[app] 🔓 Auto-decrypting', encryptedEvents.length, 'encrypted events')
for (const evt of encryptedEvents) {
try {
if (onDecryptStart) {
onDecryptStart(evt.id)
}
// Trigger decryption - this unlocks the content for the bookmark collection
await collectBookmarksFromEvents([evt], activeAccount, signerCandidate)
decryptedCount++
console.log('[app] ✅ Auto-decrypted:', evt.id.slice(0, 8))
if (onDecryptComplete) {
onDecryptComplete(evt.id, true)
}
} catch (error) {
console.error('[app] ❌ Auto-decrypt failed:', evt.id.slice(0, 8), error)
if (onDecryptComplete) {
onDecryptComplete(evt.id, false)
}
}
}
}
console.log('[app] ✅ Bookmark streaming complete:', dedupedEvents.length, 'events,', decryptedCount, 'decrypted')
return { events: dedupedEvents, decryptedCount }
}