use new nostr-sdk with hints (full outbox) support for improved event fetching that hopefully works.

data.go

@@ -12,6 +12,7 @@ import (
 	"github.com/nbd-wtf/go-nostr/nip52"
 	"github.com/nbd-wtf/go-nostr/nip53"
 	"github.com/nbd-wtf/go-nostr/nip94"
+	sdk "github.com/nbd-wtf/nostr-sdk"
 )
 
 type Data struct {
@@ -43,7 +44,7 @@ type Data struct {
 
 func grabData(ctx context.Context, code string, isProfileSitemap bool) (*Data, error) {
 	// code can be a nevent, nprofile, npub or nip05 identifier, in which case we try to fetch the associated event
-	event, relays, err := getEvent(ctx, code, nil)
+	event, relays, err := getEvent(ctx, code)
 	if err != nil {
 		log.Warn().Err(err).Str("code", code).Msg("failed to fetch event for code")
 		return nil, fmt.Errorf("error fetching event: %w", err)
@@ -52,7 +53,7 @@ func grabData(ctx context.Context, code string, isProfileSitemap bool) (*Data, e
 	relaysForNip19 := make([]string, 0, 3)
 	c := 0
 	for _, relayUrl := range relays {
-		if isntRealRelay(relayUrl) {
+		if sdk.IsVirtualRelay(relayUrl) {
 			continue
 		}
 
@@ -67,8 +68,8 @@ func grabData(ctx context.Context, code string, isProfileSitemap bool) (*Data, e
 	}
 
 	data.authorRelaysPretty = make([]string, 0, len(relays))
-	for _, url := range relaysForPubkey(ctx, event.PubKey) {
+	for _, url := range sys.FetchOutboxRelays(ctx, event.PubKey, 3) {
-		if isntRealRelay(url) {
+		if sdk.IsVirtualRelay(url) {
 			continue
 		}
 		for _, excluded := range relayConfig.ExcludedRelays {
@@ -76,7 +77,7 @@ func grabData(ctx context.Context, code string, isProfileSitemap bool) (*Data, e
 				continue
 			}
 		}
-		data.authorRelaysPretty = append(data.authorRelaysPretty, trimProtocol(url))
+		data.authorRelaysPretty = append(data.authorRelaysPretty, trimProtocolAndEndingSlash(url))
 	}
 	data.authorRelaysPretty = unique(data.authorRelaysPretty)
 
@@ -104,9 +105,6 @@ func grabData(ctx context.Context, code string, isProfileSitemap bool) (*Data, e
 		for i, levt := range lastNotes {
 			data.renderableLastNotes[i] = NewEnhancedEvent(ctx, levt, []string{})
 		}
-		if err != nil {
-			return nil, err
-		}
 	case 1, 7, 30023, 30024:
 		data.templateId = Note
 		data.content = event.Content

go.mod

@@ -1,6 +1,8 @@
 module github.com/fiatjaf/njump
 
-go 1.21.4
+go 1.22
+
+toolchain go1.22.4
 
 require (
 	github.com/Kagami/go-avif v0.1.0
@@ -8,9 +10,8 @@ require (
 	github.com/a-h/templ v0.2.747
 	github.com/bytesparadise/libasciidoc v0.8.0
 	github.com/dgraph-io/badger/v4 v4.2.0
-	github.com/fiatjaf/eventstore v0.4.2
+	github.com/fiatjaf/eventstore v0.7.1
 	github.com/fiatjaf/khatru v0.4.2
-	github.com/fiatjaf/set v0.0.4
 	github.com/fogleman/gg v1.3.0
 	github.com/go-text/typesetting v0.0.0-20231221124458-48cc05a56658
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
@@ -18,15 +19,15 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/microcosm-cc/bluemonday v1.0.24
 	github.com/nbd-wtf/emoji v0.0.3
-	github.com/nbd-wtf/go-nostr v0.33.0
+	github.com/nbd-wtf/go-nostr v0.34.4
-	github.com/nbd-wtf/nostr-sdk v0.2.3
+	github.com/nbd-wtf/nostr-sdk v0.4.0
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/pelletier/go-toml v1.9.5
 	github.com/pemistahl/lingua-go v1.4.0
 	github.com/rs/cors v1.11.0
 	github.com/rs/zerolog v1.29.1
 	github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/texttheater/golang-levenshtein v1.0.1
 	github.com/tylermmorton/tmpl v0.0.0-20231025031313-5552ee818c6d
 	golang.org/x/image v0.17.0
@@ -64,7 +65,7 @@ require (
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/graph-gophers/dataloader/v7 v7.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.17.3 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect

go.sum

@@ -85,14 +85,12 @@ github.com/fasthttp/websocket v1.5.7 h1:0a6o2OfeATvtGgoMKleURhLT6JqWPg7fYfWnH4KH
 github.com/fasthttp/websocket v1.5.7/go.mod h1:bC4fxSono9czeXHQUVKxsC0sNjbm7lPJR04GDFqClfU=
 github.com/felixge/fgtrace v0.1.0 h1:cuMLI5NoBg/9IxIVmJzsxA3Aoz5eIKRca6WE1U2C1zc=
 github.com/felixge/fgtrace v0.1.0/go.mod h1:VYPh/jE5zczuRiQge0AtcpNmcLhV/epE/wpfVYQALlU=
-github.com/fiatjaf/eventstore v0.4.2 h1:GGg/Rtsa8fJhLgYDaJioYUrpqZ6OhmaqY1kwMiweY3g=
+github.com/fiatjaf/eventstore v0.7.1 h1:5f2yvEtYvsvMBNttysmXhSSum5M1qwvPzjEQ/BFue7Q=
-github.com/fiatjaf/eventstore v0.4.2/go.mod h1:Ai1fEKP2eRo/mMyvVXcXItxFrOI0gYOmO9IMDeEVde4=
+github.com/fiatjaf/eventstore v0.7.1/go.mod h1:ek/yWbanKVG767fK51Q3+6Mvi5oEHYSsdPym40nZexw=
 github.com/fiatjaf/generic-ristretto v0.0.1 h1:LUJSU87X/QWFsBXTwnH3moFe4N8AjUxT+Rfa0+bo6YM=
 github.com/fiatjaf/generic-ristretto v0.0.1/go.mod h1:cvV6ANHDA/GrfzVrig7N7i6l8CWnkVZvtQ2/wk9DPVE=
 github.com/fiatjaf/khatru v0.4.2 h1:MpGn6HAWu9v7JFRd3l/7Jfx2hCH+ZM2A8rShAvvtubQ=
 github.com/fiatjaf/khatru v0.4.2/go.mod h1:cfoaJMzrji7bjnB+Xn30I5KcJdr5ocJzhhdmVp7D4K4=
-github.com/fiatjaf/set v0.0.4 h1:1+vprHBRtVXUNHxPBFKG0ZpdU5Q793cJNUKF3i//x/Q=
-github.com/fiatjaf/set v0.0.4/go.mod h1:hdSwBrO+CwMEbYQAMaHtsib30KQLDtVjbX/1OgDK3tY=
 github.com/fiatjaf/typesetting v0.0.0-20231228183257-7c3f6f5a0ccc h1:8QpOKCVr8jpuvpmLCZUnsZ50faseCym2r6f5crpODKM=
 github.com/fiatjaf/typesetting v0.0.0-20231228183257-7c3f6f5a0ccc/go.mod h1:d22AnmeKq/on0HNv73UFriMKc4Ez6EqZAofLhAzpSzI=
 github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8=
@@ -169,8 +167,8 @@ github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
-github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
-github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -190,10 +188,10 @@ github.com/mna/pigeon v1.1.0 h1:EjlvVbkGnNGemf8OrjeJX0nH8orujY/HkJgzJtd7kxc=
 github.com/mna/pigeon v1.1.0/go.mod h1:rkFeDZ0gc+YbnrXPw0q2RlI0QRuKBBPu67fgYIyGRNg=
 github.com/nbd-wtf/emoji v0.0.3 h1:YtkT7MVPXvqU1SQjvC/CShlWexnREzqNCxmhUnL00CA=
 github.com/nbd-wtf/emoji v0.0.3/go.mod h1:tS6D9iI34qwBmWc5g8X7tVDkWXulqbTJRsvsM6QsS88=
-github.com/nbd-wtf/go-nostr v0.33.0 h1:6hZx25JAgwEOY49gCjrVZYFno7Z8L7HIwF6IMDGPjaU=
+github.com/nbd-wtf/go-nostr v0.34.4 h1:bWjUnD5B6vdK8o+Un2EKAJ8cA2o+myQKzdZa/HxqTMk=
-github.com/nbd-wtf/go-nostr v0.33.0/go.mod h1:NZQkxl96ggbO8rvDpVjcsojJqKTPwqhP4i82O7K5DJs=
+github.com/nbd-wtf/go-nostr v0.34.4/go.mod h1:NZQkxl96ggbO8rvDpVjcsojJqKTPwqhP4i82O7K5DJs=
-github.com/nbd-wtf/nostr-sdk v0.2.3 h1:wQrr92VwYhOl+r1Cg3hUvggQT4S2wcTRRFHMD3znbe4=
+github.com/nbd-wtf/nostr-sdk v0.4.0 h1:ccZ5gywIE5LrFPj/DY4D7N/3NHYZBEBnT7VW6V1ySNo=
-github.com/nbd-wtf/nostr-sdk v0.2.3/go.mod h1:AZZW6QzMk3Tc14fXRPktHFEEAAywLZ/TZf+Wdkr2ksI=
+github.com/nbd-wtf/nostr-sdk v0.4.0/go.mod h1:9xYfyBO2pSoOGGel770n/sgcj1x6wyFcX4tbb0eiOGc=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
@@ -251,8 +249,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
 github.com/texttheater/golang-levenshtein v1.0.1 h1:+cRNoVrfiwufQPhoMzB6N0Yf/Mqajr6t1lOv8GyGE2U=
 github.com/texttheater/golang-levenshtein v1.0.1/go.mod h1:PYAKrbF5sAiq9wd+H82hs7gNaen0CplQ9uvm6+enD/8=

hints.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"time"
+
+	"github.com/nbd-wtf/nostr-sdk/hints/memory"
+)
+
+// save these things to a file so we can reload them later
+func outboxHintsFileLoaderSaver(ctx context.Context) {
+	if file, err := os.Open(s.HintsMemoryDumpPath); err == nil {
+		hdb := memory.NewHintDB()
+		if err := json.NewDecoder(file).Decode(&hdb); err == nil {
+			sys.Hints = hdb
+		}
+		file.Close()
+	}
+
+	const tmp = "/tmp/njump-outbox-hints-tmp.json"
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(time.Minute * 5):
+		}
+
+		hdb := sys.Hints.(*memory.HintDB)
+		file, err := os.Create(tmp)
+		if err != nil {
+			log.Error().Err(err).Str("path", tmp).Msg("failed to create outbox hints file")
+			time.Sleep(time.Hour)
+			continue
+		}
+		file.Close()
+		json.NewEncoder(file).Encode(hdb)
+		if err := os.Rename(tmp, s.HintsMemoryDumpPath); err != nil {
+			log.Error().Err(err).Str("from", tmp).Str("to", s.HintsMemoryDumpPath).Msg("failed to move outbox hints file")
+			time.Sleep(time.Hour)
+			continue
+		}
+	}
+}

image_utils.go

@@ -231,7 +231,7 @@ func quotesAsBlockPrefixedText(ctx context.Context, lines []string) []string {
 			submatch := nostrNoteNeventMatcher.FindStringSubmatch(matchText)
 			nip19 := submatch[0][6:]
 
-			event, _, err := getEvent(ctx, nip19, nil)
+			event, _, err := getEvent(ctx, nip19)
 			if err != nil {
 				// error case concat this to previous block
 				blocks[b] += matchText
@@ -884,3 +884,14 @@ func fixed266ToFloat(i fixed.Int26_6) float32 {
 func floatToFixed266(f float32) fixed.Int26_6 {
 	return fixed.Int26_6(int(float64(f) * 64))
 }
+
+// clamp ensures val is in the inclusive range [low,high].
+func clamp(val, low, high int) int {
+	if val < low {
+		return low
+	}
+	if val > high {
+		return high
+	}
+	return val
+}

main.go

@@ -20,14 +20,15 @@ import (
 )
 
 type Settings struct {
-	Port              string   `envconfig:"PORT" default:"2999"`
+	Port                string   `envconfig:"PORT" default:"2999"`
-	Domain            string   `envconfig:"DOMAIN" default:"njump.me"`
+	Domain              string   `envconfig:"DOMAIN" default:"njump.me"`
-	DiskCachePath     string   `envconfig:"DISK_CACHE_PATH" default:"/tmp/njump-internal"`
+	DiskCachePath       string   `envconfig:"DISK_CACHE_PATH" default:"/tmp/njump-internal"`
-	EventStorePath    string   `envconfig:"EVENT_STORE_PATH" default:"/tmp/njump-db"`
+	EventStorePath      string   `envconfig:"EVENT_STORE_PATH" default:"/tmp/njump-db"`
-	TailwindDebug     bool     `envconfig:"TAILWIND_DEBUG"`
+	HintsMemoryDumpPath string   `envconfig:"HINTS_SAVE_PATH" default:"/tmp/njump-hints.json"`
-	SkipLanguageModel bool     `envconfig:"SKIP_LANGUAGE_MODEL"`
+	TailwindDebug       bool     `envconfig:"TAILWIND_DEBUG"`
-	RelayConfigPath   string   `envconfig:"RELAY_CONFIG_PATH"`
+	SkipLanguageModel   bool     `envconfig:"SKIP_LANGUAGE_MODEL"`
-	TrustedPubKeys    []string `envconfig:"TRUSTED_PUBKEYS"`
+	RelayConfigPath     string   `envconfig:"RELAY_CONFIG_PATH"`
+	TrustedPubKeys      []string `envconfig:"TRUSTED_PUBKEYS"`
 }
 
 //go:embed static/*
@@ -65,9 +66,11 @@ func main() {
 			log.Fatal().Err(err).Msgf("failed to load %q", s.RelayConfigPath)
 			return
 		}
-		if !relayConfig.Valid() {
+		if len(relayConfig.Everything) > 0 {
-			log.Fatal().Err(err).Msgf("invalid relay config file %q", s.RelayConfigPath)
+			sys.FallbackRelays = relayConfig.Everything
-			return
+		}
+		if len(relayConfig.Profiles) > 0 {
+			sys.MetadataRelays = relayConfig.Profiles
 		}
 	}
 
@@ -108,6 +111,7 @@ func main() {
 	defer cancel()
 	go updateArchives(ctx)
 	go deleteOldCachedEvents(ctx)
+	go outboxHintsFileLoaderSaver(ctx)
 
 	// expose our internal cache as a relay (mostly for debugging purposes)
 	relay := khatru.NewRelay()

nostr.go

@@ -7,11 +7,11 @@ import (
 	"time"
 
 	"github.com/fiatjaf/eventstore"
-	"github.com/fiatjaf/set"
 	"github.com/nbd-wtf/go-nostr"
 	"github.com/nbd-wtf/go-nostr/nip05"
 	"github.com/nbd-wtf/go-nostr/nip19"
 	sdk "github.com/nbd-wtf/nostr-sdk"
+	cache_memory "github.com/nbd-wtf/nostr-sdk/cache/memory"
 )
 
 type RelayConfig struct {
@@ -21,39 +21,18 @@ type RelayConfig struct {
 	ExcludedRelays []string `json:"excludeRelays"`
 }
 
-func (r *RelayConfig) Valid() bool {
-	if len(r.Everything) == 0 || len(r.Profiles) == 0 || len(r.JustIds) == 0 || len(r.ExcludedRelays) == 0 {
-		return false
-	}
-	return true
-}
-
 var (
-	pool   = nostr.NewSimplePool(context.Background())
+	metadataCache  = cache_memory.New32[sdk.ProfileMetadata](1000)
-	sys    = sdk.System(sdk.WithPool(pool))
+	relayListCache = cache_memory.New32[sdk.RelayList](5000)
+	sys            = sdk.NewSystem(
+		sdk.WithMetadataCache(metadataCache),
+		sdk.WithRelayListCache(relayListCache),
+	)
 	serial int
 
 	relayConfig = RelayConfig{
-		Everything: []string{
+		Everything: nil, // use the defaults from nostr-sdk
-			"wss://nostr-pub.wellorder.net",
+		Profiles:   nil, // use the defaults from nostr-sdk
-			"wss://relay.damus.io",
-			"wss://relay.nostr.bg",
-			"wss://nostr.wine",
-			"wss://nos.lol",
-			"wss://nostr.mom",
-			"wss://nostr.land",
-			"wss://relay.snort.social",
-			"wss://offchain.pub",
-			"wss://relay.primal.net",
-			"wss://relay.nostr.band",
-			"wss://public.relaying.io",
-		},
-		Profiles: []string{
-			"wss://purplepag.es",
-			"wss://relay.noswhere.com",
-			"wss://relay.nos.social",
-			"wss://relay.snort.social",
-		},
 		JustIds: []string{
 			"wss://cache2.primal.net/v1",
 			"wss://relay.noswhere.com",
@@ -76,14 +55,11 @@ type CachedEvent struct {
 	Relays []string     `json:"r"`
 }
 
-func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Event, []string, error) {
+func getEvent(ctx context.Context, code string) (*nostr.Event, []string, error) {
 	wdb := eventstore.RelayWrapper{Store: db}
 
-	withRelays := false
+	// this is for deciding what relays will go on nevent and nprofile later
-	if len(relayHints) > 0 {
+	priorityRelays := make(map[string]int)
-		withRelays = true
-	}
-	priorityRelays := set.NewSliceSet(relayHints...)
 
 	prefix, data, err := nip19.Decode(code)
 	if err != nil {
@@ -94,51 +70,51 @@ func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Eve
 		data = *pp
 	}
 
-	var author string
+	author := ""
+	authorRelaysPosition := 0
 
 	var filter nostr.Filter
-	relays := make([]string, 0, 25)
+	relays := make([]string, 0, 10)
-	relays = append(relays, relayHints...)
 
 	switch v := data.(type) {
 	case nostr.ProfilePointer:
 		author = v.PublicKey
 		filter.Authors = []string{v.PublicKey}
 		filter.Kinds = []int{0}
-		relays = append(relays, relayConfig.Profiles...)
 		relays = append(relays, v.Relays...)
-		priorityRelays.Add(v.Relays...)
+		authorRelaysPosition = len(v.Relays) // ensure author relays are checked after hinted relays
-		withRelays = true
+		relays = append(relays, sys.MetadataRelays...)
+		for _, r := range v.Relays {
+			priorityRelays[r] = 2
+		}
 	case nostr.EventPointer:
 		author = v.Author
 		filter.IDs = []string{v.ID}
 		relays = append(relays, v.Relays...)
 		relays = append(relays, relayConfig.JustIds...)
-		priorityRelays.Add(v.Relays...)
+		authorRelaysPosition = len(v.Relays) // ensure author relays are checked after hinted relays
-		withRelays = true
+		for _, r := range v.Relays {
+			priorityRelays[r] = 2
+		}
 	case nostr.EntityPointer:
 		author = v.PublicKey
-		filter.Authors = []string{v.PublicKey}
 		filter.Tags = nostr.TagMap{
 			"d": []string{v.Identifier},
 		}
 		if v.Kind != 0 {
 			filter.Kinds = append(filter.Kinds, v.Kind)
 		}
-		relays = append(relays, getRandomRelay(), getRandomRelay())
 		relays = append(relays, v.Relays...)
-		priorityRelays.Add(v.Relays...)
+		authorRelaysPosition = len(v.Relays) // ensure author relays are checked after hinted relays
-		withRelays = true
 	case string:
 		if prefix == "note" {
 			filter.IDs = []string{v}
-			relays = append(relays, getRandomRelay())
 			relays = append(relays, relayConfig.JustIds...)
 		} else if prefix == "npub" {
 			author = v
 			filter.Authors = []string{v}
 			filter.Kinds = []int{0}
-			relays = append(relays, relayConfig.Profiles...)
+			relays = append(relays, sys.MetadataRelays...)
 		}
 	}
 
@@ -149,14 +125,17 @@ func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Eve
 		return evt, getRelaysForEvent(evt.ID), nil
 	}
 
-	// otherwise fetch from external relays
 	if author != "" {
 		// fetch relays for author
-		authorRelays := relaysForPubkey(ctx, author, relays...)
+		authorRelays := sys.FetchOutboxRelays(ctx, author, 3)
-		if len(authorRelays) > 3 {
+		relays = slices.Insert(relays, authorRelaysPosition, authorRelays...)
-			authorRelays = authorRelays[:3]
+		for _, r := range authorRelays {
+			priorityRelays[r] = 1
 		}
-		relays = append(relays, authorRelays...)
+	}
+
+	for len(relays) < 5 {
+		relays = append(relays, getRandomRelay())
 	}
 
 	relays = unique(relays)
@@ -166,30 +145,25 @@ func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Eve
 	// actually fetch the event here
 	var result *nostr.Event
 	var successRelays []string = nil
-	if withRelays {
-		successRelays = make([]string, 0, len(relays))
-		countdown := 7.5
-		go func() {
-			for {
-				time.Sleep(500 * time.Millisecond)
-				if countdown <= 0 {
-					cancel()
-					break
-				}
-				countdown -= 0.5
-			}
-		}()
 
-		for ie := range pool.SubManyEoseNonUnique(ctx, relays, nostr.Filters{filter}) {
+	// keep track of where we have actually found the event so we can show that
-			successRelays = append(successRelays, ie.Relay.URL)
+	successRelays = make([]string, 0, len(relays))
-			result = ie.Event
+	countdown := 7.5
-			countdown = min(countdown, 1)
+	go func() {
-		}
+		for {
-	} else {
+			time.Sleep(500 * time.Millisecond)
-		ie := pool.QuerySingle(ctx, relays, filter)
+			if countdown <= 0 {
-		if ie != nil {
+				cancel()
-			result = ie.Event
+				break
+			}
+			countdown -= 0.5
 		}
+	}()
+
+	for ie := range sys.Pool.SubManyEoseNonUnique(ctx, relays, nostr.Filters{filter}) {
+		successRelays = append(successRelays, ie.Relay.URL)
+		result = ie.Event
+		countdown = min(countdown, 1)
 	}
 
 	if result == nil {
@@ -203,12 +177,9 @@ func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Eve
 	allRelays := attachRelaysToEvent(result.ID, successRelays...)
 	// put priority relays first so they get used in nevent and nprofile
 	slices.SortFunc(allRelays, func(a, b string) int {
-		if priorityRelays.Has(a) && !priorityRelays.Has(b) {
+		vpa, _ := priorityRelays[a]
-			return -1
+		vpb, _ := priorityRelays[b]
-		} else if priorityRelays.Has(b) && !priorityRelays.Has(a) {
+		return vpb - vpa
-			return 1
-		}
-		return 0
 	})
 	// keep track of what we have to delete later
 	scheduleEventExpiration(result.ID, time.Hour*24*7)
@@ -243,11 +214,11 @@ func authorLastNotes(ctx context.Context, pubkey string, isSitemap bool) []*nost
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 		defer cancel()
 
-		relays := limitAt(relaysForPubkey(ctx, pubkey), 5)
+		relays := sys.FetchOutboxRelays(ctx, pubkey, 5)
 		for len(relays) < 4 {
 			relays = unique(append(relays, getRandomRelay()))
 		}
-		ch := pool.SubManyEose(ctx, relays, nostr.Filters{filter})
+		ch := sys.Pool.SubManyEose(ctx, relays, nostr.Filters{filter})
 	out:
 		for {
 			select {
@@ -290,7 +261,7 @@ func relayLastNotes(ctx context.Context, relayUrl string, isSitemap bool) []*nos
 	ctx, cancel := context.WithTimeout(ctx, time.Second*4)
 	defer cancel()
 
-	if relay, err := pool.EnsureRelay(relayUrl); err == nil {
+	if relay, err := sys.Pool.EnsureRelay(relayUrl); err == nil {
 		lastNotes, _ = relay.QuerySync(ctx, nostr.Filter{
 			Kinds: []int{1},
 			Limit: limit,
@@ -304,31 +275,18 @@ func relayLastNotes(ctx context.Context, relayUrl string, isSitemap bool) []*nos
 	return lastNotes
 }
 
-func relaysForPubkey(ctx context.Context, pubkey string, extraRelays ...string) []string {
+func contactsForPubkey(ctx context.Context, pubkey string) []string {
-	pubkeyRelays := make([]string, 0, 12)
-	if ok := cache.GetJSON("io:"+pubkey, &pubkeyRelays); !ok {
-		ctx, cancel := context.WithTimeout(ctx, time.Millisecond*1500)
-		defer cancel()
-		pubkeyRelays = sys.FetchOutboxRelays(ctx, pubkey)
-		if len(pubkeyRelays) > 0 {
-			cache.SetJSONWithTTL("io:"+pubkey, pubkeyRelays, time.Hour*24*7)
-		}
-	}
-	return unique(pubkeyRelays)
-}
-
-func contactsForPubkey(ctx context.Context, pubkey string, extraRelays ...string) []string {
 	pubkeyContacts := make([]string, 0, 300)
 	relays := make([]string, 0, 12)
 	if ok := cache.GetJSON("cc:"+pubkey, &pubkeyContacts); !ok {
 		log.Debug().Msgf("searching contacts for %s", pubkey)
 		ctx, cancel := context.WithTimeout(ctx, time.Second*3)
 
-		pubkeyRelays := relaysForPubkey(ctx, pubkey, relays...)
+		pubkeyRelays := sys.FetchOutboxRelays(ctx, pubkey, 3)
 		relays = append(relays, pubkeyRelays...)
-		relays = append(relays, relayConfig.Profiles...)
+		relays = append(relays, sys.MetadataRelays...)
 
-		ch := pool.SubManyEose(ctx, relays, nostr.Filters{
+		ch := sys.Pool.SubManyEose(ctx, relays, nostr.Filters{
 			{
 				Kinds:   []int{3},
 				Authors: []string{pubkey},

render_event.go

@@ -16,6 +16,7 @@ import (
 	"github.com/nbd-wtf/go-nostr"
 	"github.com/nbd-wtf/go-nostr/nip05"
 	"github.com/nbd-wtf/go-nostr/nip19"
+	"github.com/nbd-wtf/nostr-sdk/hints/memory"
 	"github.com/pelletier/go-toml"
 )
 
@@ -29,6 +30,8 @@ func isValidShortcode(s string) bool {
 }
 
 func renderEvent(w http.ResponseWriter, r *http.Request) {
+	sys.Hints.(*memory.HintDB).PrintScores()
+
 	code := r.URL.Path[1:] // hopefully a nip19 code
 
 	// it's the homepage
@@ -180,7 +183,7 @@ func renderEvent(w http.ResponseWriter, r *http.Request) {
 	if len(data.event.relays) > 0 {
 		relays := make([]string, len(data.event.relays))
 		for i, r := range data.event.relays {
-			relays[i] = trimProtocol(r)
+			relays[i] = trimProtocolAndEndingSlash(r)
 		}
 		seenOnRelays = fmt.Sprintf("seen on %s", strings.Join(relays, ", "))
 	}
@@ -209,7 +212,7 @@ func renderEvent(w http.ResponseWriter, r *http.Request) {
 			}
 		} else {
 			// otherwise replace npub/nprofiles with names and trim length
-			description = replaceUserReferencesWithNames(r.Context(), []string{data.event.Content}, "", "")[0]
+			description = replaceUserReferencesWithNames(r.Context(), []string{data.event.Content}, "")[0]
 			if len(description) > 240 {
 				description = description[:240]
 			}
@@ -221,7 +224,7 @@ func renderEvent(w http.ResponseWriter, r *http.Request) {
 		strings.TrimSpace(
 			strings.Replace(
 				strings.Replace(
-					replaceUserReferencesWithNames(r.Context(), []string{data.event.Content}, "", "")[0],
+					replaceUserReferencesWithNames(r.Context(), []string{data.event.Content}, "")[0],
 					"\r\n", " ", -1),
 				"\n", " ", -1,
 			),
@@ -507,7 +510,7 @@ func renderEvent(w http.ResponseWriter, r *http.Request) {
 		})
 
 	case WikiEvent:
-		var PublishedAt = data.Kind30818Metadata.PublishedAt.Format("02 Jan 2006")
+		PublishedAt := data.Kind30818Metadata.PublishedAt.Format("02 Jan 2006")
 		npub, _ := nip19.EncodePublicKey(data.event.PubKey)
 
 		component = wikiEventTemplate(WikiPageParams{

render_relay.go

@@ -12,7 +12,7 @@ func renderRelayPage(w http.ResponseWriter, r *http.Request) {
 	hostname := r.URL.Path[3:]
 
 	if strings.HasPrefix(hostname, "wss:/") || strings.HasPrefix(hostname, "ws:/") {
-		hostname = trimProtocol(hostname)
+		hostname = trimProtocolAndEndingSlash(hostname)
 		http.Redirect(w, r, "/r/"+hostname, http.StatusFound)
 		return
 	}

utils.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"math/rand"
@@ -183,7 +182,7 @@ func attachRelaysToEvent(eventId string, relays ...string) []string {
 	// cleanup
 	filtered := make([]string, 0, len(relays))
 	for _, relay := range relays {
-		if isntRealRelay(relay) {
+		if sdk.IsVirtualRelay(relay) {
 			continue
 		}
 		filtered = append(filtered, relay)
@@ -265,7 +264,7 @@ func shortenNostrURLs(input string) string {
 }
 
 func getNameFromNip19(ctx context.Context, nip19 string) (string, bool) {
-	author, _, err := getEvent(ctx, nip19, nil)
+	author, _, err := getEvent(ctx, nip19)
 	if err != nil {
 		return nip19, false
 	}
@@ -281,7 +280,7 @@ func getNameFromNip19(ctx context.Context, nip19 string) (string, bool) {
 
 // replaces an npub/nprofile with the name of the author, if possible
 // meant to be used when plaintext is expected, not formatted HTML
-func replaceUserReferencesWithNames(ctx context.Context, input []string, prefix, suffix string) []string {
+func replaceUserReferencesWithNames(ctx context.Context, input []string, prefix string) []string {
 	// Match and replace npup1 or nprofile1
 	ctx, cancel := context.WithTimeout(ctx, time.Second*3)
 	defer cancel()
@@ -309,7 +308,7 @@ func renderQuotesAsHTML(ctx context.Context, input string, usingTelegramInstantV
 		submatch := nostrNoteNeventMatcher.FindStringSubmatch(match)
 		nip19 := submatch[1]
 
-		event, _, err := getEvent(ctx, nip19, nil)
+		event, _, err := getEvent(ctx, nip19)
 		if err != nil {
 			log.Warn().Str("nip19", nip19).Msg("failed to get nip19")
 			return nip19
@@ -402,11 +401,12 @@ func unique(strSlice []string) []string {
 	return strSlice[:j+1]
 }
 
-func trimProtocol(relay string) string {
+func trimProtocolAndEndingSlash(relay string) string {
 	relay = strings.TrimPrefix(relay, "wss://")
 	relay = strings.TrimPrefix(relay, "ws://")
-	relay = strings.TrimPrefix(relay, "wss:/") // Some browsers replace upfront '//' with '/'
+	relay = strings.TrimPrefix(relay, "wss:/") // some browsers replace upfront '//' with '/'
-	relay = strings.TrimPrefix(relay, "ws:/")  // Some browsers replace upfront '//' with '/'
+	relay = strings.TrimPrefix(relay, "ws:/")  // some browsers replace upfront '//' with '/'
+	relay = strings.TrimSuffix(relay, "/")
 	return relay
 }
 
@@ -438,23 +438,10 @@ func humanDate(createdAt nostr.Timestamp) string {
 
 func getRandomRelay() string {
 	if serial == 0 {
-		serial = rand.Intn(len(relayConfig.Everything))
+		serial = rand.Intn(len(sys.FallbackRelays))
 	}
-	serial = (serial + 1) % len(relayConfig.Everything)
+	serial = (serial + 1) % len(sys.FallbackRelays)
-	return relayConfig.Everything[serial]
+	return sys.FallbackRelays[serial]
-}
-
-func isntRealRelay(url string) bool {
-	if len(url) < 6 {
-		// this is just invalid
-		return true
-	}
-
-	// if there is a "/" after the initial "wss://" part that means this is probably a "virtual relay"
-	// like wss://feeds.nostr.band/topic or wss://filter.nostr.wine/pubkey or wss://cache2.primal.net/v1
-	// and should not be used in computing outbox model relay recommendations
-	substr := []byte(url[6:])
-	return bytes.IndexByte(substr, '/') != -1
 }
 
 func maxIndex(slice []int) int {
@@ -469,17 +456,6 @@ func maxIndex(slice []int) int {
 	return maxIndex
 }
 
-// clamp ensures val is in the inclusive range [low,high].
-func clamp(val, low, high int) int {
-	if val < low {
-		return low
-	}
-	if val > high {
-		return high
-	}
-	return val
-}
-
 func getUTCOffset(loc *time.Location) string {
 	// Get the offset from UTC
 	_, offset := time.Now().In(loc).Zone()