From eabec9b59045236e850a4c36163912a282f49202 Mon Sep 17 00:00:00 2001 From: Daniele Tonon Date: Tue, 18 Jul 2023 01:34:33 +0200 Subject: [PATCH 1/8] Add npubs archive Add more npubs --- cache.go | 33 ++++++++++++++++++++ main.go | 18 +++++++++++ nostr.go | 46 +++++++++++++++++++++++++++- render_profiles_archive.go | 52 ++++++++++++++++++++++++++++++++ templates/archive.html | 62 ++++++++++++++++++++++++++++++++++++++ utils.go | 21 +++++++++++++ 6 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 render_profiles_archive.go create mode 100644 templates/archive.html diff --git a/cache.go b/cache.go index e308468..a60a4f9 100644 --- a/cache.go +++ b/cache.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "strings" "time" "github.com/dgraph-io/badger" @@ -59,6 +60,38 @@ func (c *Cache) Get(key string) ([]byte, bool) { return val, true } +func (c *Cache) GetPaginatedkeys(prefix string, page int, size int) []string { + keys := []string{} + err := c.DB.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + it := txn.NewIterator(opts) + defer it.Close() + start := (page-1)*size + 1 + index := 1 + for it.Seek([]byte(prefix)); it.ValidForPrefix([]byte(prefix)); it.Next() { + if index < start { + index++ + continue + } + if index > start+size-1 { + break + } + item := it.Item() + k := item.Key() + keys = append(keys, strings.TrimPrefix(string(k), prefix+":")) + index++ + } + return nil + }) + + if err != nil { + log.Fatal().Err(err).Msg("") + } + + return keys +} + func (c *Cache) GetJSON(key string, recv any) bool { b, ok := c.Get(key) if !ok { diff --git a/main.go b/main.go index a798b74..1e597a8 100644 --- a/main.go +++ b/main.go @@ -1,11 +1,13 @@ package main import ( + "context" "embed" "html" "net/http" "os" "text/template" + "time" "github.com/kelseyhightower/envconfig" "github.com/rs/zerolog" @@ -30,6 +32,14 @@ var ( log = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: os.Stdout}).With().Timestamp().Logger() ) +func updateArchives(ctx context.Context) { + for { + loadNpubsArchive(ctx) + // Wait for 24 hours before executing the function again + time.Sleep(24 * time.Hour) + } +} + func main() { err := envconfig.Process("", &s) if err != nil { @@ -39,6 +49,10 @@ func main() { // initialize disk cache defer cache.initialize()() + // initialize the function to update the npubs/relays archive + ctx := context.Background() + go updateArchives(ctx) + // initialize templates // use a mapping to expressly link the templates and share them between more kinds/types templateMapping["profile"] = "profile.html" @@ -65,10 +79,14 @@ func main() { http.HandleFunc("/njump/image/", generate) http.HandleFunc("/njump/proxy/", proxy) http.Handle("/njump/static/", http.StripPrefix("/njump/", http.FileServer(http.FS(static)))) + http.HandleFunc("/npubs-archive/", renderProfilesArchive) http.HandleFunc("/", render) log.Print("listening at http://0.0.0.0:" + s.Port) if err := http.ListenAndServe("0.0.0.0:"+s.Port, nil); err != nil { log.Fatal().Err(err).Msg("") } + + select {} + } diff --git a/nostr.go b/nostr.go index 03c0a9e..302bc0b 100644 --- a/nostr.go +++ b/nostr.go @@ -34,6 +34,13 @@ var ( profiles = []string{ "wss://purplepag.es", } + + trustedPubKeys = []string{ + "7bdef7be22dd8e59f4600e044aa53a1cf975a9dc7d27df5833bc77db784a5805", // dtonon + "3bf0c63fcb93463407af97a5e5ee64fa883d107ef9e558472c4eb9aaaefa459d", // fiatjaf + "97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322", // hodlbod + "ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49", // Michael Dilger + } ) func getRelay() string { @@ -159,7 +166,7 @@ func getLastNotes(ctx context.Context, code string, limit int) []*nostr.Event { for event := range events { lastNotes = nostr.InsertEventIntoDescendingList(lastNotes, event) } - + return lastNotes } @@ -180,3 +187,40 @@ func relaysForPubkey(ctx context.Context, pubkey string, extraRelays ...string) pubkeyRelays = unique(pubkeyRelays) return pubkeyRelays } + +func contactsForPubkey(ctx context.Context, pubkey string, extraRelays ...string) []string { + pubkeyContacts := make([]string, 0, 100) + relays := make([]string, 0, 12) + if ok := cache.GetJSON("cc:"+pubkey, &pubkeyContacts); !ok { + fmt.Printf("Searching contacts for %s\n", pubkey) + ctx, cancel := context.WithTimeout(ctx, time.Millisecond*1500) + + pubkeyRelays := relaysForPubkey(ctx, pubkey, relays...) + relays = append(relays, pubkeyRelays...) + relays = append(relays, always...) + relays = append(relays, profiles...) + + ch := pool.SubManyEose(ctx, relays, nostr.Filters{ + { + Kinds: []int{3}, + Authors: []string{pubkey}, + Limit: 2, + }, + }) + + for event := range ch { + for _, tag := range event.Tags { + if tag[0] == "p" { + pubkeyContacts = append(pubkeyContacts, tag[1]) + } + } + } + + cancel() + if len(pubkeyContacts) > 0 { + cache.SetJSONWithTTL("cc:"+pubkey, pubkeyContacts, time.Hour*6) + } + } + pubkeyContacts = unique(pubkeyContacts) + return pubkeyContacts +} diff --git a/render_profiles_archive.go b/render_profiles_archive.go new file mode 100644 index 0000000..f2851f0 --- /dev/null +++ b/render_profiles_archive.go @@ -0,0 +1,52 @@ +package main + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/nbd-wtf/go-nostr/nip19" +) + +func renderProfilesArchive(w http.ResponseWriter, r *http.Request) { + resultsPerPage := 100 + lastIndex := strings.LastIndex(r.URL.Path, "/") + page := 1 + if lastIndex != -1 { + pageString := r.URL.Path[lastIndex+1:] + pageInt, err := strconv.Atoi(pageString) + if err != nil { + page = 1 + } else { + page = pageInt + } + } + + keys := cache.GetPaginatedkeys("pa", page, resultsPerPage) + npubs := []string{} + for i := 0; i < len(keys); i++ { + npub, _ := nip19.EncodePublicKey(keys[i]) + npubs = append(npubs, npub) + } + + prevPage := page - 1 + nextPage := page + 1 + if len(keys) == 0 { + prevPage = 0 + nextPage = 0 + } + + params := map[string]any{ + "nextPage": fmt.Sprint(nextPage), + "prevPage": fmt.Sprint(prevPage), + "data": npubs, + } + + w.Header().Set("Cache-Control", "max-age=604800") + + if err := tmpl.ExecuteTemplate(w, "archive.html", params); err != nil { + log.Error().Err(err).Msg("error rendering") + return + } +} diff --git a/templates/archive.html b/templates/archive.html new file mode 100644 index 0000000..b2a61c6 --- /dev/null +++ b/templates/archive.html @@ -0,0 +1,62 @@ + + + + + Nostr npubs archive + + {{template "head_common.html" }} + + + + {{template "top.html" .}} + +
+
+
+
+ Nostr npubs archive +   +
+
+ +
+
+ +
+
+

+ Nostr npubs archive +

+
+
+ +
+ {{range $index, $element := .data}} + +
{{$element | escapeString}}
+ + {{end}} +
+ + {{if not (eq .prevPage "0")}} + << Prev page + {{end}} +       + {{if not (eq .nextPage "0")}} + Next page >> + {{end}} + +
+ + + +
+
+ + {{template "footer.html"}} + + + + diff --git a/utils.go b/utils.go index 47ea82d..a6da510 100644 --- a/utils.go +++ b/utils.go @@ -1,11 +1,13 @@ package main import ( + "context" "encoding/json" "fmt" "net/http" "regexp" "strings" + "time" "github.com/gomarkdown/markdown" "github.com/gomarkdown/markdown/html" @@ -344,3 +346,22 @@ func trimProtocol(relay string) string { relay = strings.TrimPrefix(relay, "ws:/") // Some browsers replace upfront '//' with '/' return relay } + +func loadNpubsArchive(ctx context.Context) { + fmt.Println("Refreshing the npubs archive") + + contactsArchive := make([]string, 0, 500) + + for _, pubkey := range trustedPubKeys { + ctx, cancel := context.WithTimeout(ctx, time.Second*4) + pubkeyContacts := contactsForPubkey(ctx, pubkey) + contactsArchive = append(contactsArchive, pubkeyContacts...) + cancel() + } + + contactsArchive = unique(contactsArchive) + for _, contact := range contactsArchive { + fmt.Printf("Adding contact %s\n", contact) + cache.SetWithTTL("pa:"+contact, nil, time.Hour*24*90) + } +} From 37ff53f22c0a59059b731433e21fdead607d8f99 Mon Sep 17 00:00:00 2001 From: Daniele Tonon Date: Wed, 19 Jul 2023 01:06:43 +0200 Subject: [PATCH 2/8] Add relays archive --- main.go | 4 ++- ...r_profiles_archive.go => render_archive.go | 33 ++++++++++++++----- templates/archive.html | 10 +++--- utils.go | 19 +++++++++++ 4 files changed, 52 insertions(+), 14 deletions(-) rename render_profiles_archive.go => render_archive.go (52%) diff --git a/main.go b/main.go index 1e597a8..619ac39 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,7 @@ var ( func updateArchives(ctx context.Context) { for { loadNpubsArchive(ctx) + loadRelaysArchive(ctx) // Wait for 24 hours before executing the function again time.Sleep(24 * time.Hour) } @@ -79,7 +80,8 @@ func main() { http.HandleFunc("/njump/image/", generate) http.HandleFunc("/njump/proxy/", proxy) http.Handle("/njump/static/", http.StripPrefix("/njump/", http.FileServer(http.FS(static)))) - http.HandleFunc("/npubs-archive/", renderProfilesArchive) + http.HandleFunc("/npubs-archive/", renderArchive) + http.HandleFunc("/relays-archive/", renderArchive) http.HandleFunc("/", render) log.Print("listening at http://0.0.0.0:" + s.Port) diff --git a/render_profiles_archive.go b/render_archive.go similarity index 52% rename from render_profiles_archive.go rename to render_archive.go index f2851f0..acb61df 100644 --- a/render_profiles_archive.go +++ b/render_archive.go @@ -9,8 +9,8 @@ import ( "github.com/nbd-wtf/go-nostr/nip19" ) -func renderProfilesArchive(w http.ResponseWriter, r *http.Request) { - resultsPerPage := 100 +func renderArchive(w http.ResponseWriter, r *http.Request) { + resultsPerPage := 50 lastIndex := strings.LastIndex(r.URL.Path, "/") page := 1 if lastIndex != -1 { @@ -23,11 +23,26 @@ func renderProfilesArchive(w http.ResponseWriter, r *http.Request) { } } - keys := cache.GetPaginatedkeys("pa", page, resultsPerPage) - npubs := []string{} + prefix := "" + title := "" + area := strings.Split(r.URL.Path[1:], "/")[0] + if area == "npubs-archive" { + prefix = "pa" + title = "Nostr npubs archive" + } else { + prefix = "ra" + title = "Nostr relays archive" + } + + keys := cache.GetPaginatedkeys(prefix, page, resultsPerPage) + data := []string{} for i := 0; i < len(keys); i++ { - npub, _ := nip19.EncodePublicKey(keys[i]) - npubs = append(npubs, npub) + if area == "npubs-archive" { + npub, _ := nip19.EncodePublicKey(keys[i]) + data = append(data, npub) + } else { + data = append(data, keys[i]) + } } prevPage := page - 1 @@ -38,12 +53,14 @@ func renderProfilesArchive(w http.ResponseWriter, r *http.Request) { } params := map[string]any{ + "title": title, + "data": data, + "pagination_url": area , "nextPage": fmt.Sprint(nextPage), "prevPage": fmt.Sprint(prevPage), - "data": npubs, } - w.Header().Set("Cache-Control", "max-age=604800") + w.Header().Set("Cache-Control", "max-age=86400") if err := tmpl.ExecuteTemplate(w, "archive.html", params); err != nil { log.Error().Err(err).Msg("error rendering") diff --git a/templates/archive.html b/templates/archive.html index b2a61c6..a675910 100644 --- a/templates/archive.html +++ b/templates/archive.html @@ -2,7 +2,7 @@ - Nostr npubs archive + {{.title}} {{template "head_common.html" }} @@ -14,7 +14,7 @@
- Nostr npubs archive + {{.title}}  
@@ -25,7 +25,7 @@

- Nostr npubs archive + {{.title}}

@@ -39,11 +39,11 @@
{{if not (eq .prevPage "0")}} - << Prev page + << Prev page {{end}}       {{if not (eq .nextPage "0")}} - Next page >> + Next page >> {{end}}
diff --git a/utils.go b/utils.go index a6da510..df47dff 100644 --- a/utils.go +++ b/utils.go @@ -365,3 +365,22 @@ func loadNpubsArchive(ctx context.Context) { cache.SetWithTTL("pa:"+contact, nil, time.Hour*24*90) } } + +func loadRelaysArchive(ctx context.Context) { + fmt.Println("Refreshing the relays archive") + + relaysArchive := make([]string, 0, 500) + + for _, pubkey := range trustedPubKeys { + ctx, cancel := context.WithTimeout(ctx, time.Second*4) + pubkeyContacts := relaysForPubkey(ctx, pubkey) + relaysArchive = append(relaysArchive, pubkeyContacts...) + cancel() + } + + relaysArchive = unique(relaysArchive) + for _, relay := range relaysArchive { + fmt.Printf("Adding relay %s\n", relay) + cache.SetWithTTL("ra:"+relay, nil, time.Hour*24*7) + } +} \ No newline at end of file From d23463dadf8d33fc1dac020650f7144b2262fe8b Mon Sep 17 00:00:00 2001 From: Daniele Tonon Date: Thu, 27 Jul 2023 18:21:52 +0200 Subject: [PATCH 3/8] Add GetPaginatedkeys for null_cache --- null_cache.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/null_cache.go b/null_cache.go index d53f27f..bd6e2a2 100644 --- a/null_cache.go +++ b/null_cache.go @@ -10,10 +10,11 @@ var cache = Cache{} type Cache struct{} -func (c *Cache) initialize() func() { return func() {} } -func (c *Cache) Get(key string) ([]byte, bool) { return nil, false } -func (c *Cache) GetJSON(key string, recv any) bool { return false } -func (c *Cache) Set(key string, value []byte) {} -func (c *Cache) SetJSON(key string, value any) {} -func (c *Cache) SetWithTTL(key string, value []byte, ttl time.Duration) {} -func (c *Cache) SetJSONWithTTL(key string, value any, ttl time.Duration) {} +func (c *Cache) initialize() func() { return func() {} } +func (c *Cache) Get(key string) ([]byte, bool) { return nil, false } +func (c *Cache) GetJSON(key string, recv any) bool { return false } +func (c *Cache) Set(key string, value []byte) {} +func (c *Cache) SetJSON(key string, value any) {} +func (c *Cache) SetWithTTL(key string, value []byte, ttl time.Duration) {} +func (c *Cache) SetJSONWithTTL(key string, value any, ttl time.Duration) {} +func (c *Cache) GetPaginatedkeys(prefix string, page int, size int) []string { return []string{} } From fb046515cfacff4800a3b0622fc36d2354e10aa7 Mon Sep 17 00:00:00 2001 From: Daniele Tonon Date: Thu, 27 Jul 2023 22:49:02 +0200 Subject: [PATCH 4/8] Use new relay /r/... path in the archive --- render_archive.go | 18 +++++++++++------- templates/archive.html | 8 ++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/render_archive.go b/render_archive.go index acb61df..bd1b30a 100644 --- a/render_archive.go +++ b/render_archive.go @@ -24,13 +24,16 @@ func renderArchive(w http.ResponseWriter, r *http.Request) { } prefix := "" + path_prefix := "" title := "" area := strings.Split(r.URL.Path[1:], "/")[0] if area == "npubs-archive" { prefix = "pa" + path_prefix = "" title = "Nostr npubs archive" } else { prefix = "ra" + path_prefix = "r/" title = "Nostr relays archive" } @@ -40,9 +43,9 @@ func renderArchive(w http.ResponseWriter, r *http.Request) { if area == "npubs-archive" { npub, _ := nip19.EncodePublicKey(keys[i]) data = append(data, npub) - } else { + } else { data = append(data, keys[i]) - } + } } prevPage := page - 1 @@ -53,11 +56,12 @@ func renderArchive(w http.ResponseWriter, r *http.Request) { } params := map[string]any{ - "title": title, - "data": data, - "pagination_url": area , - "nextPage": fmt.Sprint(nextPage), - "prevPage": fmt.Sprint(prevPage), + "title": title, + "pathPrefix": path_prefix, + "data": data, + "paginationUrl": area, + "nextPage": fmt.Sprint(nextPage), + "prevPage": fmt.Sprint(prevPage), } w.Header().Set("Cache-Control", "max-age=86400") diff --git a/templates/archive.html b/templates/archive.html index a675910..671eab5 100644 --- a/templates/archive.html +++ b/templates/archive.html @@ -31,19 +31,19 @@
- {{range $index, $element := .data}} - + {{range $index, $element := .data }} +
{{$element | escapeString}}
 {{end}}
{{if not (eq .prevPage "0")}} - << Prev page + << Prev page {{end}}       {{if not (eq .nextPage "0")}} - Next page >> + Next page >> {{end}}
From a325740661f8615c8170b4b1a25fd03291717607 Mon Sep 17 00:00:00 2001 From: Daniele Tonon Date: Thu, 27 Jul 2023 23:29:57 +0200 Subject: [PATCH 5/8] Style pagination links --- static/styles.css | 9 +++++++++ static/styles.scss | 9 +++++++++ templates/archive.html | 4 ++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/static/styles.css b/static/styles.css index 165e56e..6b3aae7 100644 --- a/static/styles.css +++ b/static/styles.css @@ -712,6 +712,15 @@ iframe { color: #969696; } } +.container .column_content a.pagination { + color: #e32a6d; +} +.container .column_content a.pagination.next { + float: right; +} +.container .column_content a.pagination.prev { + float: left; +} .container .column_clients { position: -webkit-sticky; position: sticky; diff --git a/static/styles.scss b/static/styles.scss index f401eeb..4a9f8f3 100644 --- a/static/styles.scss +++ b/static/styles.scss @@ -657,6 +657,15 @@ iframe { } } } + a.pagination { + color: $color-accent1; + &.next { + float: right; + } + &.prev { + float: left; + } + } } .column_clients { diff --git a/templates/archive.html b/templates/archive.html index 671eab5..97afff1 100644 --- a/templates/archive.html +++ b/templates/archive.html @@ -39,11 +39,11 @@
{{if not (eq .prevPage "0")}} - << Prev page + {{end}}       {{if not (eq .nextPage "0")}} - Next page >> + {{end}} From a20b985cc870ff32de920e83f70c8a29b149c9f2 Mon Sep 17 00:00:00 2001 From: Daniele Tonon Date: Thu, 27 Jul 2023 23:23:16 +0200 Subject: [PATCH 6/8] Trim protocon from archive's relay urls --- main.go | 9 +++++---- templates/archive.html | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 619ac39..2ad3633 100644 --- a/main.go +++ b/main.go @@ -34,10 +34,10 @@ var ( func updateArchives(ctx context.Context) { for { - loadNpubsArchive(ctx) - loadRelaysArchive(ctx) - // Wait for 24 hours before executing the function again - time.Sleep(24 * time.Hour) + loadNpubsArchive(ctx) + loadRelaysArchive(ctx) + // Wait for 24 hours before executing the function again + time.Sleep(24 * time.Hour) } } @@ -68,6 +68,7 @@ func main() { "mdToHTML": mdToHTML, "escapeString": html.EscapeString, "sanitizeXSS": sanitizeXSS, + "trimProtocol": trimProtocol, } tmpl = template.Must( diff --git a/templates/archive.html b/templates/archive.html index 97afff1..32abd80 100644 --- a/templates/archive.html +++ b/templates/archive.html @@ -32,7 +32,7 @@
{{range $index, $element := .data }} - +
{{$element | escapeString}}
 {{end}} From dcf8e3a45138f1a800e2e11211daaff633727a2c Mon Sep 17 00:00:00 2001 From: Daniele Tonon Date: Thu, 27 Jul 2023 22:49:44 +0200 Subject: [PATCH 7/8] Filter out some relays from the archive --- nostr.go | 4 ++++ utils.go | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/nostr.go b/nostr.go index 302bc0b..982f4ae 100644 --- a/nostr.go +++ b/nostr.go @@ -41,6 +41,10 @@ var ( "97c70a44366a6535c145b333f973ea86dfdc2d7a99da618c40c64705ad98e322", // hodlbod "ee11a5dff40c19a555f41fe42b48f00e618c91225622ae37b6c2bb67b76c4e49", // Michael Dilger } + + excludedRelays = []string{ + "wss://filter.nostr.wine", // paid + } ) func getRelay() string { diff --git a/utils.go b/utils.go index df47dff..1bceda6 100644 --- a/utils.go +++ b/utils.go @@ -380,7 +380,16 @@ func loadRelaysArchive(ctx context.Context) { relaysArchive = unique(relaysArchive) for _, relay := range relaysArchive { + for _, excluded := range excludedRelays { + if strings.Contains(relay, excluded) { + fmt.Printf("Skypping relay %s\n", relay) + continue + } + } + if strings.Contains(relay, "/npub1") { + continue // Skip relays with personalyzed query like filter.nostr.wine + } fmt.Printf("Adding relay %s\n", relay) cache.SetWithTTL("ra:"+relay, nil, time.Hour*24*7) } -} \ No newline at end of file +} From f3781c1f76479697f6f75df4f48e91dabdc8625d Mon Sep 17 00:00:00 2001 From: Daniele Tonon Date: Thu, 27 Jul 2023 23:37:25 +0200 Subject: [PATCH 8/8] Exit updateArchives gracefully --- main.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 2ad3633..c7e64e5 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "embed" + "fmt" "html" "net/http" "os" @@ -34,9 +35,15 @@ var ( func updateArchives(ctx context.Context) { for { - loadNpubsArchive(ctx) - loadRelaysArchive(ctx) - // Wait for 24 hours before executing the function again + select { + // Check for the cancellation signal. + case <-ctx.Done(): + fmt.Println("Exit updateArchives gracefully...") + return + default: + loadNpubsArchive(ctx) + loadRelaysArchive(ctx) + } time.Sleep(24 * time.Hour) } }