From a30d48443226f3878b57f8deb4f070a542bf2197 Mon Sep 17 00:00:00 2001 From: fiatjaf Date: Sun, 6 Oct 2024 18:26:06 -0300 Subject: [PATCH] using leafdb instead of the messy badger db we had. --- cache.go | 139 ---------------- go.mod | 10 +- go.sum | 83 +--------- internal.pb.go | 358 ++++++++++++++++++++++++++++++++++++++++ internal.proto | 22 +++ internaldb.go | 203 +++++++++++++++++++++++ justfile | 6 +- main.go | 20 ++- nostr.go | 125 ++++++-------- null_cache.go | 27 --- redirect.go | 20 +-- render_archive.go | 10 +- render_profile.go | 2 + render_relay.go | 24 ++- render_sitemap_index.go | 9 +- routines.go | 77 ++++----- utils.go | 38 ----- 17 files changed, 721 insertions(+), 452 deletions(-) delete mode 100644 cache.go create mode 100644 internal.pb.go create mode 100644 internal.proto create mode 100644 internaldb.go delete mode 100644 null_cache.go diff --git a/cache.go b/cache.go deleted file mode 100644 index 6d83870..0000000 --- a/cache.go +++ /dev/null @@ -1,139 +0,0 @@ -//go:build !nocache - -package main - -import ( - "encoding/json" - "strings" - "time" - - "github.com/dgraph-io/badger/v4" -) - -var cache = Cache{} - -type Cache struct { - *badger.DB -} - -func (c *Cache) initializeCache() func() { - db, err := badger.Open(badger.DefaultOptions(s.DiskCachePath)) - if err != nil { - log.Fatal().Err(err).Str("path", s.DiskCachePath).Msg("failed to open badger") - } - c.DB = db - - go func() { - ticker := time.NewTicker(2 * time.Hour) - defer ticker.Stop() - for range ticker.C { - again: - err := db.RunValueLogGC(0.8) - if err == nil { - goto again - } - } - }() - - return func() { - db.Close() - } -} - -func (c *Cache) Delete(key string) error { - return c.DB.Update(func(txn *badger.Txn) error { - return txn.Delete([]byte(key)) - }) -} - -func (c *Cache) Get(key string) ([]byte, bool) { - var val []byte - err := c.DB.View(func(txn *badger.Txn) error { - b, err := txn.Get([]byte(key)) - if err != nil { - return err - } - - val, err = b.ValueCopy(nil) - return err - }) - - if err == badger.ErrKeyNotFound { - return nil, false - } - if err != nil { - log.Fatal().Err(err).Str("key", key).Msg("error getting key from cache") - } - - return val, true -} - -func (c *Cache) GetPaginatedKeys(prefix string, page int, size int) []string { - keys := []string{} - err := c.DB.View(func(txn *badger.Txn) error { - opts := badger.DefaultIteratorOptions - opts.PrefetchValues = false - it := txn.NewIterator(opts) - defer it.Close() - start := (page-1)*size + 1 - index := 1 - for it.Seek([]byte(prefix)); it.ValidForPrefix([]byte(prefix)); it.Next() { - if index < start { - index++ - continue - } - if index > start+size-1 { - break - } - item := it.Item() - k := item.Key() - keys = append(keys, strings.TrimPrefix(string(k), prefix+":")) - index++ - } - return nil - }) - if err != nil { - log.Fatal().Err(err).Msg("") - } - - return keys -} - -func (c *Cache) GetJSON(key string, recv any) bool { - b, ok := c.Get(key) - if !ok { - return ok - } - json.Unmarshal(b, recv) - return true -} - -func (c *Cache) Set(key string, value []byte) { - err := c.DB.Update(func(txn *badger.Txn) error { - return txn.Set([]byte(key), value) - }) - if err != nil { - log.Fatal().Err(err).Msg("") - } -} - -func (c *Cache) SetJSON(key string, value any) { - j, _ := json.Marshal(value) - c.Set(key, j) -} - -func (c *Cache) SetWithTTL(key string, value []byte, ttl time.Duration) { - err := c.DB.Update(func(txn *badger.Txn) error { - return txn.SetEntry( - badger.NewEntry([]byte(key), value).WithTTL(ttl), - ) - }) - if err != nil { - log.Fatal().Err(err).Msg("") - } -} - -func (c *Cache) SetJSONWithTTL(key string, value any, ttl time.Duration) { - j, _ := json.Marshal(value) - c.SetWithTTL(key, j, ttl) -} diff --git a/go.mod b/go.mod index 1d94a4c..18f23bb 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module github.com/fiatjaf/njump go 1.23.0 require ( + fiatjaf.com/leafdb v0.0.6 github.com/PuerkitoBio/goquery v1.8.1 github.com/a-h/templ v0.2.771 github.com/bytesparadise/libasciidoc v0.8.0 - github.com/dgraph-io/badger/v4 v4.2.0 github.com/fiatjaf/eventstore v0.11.1 github.com/fiatjaf/khatru v0.8.1 github.com/fogleman/gg v1.3.0 @@ -27,6 +27,7 @@ require ( github.com/texttheater/golang-levenshtein v1.0.1 github.com/tylermmorton/tmpl v0.0.0-20231025031313-5552ee818c6d golang.org/x/image v0.17.0 + google.golang.org/protobuf v1.34.2 mvdan.cc/xurls/v2 v2.5.0 ) @@ -43,7 +44,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fasthttp/websocket v1.5.7 // indirect @@ -52,12 +52,8 @@ require ( github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.2.1 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/graph-gophers/dataloader/v7 v7.1.0 // indirect @@ -77,7 +73,6 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.51.0 // indirect - go.opencensus.io v0.24.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.29.0 // indirect @@ -85,7 +80,6 @@ require ( golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/tools v0.25.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d95e7ab..5b555d3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +fiatjaf.com/leafdb v0.0.6 h1:/qRX7uwqWEbtdDQBMAhoXnOrl01HZ1CqMvWjQRXqlGw= +fiatjaf.com/leafdb v0.0.6/go.mod h1:UAmS4uwhlAzFdNZUUG6vguJfnahqgEa+b7HU/Bi05wk= github.com/DataDog/gostackparse v0.5.0 h1:jb72P6GFHPHz2W0onsN51cS3FkaMDcjb0QzgxxA4gDk= github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= github.com/PowerDNS/lmdb-go v1.9.2 h1:Cmgerh9y3ZKBZGz1irxSShhfmFyRUh+Zdk4cZk7ZJvU= @@ -45,15 +45,11 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bytesparadise/libasciidoc v0.8.0 h1:iWAlYR7gm4Aes3NSvuGQyzRavatQpUBAJZyU9uMmwm0= github.com/bytesparadise/libasciidoc v0.8.0/go.mod h1:Q2ZeBQ1fko5+NTUTs8rGu9gjTtbVaD6Qxg37GOPYdN4= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,21 +62,12 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeC github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= -github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fasthttp/websocket v1.5.7 h1:0a6o2OfeATvtGgoMKleURhLT6JqWPg7fYfWnH4KHau4= github.com/fasthttp/websocket v1.5.7/go.mod h1:bC4fxSono9czeXHQUVKxsC0sNjbm7lPJR04GDFqClfU= github.com/felixge/fgtrace v0.1.0 h1:cuMLI5NoBg/9IxIVmJzsxA3Aoz5eIKRca6WE1U2C1zc= @@ -108,49 +95,32 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd h1:PppHBegd3uPZ3Y/Iax/2mlCFJm1w4Qf/zP1MdW4ju2o= github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= -github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= -github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/graph-gophers/dataloader/v7 v7.1.0 h1:Wn8HGF/q7MNXcvfaBnLEPEFJttVHR8zuEqP1obys/oc= @@ -164,8 +134,6 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= @@ -213,7 +181,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/puzpuzpuz/xsync/v3 v3.4.0 h1:DuVBAdXuGFHv8adVXjWWZ63pJq+NRXOWVXlKDBZ+mJ4= github.com/puzpuzpuz/xsync/v3 v3.4.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= @@ -232,15 +199,9 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= @@ -259,42 +220,28 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco= golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -302,16 +249,12 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -333,7 +276,6 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= @@ -350,15 +292,9 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190830223141-573d9926052a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= @@ -366,25 +302,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= @@ -403,7 +326,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= mvdan.cc/xurls/v2 v2.5.0 h1:lyBNOm8Wo71UknhUs4QTFUNNMyxy2JEIaKKo0RWOh+8= mvdan.cc/xurls/v2 v2.5.0/go.mod h1:yQgaGQ1rFtJUzkmKiHYSSfuQxqfYmd//X6PxvholpeE= diff --git a/internal.pb.go b/internal.pb.go new file mode 100644 index 0000000..3654f87 --- /dev/null +++ b/internal.pb.go @@ -0,0 +1,358 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.34.2 +// protoc v5.28.1 +// source: internal.proto + +package main + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type CachedEvent struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Expiry int64 `protobuf:"varint,2,opt,name=expiry,proto3" json:"expiry,omitempty"` + Relays []string `protobuf:"bytes,3,rep,name=relays,proto3" json:"relays,omitempty"` +} + +func (x *CachedEvent) Reset() { + *x = CachedEvent{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CachedEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CachedEvent) ProtoMessage() {} + +func (x *CachedEvent) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CachedEvent.ProtoReflect.Descriptor instead. +func (*CachedEvent) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{0} +} + +func (x *CachedEvent) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *CachedEvent) GetExpiry() int64 { + if x != nil { + return x.Expiry + } + return 0 +} + +func (x *CachedEvent) GetRelays() []string { + if x != nil { + return x.Relays + } + return nil +} + +type FollowListArchive struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Source string `protobuf:"bytes,1,opt,name=source,proto3" json:"source,omitempty"` + Pubkeys []string `protobuf:"bytes,2,rep,name=pubkeys,proto3" json:"pubkeys,omitempty"` +} + +func (x *FollowListArchive) Reset() { + *x = FollowListArchive{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FollowListArchive) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FollowListArchive) ProtoMessage() {} + +func (x *FollowListArchive) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FollowListArchive.ProtoReflect.Descriptor instead. +func (*FollowListArchive) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{1} +} + +func (x *FollowListArchive) GetSource() string { + if x != nil { + return x.Source + } + return "" +} + +func (x *FollowListArchive) GetPubkeys() []string { + if x != nil { + return x.Pubkeys + } + return nil +} + +type PubKeyArchive struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pubkey string `protobuf:"bytes,1,opt,name=pubkey,proto3" json:"pubkey,omitempty"` +} + +func (x *PubKeyArchive) Reset() { + *x = PubKeyArchive{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PubKeyArchive) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PubKeyArchive) ProtoMessage() {} + +func (x *PubKeyArchive) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PubKeyArchive.ProtoReflect.Descriptor instead. +func (*PubKeyArchive) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{2} +} + +func (x *PubKeyArchive) GetPubkey() string { + if x != nil { + return x.Pubkey + } + return "" +} + +type EventInRelay struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *EventInRelay) Reset() { + *x = EventInRelay{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EventInRelay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EventInRelay) ProtoMessage() {} + +func (x *EventInRelay) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EventInRelay.ProtoReflect.Descriptor instead. +func (*EventInRelay) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{3} +} + +func (x *EventInRelay) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +var File_internal_proto protoreflect.FileDescriptor + +var file_internal_proto_rawDesc = []byte{ + 0x0a, 0x0e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0x4d, 0x0a, 0x0b, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, + 0x16, 0x0a, 0x06, 0x65, 0x78, 0x70, 0x69, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x06, 0x65, 0x78, 0x70, 0x69, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x6c, 0x61, 0x79, 0x73, 0x22, + 0x45, 0x0a, 0x11, 0x46, 0x6f, 0x6c, 0x6c, 0x6f, 0x77, 0x4c, 0x69, 0x73, 0x74, 0x41, 0x72, 0x63, + 0x68, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x70, + 0x75, 0x62, 0x6b, 0x65, 0x79, 0x73, 0x22, 0x27, 0x0a, 0x0d, 0x50, 0x75, 0x62, 0x4b, 0x65, 0x79, + 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x75, 0x62, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x70, 0x75, 0x62, 0x6b, 0x65, 0x79, 0x22, + 0x1e, 0x0a, 0x0c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x52, 0x65, 0x6c, 0x61, 0x79, 0x12, + 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x42, + 0x1f, 0x5a, 0x1d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x69, + 0x61, 0x74, 0x6a, 0x61, 0x66, 0x2f, 0x6e, 0x6a, 0x75, 0x6d, 0x70, 0x3b, 0x6d, 0x61, 0x69, 0x6e, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_internal_proto_rawDescOnce sync.Once + file_internal_proto_rawDescData = file_internal_proto_rawDesc +) + +func file_internal_proto_rawDescGZIP() []byte { + file_internal_proto_rawDescOnce.Do(func() { + file_internal_proto_rawDescData = protoimpl.X.CompressGZIP(file_internal_proto_rawDescData) + }) + return file_internal_proto_rawDescData +} + +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_internal_proto_goTypes = []any{ + (*CachedEvent)(nil), // 0: CachedEvent + (*FollowListArchive)(nil), // 1: FollowListArchive + (*PubKeyArchive)(nil), // 2: PubKeyArchive + (*EventInRelay)(nil), // 3: EventInRelay +} +var file_internal_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_internal_proto_init() } +func file_internal_proto_init() { + if File_internal_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_internal_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*CachedEvent); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*FollowListArchive); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_proto_msgTypes[2].Exporter = func(v any, i int) any { + switch v := v.(*PubKeyArchive); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_proto_msgTypes[3].Exporter = func(v any, i int) any { + switch v := v.(*EventInRelay); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_internal_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_internal_proto_goTypes, + DependencyIndexes: file_internal_proto_depIdxs, + MessageInfos: file_internal_proto_msgTypes, + }.Build() + File_internal_proto = out.File + file_internal_proto_rawDesc = nil + file_internal_proto_goTypes = nil + file_internal_proto_depIdxs = nil +} diff --git a/internal.proto b/internal.proto new file mode 100644 index 0000000..b39291e --- /dev/null +++ b/internal.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +option go_package = "github.com/fiatjaf/njump;main"; + +message CachedEvent { + string id = 1; + int64 expiry = 2; + repeated string relays = 3; +} + +message FollowListArchive { + string source = 1; + repeated string pubkeys = 2; +} + +message PubKeyArchive { + string pubkey = 1; +} + +message EventInRelay { + string id = 1; +} diff --git a/internaldb.go b/internaldb.go new file mode 100644 index 0000000..449f4de --- /dev/null +++ b/internaldb.go @@ -0,0 +1,203 @@ +package main + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "iter" + "slices" + "time" + + "fiatjaf.com/leafdb" + "github.com/nbd-wtf/go-nostr" + "github.com/nbd-wtf/go-nostr/sdk" + "google.golang.org/protobuf/proto" +) + +const ( + TypeCachedEvent leafdb.DataType = 0 + TypeFollowListArchive leafdb.DataType = 3 + TypePubKeyArchive leafdb.DataType = 4 + TypeEventInRelay leafdb.DataType = 5 +) + +func NewInternalDB(path string) (*InternalDB, error) { + ldb, err := leafdb.New(path, leafdb.Options[proto.Message]{ + Encode: func(t leafdb.DataType, msg proto.Message) ([]byte, error) { + return proto.Marshal(msg) + }, + Decode: func(t leafdb.DataType, buf []byte) (proto.Message, error) { + var v proto.Message + switch t { + case TypeCachedEvent: + v = &CachedEvent{} + case TypeFollowListArchive: + v = &FollowListArchive{} + case TypePubKeyArchive: + v = &PubKeyArchive{} + case TypeEventInRelay: + v = &EventInRelay{} + default: + return nil, fmt.Errorf("what is this? %v", t) + } + err := proto.Unmarshal(buf, v) + return v, err + }, + Indexes: map[string]leafdb.IndexDefinition[proto.Message]{ + "expiring-when": { + Version: 1, + Types: []leafdb.DataType{TypeCachedEvent}, + Emit: func(t leafdb.DataType, data proto.Message, emit func([]byte)) { + ee := data.(*CachedEvent) + emit(binary.BigEndian.AppendUint32(nil, uint32(ee.Expiry))) + }, + }, + "cached-id": { + Version: 1, + Types: []leafdb.DataType{TypeCachedEvent}, + Emit: func(t leafdb.DataType, data proto.Message, emit func([]byte)) { + ee := data.(*CachedEvent) + internal, err := hex.DecodeString(ee.Id[0:16]) + if err != nil { + log.Fatal().Err(err).Str("id", ee.Id).Msg("failed to decode event id hex") + return + } + emit(internal) + }, + }, + "follow-list-by-source": { + Version: 1, + Types: []leafdb.DataType{TypeFollowListArchive}, + Emit: func(t leafdb.DataType, value proto.Message, emit func([]byte)) { + fla := value.(*FollowListArchive) + pkb, _ := hex.DecodeString(fla.Source[0:16]) + emit(pkb) + }, + }, + }, + Views: map[string]leafdb.ViewDefinition[proto.Message]{ + "pubkey-archive": { + Version: 1, + Types: []leafdb.DataType{TypeFollowListArchive}, + Emit: func(t leafdb.DataType, value proto.Message, emit func(idxkey []byte, t leafdb.DataType, value proto.Message)) { + fla := value.(*FollowListArchive) + for _, pubkey := range fla.Pubkeys { + emit([]byte{1}, TypePubKeyArchive, &PubKeyArchive{Pubkey: pubkey}) + } + }, + }, + "events-in-relay": { + Version: 1, + Types: []leafdb.DataType{TypeCachedEvent}, + Emit: func(t leafdb.DataType, value proto.Message, emit func(idxkey []byte, t leafdb.DataType, value proto.Message)) { + ee := value.(*CachedEvent) + for _, r := range ee.Relays { + emit([]byte(trimProtocolAndEndingSlash(r)), TypeEventInRelay, &EventInRelay{Id: ee.Id}) + } + }, + }, + }, + }) + if err != nil { + return nil, err + } + + return &InternalDB{ldb}, err +} + +type InternalDB struct { + *leafdb.DB[proto.Message] +} + +func (internal *InternalDB) scheduleEventExpiration(eventId string) { + idxkey, _ := hex.DecodeString(eventId[0:16]) + if err := internal.UpdateQuery( + leafdb.PrefixQuery("cached-id", idxkey), + func(t leafdb.DataType, data proto.Message) (proto.Message, error) { + ee := data.(*CachedEvent) + ee.Expiry = time.Now().Add(time.Hour * 24 * 7).Unix() + return ee, nil + }, + ); err != nil { + log.Fatal().Err(err).Msg("failed to update scheduled expirations") + } +} + +func (internal *InternalDB) deleteExpiredEvents(now nostr.Timestamp) (eventIds []string, err error) { + deleted, err := internal.DB.DeleteQuery(leafdb.QueryParams{ + Index: "expiring-when", + StartKey: []byte{0}, + EndKey: binary.BigEndian.AppendUint32(nil, uint32(now)), + }) + if err != nil { + return nil, err + } + + ids := make([]string, len(deleted)) + for i, d := range deleted { + ids[i] = d.Value.(*CachedEvent).Id + } + return ids, nil +} + +func (internal *InternalDB) notCached(id string) error { + idb, _ := hex.DecodeString(id[0:16]) + _, err := internal.DB.DeleteQuery(leafdb.ExactQuery("cached-id", idb)) + return err +} + +func (internal *InternalDB) overwriteFollowListArchive(fla *FollowListArchive) error { + _, err := internal.DB.AddOrReplace("follow-list-by-source", TypeFollowListArchive, fla) + return err +} + +func (internal *InternalDB) attachRelaysToEvent(eventId string, relays ...string) (allRelays []string) { + idb, _ := hex.DecodeString(eventId[0:16]) + if _, err := internal.DB.Upsert("cached-id", idb, TypeCachedEvent, func(t leafdb.DataType, value proto.Message) (proto.Message, error) { + var ee *CachedEvent + if value == nil { + ee = &CachedEvent{ + Id: eventId, + Relays: make([]string, 0, len(relays)), + Expiry: time.Now().Add(time.Hour * 24 * 7).Unix(), + } + } else { + ee = value.(*CachedEvent) + } + for _, r := range relays { + r = nostr.NormalizeURL(r) + if sdk.IsVirtualRelay(r) { + continue + } + if !slices.Contains(ee.Relays, r) { + ee.Relays = append(ee.Relays, r) + } + } + allRelays = ee.Relays + return ee, nil + }); err != nil { + log.Error().Err(err).Str("id", eventId).Strs("relays", relays).Msg("failed to attach relays to event") + } + + return allRelays +} + +func (internal *InternalDB) getRelaysForEvent(eventId string) []string { + idb, _ := hex.DecodeString(eventId[0:16]) + for value := range internal.DB.Query(leafdb.ExactQuery("cached-id", idb)) { + evtr := value.(*CachedEvent) + return evtr.Relays + } + return nil +} + +func (internal *InternalDB) getEventsInRelay(hostname string) iter.Seq[string] { + return func(yield func(string) bool) { + for value := range internal.DB.View(leafdb.ExactQuery("events-in-relay", []byte(hostname))) { + evtid := value.(*EventInRelay) + if !yield(evtid.Id) { + break + } + } + } +} diff --git a/justfile b/justfile index 8a9afa7..6d38257 100644 --- a/justfile +++ b/justfile @@ -22,15 +22,15 @@ libsecp256k1: templ: templ generate +protobuf: + protoc --proto_path=. --go_out=. --go_opt=paths=source_relative internal.proto + prettier: prettier -w templates/*.html tailwind: tailwind -i base.css -o static/tailwind-bundle.min.css --minify -test: - go test -tags=nocache - check-samples: #!/usr/bin/env xonsh base_url = ${...}.get('SERVICE_URL') diff --git a/main.go b/main.go index c80f793..e69a7a9 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "fmt" "html/template" "net/http" + "net/http/pprof" "os" "os/signal" "strings" @@ -21,7 +22,7 @@ import ( type Settings struct { Port string `envconfig:"PORT" default:"2999"` Domain string `envconfig:"DOMAIN" default:"njump.me"` - DiskCachePath string `envconfig:"DISK_CACHE_PATH" default:"/tmp/njump-internal"` + InternalDBPath string `envconfig:"DISK_CACHE_PATH" default:"/tmp/njump-internal"` EventStorePath string `envconfig:"EVENT_STORE_PATH" default:"/tmp/njump-db"` HintsMemoryDumpPath string `envconfig:"HINTS_SAVE_PATH" default:"/tmp/njump-hints.json"` TailwindDebug bool `envconfig:"TAILWIND_DEBUG"` @@ -33,8 +34,10 @@ type Settings struct { var static embed.FS var ( - s Settings - log = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: os.Stdout}).With().Timestamp().Logger() + s Settings + log = zerolog.New(os.Stderr).Output(zerolog.ConsoleWriter{Out: os.Stdout}). + With().Timestamp().Logger() + internal *InternalDB tailwindDebugStuff template.HTML ) @@ -104,7 +107,11 @@ func main() { initializeImageDrawingStuff() // internal db - defer cache.initializeCache()() + internal, err = NewInternalDB(s.InternalDBPath) + if err != nil { + log.Fatal().Err(err).Msg("failed to start internal db") + return + } // initialize routines ctx, cancel := context.WithCancel(context.Background()) @@ -139,6 +146,11 @@ func main() { mux.HandleFunc("/e/", redirectFromESlash) mux.HandleFunc("/p/", redirectFromPSlash) mux.HandleFunc("/favicon.ico", redirectToFavicon) + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) mux.HandleFunc("/embed/{code}", renderEmbedjs) mux.HandleFunc("/about", renderAbout) mux.HandleFunc("/{code}", renderEvent) diff --git a/nostr.go b/nostr.go index 91fcdad..858dc33 100644 --- a/nostr.go +++ b/nostr.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "iter" "slices" "sync" "time" @@ -42,11 +43,6 @@ var ( } ) -type CachedEvent struct { - Event *nostr.Event `json:"e"` - Relays []string `json:"r"` -} - func initSystem() func() { db := &lmdb.LMDBBackend{ Path: s.EventStorePath, @@ -113,10 +109,10 @@ func getEvent(ctx context.Context, code string) (*nostr.Event, []string, error) // unless it's a metadata event // (people complaining about njump keeping their metadata will try to load their metadata all the time) if evt.Kind != 0 { - scheduleEventExpiration(evt.ID, time.Hour*24*7) + internal.scheduleEventExpiration(evt.ID) } - return evt, getRelaysForEvent(evt.ID), nil + return evt, internal.getRelaysForEvent(evt.ID), nil } if author != "" { @@ -162,7 +158,7 @@ func getEvent(ctx context.Context, code string) (*nostr.Event, []string, error) subManyCtx, relays, nostr.Filters{filter}, - nostr.WithLabel("fetching "+prefix), + nostr.WithLabel("fetching-"+prefix), ) { fetchProfileOnce.Do(func() { go sys.FetchProfileMetadata(ctx, ie.PubKey) @@ -183,7 +179,7 @@ func getEvent(ctx context.Context, code string) (*nostr.Event, []string, error) // save stuff in cache and in internal store sys.StoreRelay.Publish(ctx, *result) // save relays if we got them - allRelays := attachRelaysToEvent(result.ID, successRelays...) + allRelays := internal.attachRelaysToEvent(result.ID, successRelays...) // put priority relays first so they get used in nevent and nprofile slices.SortFunc(allRelays, func(a, b string) int { vpa, _ := priorityRelays[a] @@ -191,7 +187,7 @@ func getEvent(ctx context.Context, code string) (*nostr.Event, []string, error) return vpb - vpa }) // keep track of what we have to delete later - scheduleEventExpiration(result.ID, time.Hour*24*7) + internal.scheduleEventExpiration(result.ID) return result, allRelays, nil } @@ -228,7 +224,7 @@ func authorLastNotes(ctx context.Context, pubkey string, isSitemap bool) []Enhan lastNotes = append(lastNotes, NewEnhancedEvent(ctx, evt)) if store { sys.Store.SaveEvent(ctx, evt) - scheduleEventExpiration(evt.ID, time.Hour*24) + internal.scheduleEventExpiration(evt.ID) } } } @@ -254,13 +250,13 @@ func authorLastNotes(ctx context.Context, pubkey string, isSitemap bool) []Enhan } ee := NewEnhancedEvent(ctx, ie.Event) - ee.relays = unique(append([]string{ie.Relay.URL}, getRelaysForEvent(ie.Event.ID)...)) + ee.relays = unique(append([]string{ie.Relay.URL}, internal.getRelaysForEvent(ie.Event.ID)...)) lastNotes = append(lastNotes, ee) if store { sys.Store.SaveEvent(ctx, ie.Event) - attachRelaysToEvent(ie.Event.ID, ie.Relay.URL) - scheduleEventExpiration(ie.Event.ID, time.Hour*24) + internal.attachRelaysToEvent(ie.Event.ID, ie.Relay.URL) + internal.scheduleEventExpiration(ie.Event.ID) } case <-ctx.Done(): break out @@ -273,79 +269,50 @@ func authorLastNotes(ctx context.Context, pubkey string, isSitemap bool) []Enhan return lastNotes } -func relayLastNotes(ctx context.Context, relayUrl string, isSitemap bool) []*nostr.Event { - key := "" - limit := 1000 - if isSitemap { - key = "rlns:" + nostr.NormalizeURL(relayUrl) - limit = 5000 - } else { - key = "rln:" + nostr.NormalizeURL(relayUrl) - } - - lastNotes := make([]*nostr.Event, 0, limit) - if ok := cache.GetJSON(key, &lastNotes); ok { - return lastNotes - } - +func relayLastNotes(ctx context.Context, hostname string, limit int) iter.Seq[*nostr.Event] { ctx, cancel := context.WithTimeout(ctx, time.Second*4) - defer cancel() - if relay, err := sys.Pool.EnsureRelay(relayUrl); err == nil { - lastNotes, _ = relay.QuerySync(ctx, nostr.Filter{ - Kinds: []int{1}, - Limit: limit, - }) - } + return func(yield func(*nostr.Event) bool) { + defer cancel() - slices.SortFunc(lastNotes, func(a, b *nostr.Event) int { return int(b.CreatedAt - a.CreatedAt) }) - if len(lastNotes) > 0 { - cache.SetJSONWithTTL(key, lastNotes, time.Hour*24) - } - return lastNotes -} - -func contactsForPubkey(ctx context.Context, pubkey string) []string { - pubkeyContacts := make([]string, 0, 300) - relays := make([]string, 0, 12) - if ok := cache.GetJSON("cc:"+pubkey, &pubkeyContacts); !ok { - log.Debug().Msgf("searching contacts for %s", pubkey) - ctx, cancel := context.WithTimeout(ctx, time.Second*3) - - pubkeyRelays := sys.FetchOutboxRelays(ctx, pubkey, 3) - relays = append(relays, pubkeyRelays...) - relays = append(relays, sys.MetadataRelays...) - - ch := sys.Pool.SubManyEose( - ctx, - relays, - nostr.Filters{{Kinds: []int{3}, Authors: []string{pubkey}, Limit: 2}}, - nostr.WithLabel("contacts"), - ) - - for { - select { - case evt, more := <-ch: - if !more { - goto end - } - for _, tag := range evt.Tags { - if tag[0] == "p" { - pubkeyContacts = append(pubkeyContacts, tag[1]) - } - } - case <-ctx.Done(): - goto end + for id := range internal.getEventsInRelay(hostname) { + res, _ := sys.StoreRelay.QuerySync(ctx, nostr.Filter{IDs: []string{id}}) + if len(res) == 0 { + internal.notCached(id) + continue + } + limit-- + if !yield(res[0]) { + return + } + if limit == 0 { + return } } - end: - cancel() - if len(pubkeyContacts) > 0 { - cache.SetJSONWithTTL("cc:"+pubkey, pubkeyContacts, time.Hour*6) + if limit > 0 { + limit = max(limit, 50) + + if relay, err := sys.Pool.EnsureRelay(hostname); err == nil { + ch, err := relay.QueryEvents(ctx, nostr.Filter{ + Kinds: []int{1}, + Limit: limit, + }) + if err != nil { + log.Error().Err(err).Stringer("relay", relay).Msg("failed to fetch relay notes") + return + } + + for evt := range ch { + sys.StoreRelay.Publish(ctx, *evt) + internal.attachRelaysToEvent(evt.ID, hostname) + if !yield(evt) { + return + } + } + } } } - return unique(pubkeyContacts) } func relaysPretty(ctx context.Context, pubkey string) []string { diff --git a/null_cache.go b/null_cache.go deleted file mode 100644 index fd35f6c..0000000 --- a/null_cache.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build nocache - -package main - -import ( - "time" - - "github.com/fiatjaf/eventstore" - "github.com/fiatjaf/eventstore/nullstore" -) - -var ( - cache = Cache{} - db eventstore.Store = &nullstore.NullStore{} -) - -type Cache struct{} - -func (c *Cache) initialize() func() { return func() {} } -func (c *Cache) Get(key string) ([]byte, bool) { return nil, false } -func (c *Cache) GetJSON(key string, recv any) bool { return false } -func (c *Cache) Set(key string, value []byte) {} -func (c *Cache) SetJSON(key string, value any) {} -func (c *Cache) SetWithTTL(key string, value []byte, ttl time.Duration) {} -func (c *Cache) SetJSONWithTTL(key string, value any, ttl time.Duration) {} -func (c *Cache) GetPaginatedKeys(prefix string, page int, size int) []string { return []string{} } -func (c *Cache) Delete(key string) {} diff --git a/redirect.go b/redirect.go index ba42ee1..427727f 100644 --- a/redirect.go +++ b/redirect.go @@ -5,6 +5,7 @@ import ( "math/rand" "net/http" + "fiatjaf.com/leafdb" "github.com/nbd-wtf/go-nostr/nip19" ) @@ -25,13 +26,12 @@ func redirectToRandom(w http.ResponseWriter, r *http.Request) { // 50% of chance of picking a pubkey if ra := rand.Intn(2); ra == 0 { - set := make([]string, 0, 50) - for _, pubkey := range cache.GetPaginatedKeys("pa", 1, 50) { - set = append(set, pubkey) - } - if s := len(set); s > 0 { - pick := set[rand.Intn(s)] - npub, _ := nip19.EncodePublicKey(pick) + + params := leafdb.AnyQuery("pubkey-archive") + params.Skip = rand.Intn(50) + for val := range internal.View(params) { + pka := val.(*PubKeyArchive) + npub, _ := nip19.EncodePublicKey(pka.Pubkey) target = "/" + npub return } @@ -39,10 +39,8 @@ func redirectToRandom(w http.ResponseWriter, r *http.Request) { // otherwise try to pick an event const RELAY = "wss://nostr.wine" - lastEvents := relayLastNotes(r.Context(), RELAY, false) - if s := len(lastEvents); s > 0 { - pick := lastEvents[rand.Intn(s)] - nevent, _ := nip19.EncodeEvent(pick.ID, []string{RELAY}, pick.PubKey) + for evt := range relayLastNotes(r.Context(), RELAY, 1) { + nevent, _ := nip19.EncodeEvent(evt.ID, []string{RELAY}, evt.PubKey) target = "/" + nevent return } diff --git a/render_archive.go b/render_archive.go index a496fe7..c506631 100644 --- a/render_archive.go +++ b/render_archive.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "fiatjaf.com/leafdb" "github.com/nbd-wtf/go-nostr/nip19" ) @@ -28,9 +29,12 @@ func renderArchive(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path[1:], "npubs-archive") { pathPrefix = "" data = make([]string, 0, 5000) - keys := cache.GetPaginatedKeys("pa:", page, 5000) - for _, key := range keys { - npub, _ := nip19.EncodePublicKey(key[3:]) + params := leafdb.AnyQuery("pubkey-archive") + params.Skip = (page - 1) * 5000 + params.Limit = 5000 + for val := range internal.View(params) { + pka := val.(*PubKeyArchive) + npub, _ := nip19.EncodePublicKey(pka.Pubkey) data = append(data, npub) } } else if strings.HasPrefix(r.URL.Path[1:], "relays-archive") { diff --git a/render_profile.go b/render_profile.go index 6a6475c..b3e7a3e 100644 --- a/render_profile.go +++ b/render_profile.go @@ -35,6 +35,8 @@ func renderProfile(ctx context.Context, r *http.Request, w http.ResponseWriter, } errorTemplate(ErrorPageParams{Errors: errMsg}).Render(ctx, w) return + } else { + internal.scheduleEventExpiration(profile.Event.ID) } createdAt := profile.Event.CreatedAt.Time().Format("2006-01-02T15:04:05Z07:00") diff --git a/render_relay.go b/render_relay.go index 2811607..fc800ae 100644 --- a/render_relay.go +++ b/render_relay.go @@ -41,16 +41,24 @@ func renderRelayPage(w http.ResponseWriter, r *http.Request) { } // last notes - lastNotes := relayLastNotes(r.Context(), hostname, isSitemap) - renderableLastNotes := make([]EnhancedEvent, len(lastNotes)) - lastEventAt := time.Now() - if len(lastNotes) > 0 { - lastEventAt = time.Unix(int64(lastNotes[0].CreatedAt), 0) + limit := 50 + if isSitemap { + limit = 500 } - for i, levt := range lastNotes { - ee := NewEnhancedEvent(nil, levt) + renderableLastNotes := make([]EnhancedEvent, 0, limit) + var lastEventAt *time.Time + for evt := range relayLastNotes(r.Context(), hostname, limit) { + ee := NewEnhancedEvent(nil, evt) ee.relays = []string{"wss://" + hostname} - renderableLastNotes[i] = ee + renderableLastNotes = append(renderableLastNotes, ee) + if lastEventAt == nil { + last := time.Unix(int64(evt.CreatedAt), 0) + lastEventAt = &last + } + } + if lastEventAt == nil { + now := time.Now() + lastEventAt = &now } if len(renderableLastNotes) != 0 { diff --git a/render_sitemap_index.go b/render_sitemap_index.go index b752f10..8414b24 100644 --- a/render_sitemap_index.go +++ b/render_sitemap_index.go @@ -3,14 +3,17 @@ package main import ( "net/http" + "fiatjaf.com/leafdb" "github.com/nbd-wtf/go-nostr/nip19" ) func renderSitemapIndex(w http.ResponseWriter, r *http.Request) { npubs := make([]string, 0, 5000) - keys := cache.GetPaginatedKeys("pa:", 1, 5000) - for _, key := range keys { - npub, _ := nip19.EncodePublicKey(key[3:]) + params := leafdb.AnyQuery("pubkey-archive") + params.Limit = 5000 + for val := range internal.View(params) { + pka := val.(*PubKeyArchive) + npub, _ := nip19.EncodePublicKey(pka.Pubkey) npubs = append(npubs, npub) } diff --git a/routines.go b/routines.go index 569b691..16f39c7 100644 --- a/routines.go +++ b/routines.go @@ -2,7 +2,6 @@ package main import ( "context" - "strings" "time" "github.com/nbd-wtf/go-nostr" @@ -11,16 +10,29 @@ import ( var npubsArchive = make([]string, 0, 5000) func updateArchives(ctx context.Context) { - // do this so we don't run this every time we restart it locally - - time.Sleep(10 * time.Minute) - for { - loadNpubsArchive(ctx) select { case <-ctx.Done(): return - case <-time.After(24 * time.Hour): + case <-time.After(24 * time.Hour * 3): + log.Debug().Msg("refreshing the npubs archive") + + for _, pubkey := range s.TrustedPubKeys { + ctx, cancel := context.WithTimeout(ctx, time.Second*4) + follows := sys.FetchFollowList(ctx, pubkey) + fla := &FollowListArchive{ + Source: pubkey, + Pubkeys: make([]string, 0, 2000), + } + for _, follow := range follows.Items { + fla.Pubkeys = append(fla.Pubkeys, follow.Pubkey) + } + cancel() + + if err := internal.overwriteFollowListArchive(fla); err != nil { + log.Fatal().Err(err).Msg("failed to overwrite archived pubkeys") + } + } } } } @@ -30,52 +42,21 @@ func deleteOldCachedEvents(ctx context.Context) { select { case <-ctx.Done(): return - case <-time.After(time.Hour): + case <-time.After(time.Hour * 6): log.Debug().Msg("deleting old cached events") - now := time.Now().Unix() - for _, key := range cache.GetPaginatedKeys("ttl:", 1, 500) { - spl := strings.Split(key, ":") - if len(spl) != 2 { - log.Error().Str("key", key).Msg("broken 'ttl:' key") - continue - } - - var expires int64 - if ok := cache.GetJSON(key, &expires); !ok { - log.Error().Str("key", key).Msg("failed to get 'ttl:' key") - continue - } - - if expires < now { - // time to delete this - id := spl[1] - res, _ := sys.StoreRelay.QuerySync(ctx, nostr.Filter{IDs: []string{id}}) - if len(res) > 0 { - log.Debug().Msgf("deleting %s", res[0].ID) - if err := sys.Store.DeleteEvent(ctx, res[0]); err != nil { - log.Warn().Err(err).Stringer("event", res[0]).Msg("failed to delete") + if ids, err := internal.deleteExpiredEvents(nostr.Now()); err != nil { + log.Fatal().Err(err).Msg("failed to delete expired events") + } else { + if ch, err := sys.Store.QueryEvents(ctx, nostr.Filter{IDs: ids}); err != nil { + log.Fatal().Err(err).Strs("ids", ids).Msg("fail to delete cached events") + } else { + for evt := range ch { + if err := sys.Store.DeleteEvent(ctx, evt); err != nil { + log.Error().Err(err).Stringer("event", evt).Msg("failed to delete this cached event") } } - cache.Delete(key) } } } } } - -func loadNpubsArchive(ctx context.Context) { - log.Debug().Msg("refreshing the npubs archive") - - contactsArchive := make([]string, 0, 500) - for _, pubkey := range s.TrustedPubKeys { - ctx, cancel := context.WithTimeout(ctx, time.Second*4) - pubkeyContacts := contactsForPubkey(ctx, pubkey) - contactsArchive = append(contactsArchive, pubkeyContacts...) - cancel() - } - - for _, contact := range unique(contactsArchive) { - log.Debug().Msgf("adding contact %s", contact) - cache.SetWithTTL("pa:"+contact, nil, time.Hour*24*90) - } -} diff --git a/utils.go b/utils.go index 3cad08e..2dc93d6 100644 --- a/utils.go +++ b/utils.go @@ -21,7 +21,6 @@ import ( "github.com/nbd-wtf/go-nostr" "github.com/nbd-wtf/go-nostr/nip19" - "github.com/nbd-wtf/go-nostr/sdk" ) const ( @@ -181,43 +180,6 @@ func getPreviewStyle(r *http.Request) Style { } } -func attachRelaysToEvent(eventId string, relays ...string) []string { - key := "rls:" + eventId - existingRelays := make([]string, 0, 10) - if exists := cache.GetJSON(key, &existingRelays); exists { - relays = unique(append(existingRelays, relays...)) - } - - // cleanup - filtered := make([]string, 0, len(relays)) - for _, relay := range relays { - if sdk.IsVirtualRelay(relay) { - continue - } - filtered = append(filtered, relay) - } - - cache.SetJSONWithTTL(key, filtered, time.Hour*24*7) - return filtered -} - -func getRelaysForEvent(eventId string) []string { - key := "rls:" + eventId - relays := make([]string, 0, 10) - cache.GetJSON(key, &relays) - return relays -} - -func scheduleEventExpiration(id string, ts time.Duration) { - key := "ttl:" + id - nextExpiration := time.Now().Add(ts).Unix() - var currentExpiration int64 - if exists := cache.GetJSON(key, ¤tExpiration); exists { - return - } - cache.SetJSON(key, nextExpiration) -} - func replaceURLsWithTags(input string, imageReplacementTemplate, videoReplacementTemplate string, skipLinks bool) string { return urlMatcher.ReplaceAllStringFunc(input, func(match string) string { switch {