diff --git a/cache.go b/cache.go index 9b52760..cb01858 100644 --- a/cache.go +++ b/cache.go @@ -8,9 +8,14 @@ import ( "time" "github.com/dgraph-io/badger" + "github.com/fiatjaf/eventstore" + eventstore_badger "github.com/fiatjaf/eventstore/badger" ) -var cache = Cache{} +var ( + cache = Cache{} + db eventstore.Store = &eventstore_badger.BadgerBackend{} +) type Cache struct { *badger.DB @@ -38,6 +43,12 @@ func (c *Cache) initialize() func() { return func() { db.Close() } } +func (c *Cache) Delete(key string) error { + return c.DB.Update(func(txn *badger.Txn) error { + return txn.Delete([]byte(key)) + }) +} + func (c *Cache) Get(key string) ([]byte, bool) { var val []byte err := c.DB.View(func(txn *badger.Txn) error { diff --git a/go.mod b/go.mod index ee65c7f..a5da135 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/apatters/go-wordwrap v1.0.0 github.com/dgraph-io/badger v1.6.2 + github.com/fiatjaf/eventstore v0.2.0 github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a github.com/kelseyhightower/envconfig v1.4.0 github.com/lukevers/freetype-go v0.0.0-20150513150840-77e276735410 @@ -28,16 +29,21 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/dgraph-io/badger/v4 v4.2.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/fiatjaf/eventstore v0.1.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.2.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect @@ -46,6 +52,7 @@ require ( github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + go.opencensus.io v0.22.5 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/go.sum b/go.sum index 0a4f0de..0ac8667 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -36,6 +37,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -54,6 +56,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3 github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= @@ -61,8 +65,8 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/fiatjaf/eventstore v0.1.0 h1:/g7VTw6dsXmjICD3rBuHNIvAammHJ5unrKJ71Dz+VTs= -github.com/fiatjaf/eventstore v0.1.0/go.mod h1:juMei5HL3HJi6t7vZjj7VdEItDPu31+GLROepdUK4tw= +github.com/fiatjaf/eventstore v0.2.0 h1:y6YEH+TQIvpjvRxFkvpvFnh0+EyaoEsZYvfi+ueW6GU= +github.com/fiatjaf/eventstore v0.2.0/go.mod h1:juMei5HL3HJi6t7vZjj7VdEItDPu31+GLROepdUK4tw= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= @@ -72,9 +76,14 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.2.0 h1:u0p9s3xLYpZCA1z5JgCkMeB34CKCMMQbM+G8Ii7YD0I= github.com/gobwas/ws v1.2.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -86,9 +95,12 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a h1:AWZzzFrqyjYlRloN6edwTLTUbKxf5flLXNuTBDm3Ews= github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -107,7 +119,11 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -179,25 +195,50 @@ github.com/tylermmorton/tmpl v0.0.0-20230817025807-fd8b24ce5c3d h1:WCBWmVpkAG1Ku github.com/tylermmorton/tmpl v0.0.0-20230817025807-fd8b24ce5c3d/go.mod h1:aFT85F39qRY7ZZT5pHU01s1Ru3o9EOmbd+UjbrxxHw4= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -205,6 +246,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -215,8 +257,22 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -238,5 +294,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= mvdan.cc/xurls/v2 v2.5.0 h1:lyBNOm8Wo71UknhUs4QTFUNNMyxy2JEIaKKo0RWOh+8= mvdan.cc/xurls/v2 v2.5.0/go.mod h1:yQgaGQ1rFtJUzkmKiHYSSfuQxqfYmd//X6PxvholpeE= diff --git a/main.go b/main.go index 7ad3679..eb77a91 100644 --- a/main.go +++ b/main.go @@ -8,18 +8,19 @@ import ( "net/http" "os" "strings" - "time" + "github.com/fiatjaf/eventstore/badger" "github.com/kelseyhightower/envconfig" "github.com/rs/cors" "github.com/rs/zerolog" ) type Settings struct { - Port string `envconfig:"PORT" default:"2999"` - Domain string `envconfig:"DOMAIN" default:"njump.me"` - DiskCachePath string `envconfig:"DISK_CACHE_PATH" default:"/tmp/njump-cache"` - TailwindDebug bool `envconfig:"TAILWIND_DEBUG"` + Port string `envconfig:"PORT" default:"2999"` + Domain string `envconfig:"DOMAIN" default:"njump.me"` + DiskCachePath string `envconfig:"DISK_CACHE_PATH" default:"/tmp/njump-cache"` + EventStorePath string `envconfig:"EVENT_STORE_PATH" default:"/tmp/njump-db"` + TailwindDebug bool `envconfig:"TAILWIND_DEBUG"` } //go:embed static/* @@ -34,22 +35,6 @@ var ( tailwindDebugStuff template.HTML ) -func updateArchives(ctx context.Context) { - // do this so we don't run this every time we restart it locally - time.Sleep(10 * time.Minute) - - for { - select { - case <-ctx.Done(): - return - default: - loadNpubsArchive(ctx) - loadRelaysArchive(ctx) - } - time.Sleep(24 * time.Hour) - } -} - func main() { err := envconfig.Process("", &s) if err != nil { @@ -89,9 +74,18 @@ func main() { // initialize disk cache defer cache.initialize()() - // initialize the function to update the npubs/relays archive + // initialize eventstore database + if badgerBackend, ok := db.(*badger.BadgerBackend); ok { + // it may be NullStore, in which case we do nothing + badgerBackend.Path = s.EventStorePath + } + db.Init() + defer db.Close() + + // initialize routines ctx := context.Background() go updateArchives(ctx) + go deleteOldCachedEvents(ctx) // routes mux := http.NewServeMux() diff --git a/nostr.go b/nostr.go index 1544211..d6cec31 100644 --- a/nostr.go +++ b/nostr.go @@ -2,12 +2,12 @@ package main import ( "context" - "encoding/json" "fmt" "math/rand" "net/url" "time" + "github.com/fiatjaf/eventstore" "github.com/nbd-wtf/go-nostr" "github.com/nbd-wtf/go-nostr/nip05" "github.com/nbd-wtf/go-nostr/nip19" @@ -24,6 +24,7 @@ var ( } everything = []string{ "wss://nostr-pub.wellorder.net", + "wss://saltivka.org", "wss://relay.damus.io", "wss://relay.nostr.bg", "wss://nostr.wine", @@ -65,14 +66,9 @@ func getRelay() string { } func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Event, []string, error) { - if b, ok := cache.Get(code); ok { - v := CachedEvent{} - err := json.Unmarshal(b, &v) - return v.Event, v.Relays, err - } + wdb := eventstore.RelayWrapper{Store: db} withRelays := false - if len(relayHints) > 0 { withRelays = true } @@ -135,6 +131,12 @@ func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Eve } } + // try to fetch in our internal eventstore first + if res, _ := wdb.QuerySync(ctx, filter); len(res) != 0 { + return res[0], nil, err + } + + // otherwise fetch from external relays if author != "" { // fetch relays for author authorRelays := relaysForPubkey(ctx, author, relays...) @@ -143,7 +145,6 @@ func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Eve } relays = append(relays, authorRelays...) } - for len(relays) < 5 { relays = append(relays, getRelay()) } @@ -187,57 +188,73 @@ func getEvent(ctx context.Context, code string, relayHints []string) (*nostr.Eve return nil, nil, fmt.Errorf("couldn't find this %s", prefix) } - cache.SetJSONWithTTL(code, CachedEvent{Event: result, Relays: successRelays}, time.Hour*24*7) + // save stuff in cache and in internal store + wdb.Publish(ctx, *result) + // save relays if we got them + attachRelaysToEvent(result, successRelays...) + // keep track of what we have to delete later + scheduleEventExpiration(result.ID, time.Hour*24*7) + return result, successRelays, nil } func authorLastNotes(ctx context.Context, pubkey string, relays []string, isSitemap bool) []*nostr.Event { - key := "" limit := 100 + store := true + useLocalStore := true if isSitemap { - key = "lns:" + pubkey limit = 50000 - } else { - key = "ln:" + pubkey + store = false + useLocalStore = false } - lastNotes := make([]*nostr.Event, 0, limit) - if ok := cache.GetJSON(key, &lastNotes); ok { - return lastNotes + filter := nostr.Filter{ + Kinds: []int{nostr.KindTextNote}, + Authors: []string{pubkey}, + Limit: limit, } + var lastNotes []*nostr.Event - ctx, cancel := context.WithTimeout(ctx, time.Second*4) - defer cancel() - - relays = append(relays, getRelay()) - relays = append(relays, getRelay()) - relays = unique(relays) - - ch := pool.SubManyEose(ctx, relays, nostr.Filters{ - { - Kinds: []int{nostr.KindTextNote}, - Authors: []string{pubkey}, - Limit: limit, - }, - }) - - for { - select { - case ie, more := <-ch: - if !more { - goto end + // fetch from external relays asynchronously + external := make(chan []*nostr.Event) + go func() { + notes := make([]*nostr.Event, 0, filter.Limit) + defer func() { + external <- notes + }() + ctx, cancel := context.WithTimeout(ctx, time.Second*4) + defer cancel() + relays = unique(append(relays, getRelay(), getRelay())) + ch := pool.SubManyEose(ctx, relays, nostr.Filters{filter}) + for { + select { + case ie, more := <-ch: + if !more { + return + } + notes = append(lastNotes, ie.Event) + if store { + db.SaveEvent(ctx, ie.Event) + attachRelaysToEvent(ie.Event, ie.Relay.URL) + scheduleEventExpiration(ie.Event.ID, time.Hour*24) + } + case <-ctx.Done(): + return } - lastNotes = append(lastNotes, ie.Event) - case <-ctx.Done(): - goto end } + }() + + // fetch from local store if available + if useLocalStore { + lastNotes, _ = eventstore.RelayWrapper{Store: db}.QuerySync(ctx, filter) + } + if len(lastNotes) < 2 { + // if we didn't get enough notes (or if we didn't even query the local store), wait for the external relays + lastNotes = <-external } -end: + // sort before returning slices.SortFunc(lastNotes, func(a, b *nostr.Event) bool { return a.CreatedAt > b.CreatedAt }) - if len(lastNotes) > 0 { - cache.SetJSONWithTTL(key, lastNotes, time.Hour*24) - } return lastNotes } diff --git a/null_cache.go b/null_cache.go index 73c975b..1fe9c6b 100644 --- a/null_cache.go +++ b/null_cache.go @@ -4,6 +4,9 @@ package main import ( "time" + + "github.com/fiatjaf/eventstore" + "github.com/fiatjaf/eventstore/nullstore" ) var cache = Cache{} @@ -18,3 +21,5 @@ func (c *Cache) SetJSON(key string, value any) {} func (c *Cache) SetWithTTL(key string, value []byte, ttl time.Duration) {} func (c *Cache) SetJSONWithTTL(key string, value any, ttl time.Duration) {} func (c *Cache) GetPaginatedKeys(prefix string, page int, size int) []string { return []string{} } + +var db eventstore.Store = nullstore.NullStore{} diff --git a/render_archive.go b/render_archive.go index 971c087..00c1d10 100644 --- a/render_archive.go +++ b/render_archive.go @@ -42,11 +42,11 @@ func renderArchive(w http.ResponseWriter, r *http.Request) { } if area == "npubs-archive" { - prefix = "pa" + prefix = "pa:" pathPrefix = "" title = "Nostr npubs archive" } else { - prefix = "ra" + prefix = "ra:" pathPrefix = "r/" title = "Nostr relays archive" } diff --git a/routines.go b/routines.go new file mode 100644 index 0000000..776d287 --- /dev/null +++ b/routines.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "strings" + "time" + + "github.com/fiatjaf/eventstore" + "github.com/nbd-wtf/go-nostr" +) + +func updateArchives(ctx context.Context) { + // do this so we don't run this every time we restart it locally + time.Sleep(10 * time.Minute) + + for { + select { + case <-ctx.Done(): + return + default: + loadNpubsArchive(ctx) + loadRelaysArchive(ctx) + } + time.Sleep(24 * time.Hour) + } +} + +func deleteOldCachedEvents(ctx context.Context) { + wdb := eventstore.RelayWrapper{Store: db} + + for { + time.Sleep(time.Hour) + log.Debug().Msg("deleting old cached events") + now := time.Now().Unix() + for _, key := range cache.GetPaginatedKeys("ttl:", 1, 500) { + spl := strings.Split(key, ":") + if len(spl) != 2 { + log.Error().Str("key", key).Msg("broken 'ttl:' key") + continue + } + + var expires int64 + if ok := cache.GetJSON(key, &expires); !ok { + log.Error().Str("key", key).Msg("failed to get 'ttl:' key") + continue + } + + if expires < now { + // time to delete this + id := spl[2] + res, _ := wdb.QuerySync(ctx, nostr.Filter{IDs: []string{id}}) + if len(res) > 0 { + log.Debug().Msgf("deleting %s", res[0].ID) + if err := db.DeleteEvent(ctx, res[0]); err != nil { + log.Warn().Err(err).Stringer("event", res[0]).Msg("failed to delete") + } + } + cache.Delete(key) + } + } + } +} + +func loadNpubsArchive(ctx context.Context) { + log.Debug().Msg("refreshing the npubs archive") + + contactsArchive := make([]string, 0, 500) + + for _, pubkey := range trustedPubKeys { + ctx, cancel := context.WithTimeout(ctx, time.Second*4) + pubkeyContacts := contactsForPubkey(ctx, pubkey) + contactsArchive = append(contactsArchive, pubkeyContacts...) + cancel() + } + + contactsArchive = unique(contactsArchive) + for _, contact := range contactsArchive { + log.Debug().Msgf("adding contact %s", contact) + cache.SetWithTTL("pa:"+contact, nil, time.Hour*24*90) + } +} + +func loadRelaysArchive(ctx context.Context) { + log.Debug().Msg("refreshing the relays archive") + + relaysArchive := make([]string, 0, 500) + + for _, pubkey := range trustedPubKeys { + ctx, cancel := context.WithTimeout(ctx, time.Second*4) + pubkeyContacts := relaysForPubkey(ctx, pubkey) + relaysArchive = append(relaysArchive, pubkeyContacts...) + cancel() + } + + relaysArchive = unique(relaysArchive) + for _, relay := range relaysArchive { + for _, excluded := range excludedRelays { + if strings.Contains(relay, excluded) { + log.Debug().Msgf("skipping relay %s", relay) + continue + } + } + if strings.Contains(relay, "/npub1") { + continue // skip relays with personalyzed query like filter.nostr.wine + } + log.Debug().Msgf("adding relay %s", relay) + cache.SetWithTTL("ra:"+relay, nil, time.Hour*24*7) + } +} diff --git a/utils.go b/utils.go index 3583b34..d3356b0 100644 --- a/utils.go +++ b/utils.go @@ -239,6 +239,27 @@ func getParentNevent(event *nostr.Event) string { return parentNevent } +func attachRelaysToEvent(event *nostr.Event, relays ...string) { + key := "rls:" + event.ID + existingRelays := make([]string, 0, 10) + if exists := cache.GetJSON(key, &existingRelays); exists { + relays = unique(append(existingRelays, relays...)) + } + cache.SetJSONWithTTL(key, relays, time.Hour*24*7) +} + +func scheduleEventExpiration(eventId string, ts time.Duration) { + key := "ttl:" + eventId + nextExpiration := time.Now().Add(ts).Unix() + var currentExpiration int64 + if exists := cache.GetJSON(key, ¤tExpiration); exists { + if nextExpiration < currentExpiration { + return + } + } + cache.SetJSON(key, nextExpiration) +} + // Rendering functions // ### ### ### ### ### ### ### ### ### ### ### @@ -471,53 +492,6 @@ func normalizeWebsiteURL(u string) string { return "https://" + u } -func loadNpubsArchive(ctx context.Context) { - log.Debug().Msg("refreshing the npubs archive") - - contactsArchive := make([]string, 0, 500) - - for _, pubkey := range trustedPubKeys { - ctx, cancel := context.WithTimeout(ctx, time.Second*4) - pubkeyContacts := contactsForPubkey(ctx, pubkey) - contactsArchive = append(contactsArchive, pubkeyContacts...) - cancel() - } - - contactsArchive = unique(contactsArchive) - for _, contact := range contactsArchive { - log.Debug().Msgf("adding contact %s", contact) - cache.SetWithTTL("pa:"+contact, nil, time.Hour*24*90) - } -} - -func loadRelaysArchive(ctx context.Context) { - log.Debug().Msg("refreshing the relays archive") - - relaysArchive := make([]string, 0, 500) - - for _, pubkey := range trustedPubKeys { - ctx, cancel := context.WithTimeout(ctx, time.Second*4) - pubkeyContacts := relaysForPubkey(ctx, pubkey) - relaysArchive = append(relaysArchive, pubkeyContacts...) - cancel() - } - - relaysArchive = unique(relaysArchive) - for _, relay := range relaysArchive { - for _, excluded := range excludedRelays { - if strings.Contains(relay, excluded) { - log.Debug().Msgf("skipping relay %s", relay) - continue - } - } - if strings.Contains(relay, "/npub1") { - continue // skip relays with personalyzed query like filter.nostr.wine - } - log.Debug().Msgf("adding relay %s", relay) - cache.SetWithTTL("ra:"+relay, nil, time.Hour*24*7) - } -} - func eventToHTML(evt *nostr.Event) template.HTML { tagsHTML := "[" for t, tag := range evt.Tags {