aljaz/crawler_v2
1
0
Fork 0
mirror of https://github.com/aljazceru/crawler_v2.git synced 2025-12-17 07:24:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

using nastro instead of old eventstore

Browse Source
This commit is contained in:
pippellia-btc
2025-07-10 18:23:44 +02:00
parent 3cac4b98e5
commit a7b60b6867
8 changed files with 20 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/crawler/main.go
View File

@@ -9,6 +9,7 @@ import (
"sync"
"time"
"github.com/pippellia-btc/nastro/sqlite"
"github.com/vertex-lab/crawler_v2/pkg/config"
"github.com/vertex-lab/crawler_v2/pkg/graph"
"github.com/vertex-lab/crawler_v2/pkg/pipe"
@@ -16,7 +17,6 @@ import (
"github.com/nbd-wtf/go-nostr"
"github.com/redis/go-redis/v9"
"github.com/vertex-lab/relay/pkg/eventstore"
)
/*
@@ -41,7 +41,7 @@ func main() {
events := make(chan *nostr.Event, config.EventsCapacity)
pubkeys := make(chan string, config.PubkeysCapacity)
store, err := eventstore.New(config.SQLiteURL)
store, err := sqlite.New(config.SQLiteURL)
if err != nil {
panic(err)
}

4
cmd/sync/main.go
View File

@@ -9,6 +9,7 @@ import (
"sync"
"time"
"github.com/pippellia-btc/nastro/sqlite"
"github.com/vertex-lab/crawler_v2/pkg/config"
"github.com/vertex-lab/crawler_v2/pkg/graph"
"github.com/vertex-lab/crawler_v2/pkg/pipe"
@@ -16,7 +17,6 @@ import (
"github.com/nbd-wtf/go-nostr"
"github.com/redis/go-redis/v9"
"github.com/vertex-lab/relay/pkg/eventstore"
)
/*
@@ -40,7 +40,7 @@ func main() {
events := make(chan *nostr.Event, config.EventsCapacity)
pubkeys := make(chan string, config.PubkeysCapacity)
store, err := eventstore.New(config.SQLiteURL)
store, err := sqlite.New(config.SQLiteURL)
if err != nil {
panic(err)
}

4
go.mod
View File

@@ -7,9 +7,9 @@ toolchain go1.24.3
require (
github.com/joho/godotenv v1.5.1
github.com/nbd-wtf/go-nostr v0.51.12
github.com/pippellia-btc/nastro v0.1.3
github.com/pippellia-btc/slicex v0.2.4
github.com/redis/go-redis/v9 v9.8.0
github.com/vertex-lab/relay v0.4.7
)
require (
@@ -28,7 +28,7 @@ require (
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-sqlite3 v1.14.24 // indirect
github.com/mattn/go-sqlite3 v1.14.28 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect

10
go.sum
View File

@@ -43,8 +43,8 @@ github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQe
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -52,8 +52,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/nbd-wtf/go-nostr v0.51.12 h1:MRQcrShiW/cHhnYSVDQ4SIEc7DlYV7U7gg/l4H4gbbE=
github.com/nbd-wtf/go-nostr v0.51.12/go.mod h1:IF30/Cm4AS90wd1GjsFJbBqq7oD1txo+2YUFYXqK3Nc=
github.com/pippellia-btc/slicex v0.2.3 h1:QNGp1UtdlAOeTPvYrttS6tFZnFISQRcuhVMMTZqx/B4=
github.com/pippellia-btc/slicex v0.2.3/go.mod h1:fu7VjA9Cdk76wIUlkzWOYiMG8/VEs1fJiUhkKqEopd8=
github.com/pippellia-btc/nastro v0.1.3 h1:fL2AUy/1ZcBwIW9tqnlxTC3tg8e8AC/DABqv/Bpe998=
github.com/pippellia-btc/nastro v0.1.3/go.mod h1:K5EWZqmFjSvPwZceUL3VfJRMagVydXeb1Adnd1rXwkI=
github.com/pippellia-btc/slicex v0.2.4 h1:zVZ7c0pZu01LL0f+cWPrgBdPJHQ3iMyGjf7ucGoO6RY=
github.com/pippellia-btc/slicex v0.2.4/go.mod h1:fu7VjA9Cdk76wIUlkzWOYiMG8/VEs1fJiUhkKqEopd8=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -82,8 +82,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/vertex-lab/relay v0.4.7 h1:VsOmkJNfMI9iz+UonD1StHemOvgt/33COgqpJA1COEk=
github.com/vertex-lab/relay v0.4.7/go.mod h1:K3Utw2y0FhhDigT86A2gc/ZqK6MrF06mw6p0CihD4I4=
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=

7
pkg/pipe/engine.go
View File

@@ -8,13 +8,13 @@ import (
"log"
"time"
"github.com/pippellia-btc/nastro"
"github.com/pippellia-btc/slicex"
"github.com/vertex-lab/crawler_v2/pkg/graph"
"github.com/vertex-lab/crawler_v2/pkg/redb"
"github.com/vertex-lab/crawler_v2/pkg/walks"
"github.com/nbd-wtf/go-nostr"
"github.com/vertex-lab/relay/pkg/eventstore"
)
type EngineConfig struct {
@@ -44,7 +44,7 @@ func (c EngineConfig) Print() {
func Engine(
ctx context.Context,
config EngineConfig,
store *eventstore.Store,
store nastro.Store,
db redb.RedisDB,
events chan *nostr.Event) {
@@ -72,7 +72,7 @@ func Engine(
func Archiver(
ctx context.Context,
config EngineConfig,
store *eventstore.Store,
store nastro.Store,
events chan *nostr.Event,
onReplace func(*nostr.Event) error) {
@@ -204,7 +204,6 @@ func computeDelta(ctx context.Context, db redb.RedisDB, cache *walks.CachedWalke
if err != nil {
return graph.Delta{}, fmt.Errorf("failed to compute delta: %w", err)
}
return graph.NewDelta(event.Kind, author.ID, oldFollows, newFollows), nil
}

8
pkg/pipe/intake.go
View File

@@ -8,7 +8,7 @@ import (
"time"
"github.com/nbd-wtf/go-nostr"
"github.com/vertex-lab/relay/pkg/eventstore"
"github.com/pippellia-btc/nastro"
)
var (
@@ -257,7 +257,7 @@ func fetch(ctx context.Context, pool *nostr.SimplePool, relays, pubkeys []string
func FetcherDB(
ctx context.Context,
config FetcherConfig,
store *eventstore.Store,
store nastro.Store,
pubkeys <-chan string,
send func(*nostr.Event) error) {
@@ -281,7 +281,7 @@ func FetcherDB(
continue
}
events, err := store.Query(ctx, &nostr.Filter{Kinds: Kinds, Authors: batch})
events, err := store.Query(ctx, nostr.Filter{Kinds: Kinds, Authors: batch})
if err != nil {
log.Printf("FetcherDB: %v", err)
}
@@ -300,7 +300,7 @@ func FetcherDB(
continue
}
events, err := store.Query(ctx, &nostr.Filter{Kinds: Kinds, Authors: batch})
events, err := store.Query(ctx, nostr.Filter{Kinds: Kinds, Authors: batch})
if err != nil {
log.Printf("FetcherDB: %v", err)
}

1
pkg/pipe/intake_test.go
View File

@@ -24,7 +24,6 @@ func TestFirehose(t *testing.T) {
checker := mockChecker{pubkey: pip}
config := NewFirehoseConfig()
Firehose(ctx, config, checker, print)
}

8
tests/e2e/e2e_test.go
View File

@@ -106,17 +106,14 @@ func TestPagerank(t *testing.T) {
walker := walks.NewSimpleWalker(followMap)
store := test.NewWalkStore()
fmt.Println(" > db copied")
fmt.Printf(" > generating walks...\n")
fmt.Printf("---------------------------------\n\n")
var active int
for i, ID := range nodes {
if i%1000 == 0 {
fmt.Printf("\033[1A")
fmt.Print("\033[J")
fmt.Printf("progress %d/%d...\n", i+1, len(nodes))
fmt.Printf(" > generating walks %d/%d...\n", i+1, len(nodes))
}
node, err := db.NodeByID(ctx, ID)
@@ -142,8 +139,9 @@ func TestPagerank(t *testing.T) {
expected := expectedDistance(active, len(nodes))
distance := test.Distance(original, recomputed)
fmt.Printf("expected distance %f, got %f\n", expected, distance)
fmt.Println("---------------------------------")
fmt.Printf("expected distance %f, got %f\n", expected, distance)
if distance > expected {
t.Fatalf("distance is higher than expected!")
}