diff --git a/cmd/crawler/main.go b/cmd/crawler/main.go index 8f3f106..cfcfecd 100644 --- a/cmd/crawler/main.go +++ b/cmd/crawler/main.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/pippellia-btc/nastro/sqlite" "github.com/vertex-lab/crawler_v2/pkg/config" "github.com/vertex-lab/crawler_v2/pkg/graph" "github.com/vertex-lab/crawler_v2/pkg/pipe" @@ -16,7 +17,6 @@ import ( "github.com/nbd-wtf/go-nostr" "github.com/redis/go-redis/v9" - "github.com/vertex-lab/relay/pkg/eventstore" ) /* @@ -41,7 +41,7 @@ func main() { events := make(chan *nostr.Event, config.EventsCapacity) pubkeys := make(chan string, config.PubkeysCapacity) - store, err := eventstore.New(config.SQLiteURL) + store, err := sqlite.New(config.SQLiteURL) if err != nil { panic(err) } diff --git a/cmd/sync/main.go b/cmd/sync/main.go index 923d196..855bc75 100644 --- a/cmd/sync/main.go +++ b/cmd/sync/main.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/pippellia-btc/nastro/sqlite" "github.com/vertex-lab/crawler_v2/pkg/config" "github.com/vertex-lab/crawler_v2/pkg/graph" "github.com/vertex-lab/crawler_v2/pkg/pipe" @@ -16,7 +17,6 @@ import ( "github.com/nbd-wtf/go-nostr" "github.com/redis/go-redis/v9" - "github.com/vertex-lab/relay/pkg/eventstore" ) /* @@ -40,7 +40,7 @@ func main() { events := make(chan *nostr.Event, config.EventsCapacity) pubkeys := make(chan string, config.PubkeysCapacity) - store, err := eventstore.New(config.SQLiteURL) + store, err := sqlite.New(config.SQLiteURL) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index ff03426..06852a1 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ toolchain go1.24.3 require ( github.com/joho/godotenv v1.5.1 github.com/nbd-wtf/go-nostr v0.51.12 + github.com/pippellia-btc/nastro v0.1.3 github.com/pippellia-btc/slicex v0.2.4 github.com/redis/go-redis/v9 v9.8.0 - github.com/vertex-lab/relay v0.4.7 ) require ( @@ -28,7 +28,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/mailru/easyjson v0.9.0 // indirect - github.com/mattn/go-sqlite3 v1.14.24 // indirect + github.com/mattn/go-sqlite3 v1.14.28 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect diff --git a/go.sum b/go.sum index 2248a83..b92ada5 100644 --- a/go.sum +++ b/go.sum @@ -43,8 +43,8 @@ github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQe github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= -github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= -github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= +github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -52,8 +52,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nbd-wtf/go-nostr v0.51.12 h1:MRQcrShiW/cHhnYSVDQ4SIEc7DlYV7U7gg/l4H4gbbE= github.com/nbd-wtf/go-nostr v0.51.12/go.mod h1:IF30/Cm4AS90wd1GjsFJbBqq7oD1txo+2YUFYXqK3Nc= -github.com/pippellia-btc/slicex v0.2.3 h1:QNGp1UtdlAOeTPvYrttS6tFZnFISQRcuhVMMTZqx/B4= -github.com/pippellia-btc/slicex v0.2.3/go.mod h1:fu7VjA9Cdk76wIUlkzWOYiMG8/VEs1fJiUhkKqEopd8= +github.com/pippellia-btc/nastro v0.1.3 h1:fL2AUy/1ZcBwIW9tqnlxTC3tg8e8AC/DABqv/Bpe998= +github.com/pippellia-btc/nastro v0.1.3/go.mod h1:K5EWZqmFjSvPwZceUL3VfJRMagVydXeb1Adnd1rXwkI= github.com/pippellia-btc/slicex v0.2.4 h1:zVZ7c0pZu01LL0f+cWPrgBdPJHQ3iMyGjf7ucGoO6RY= github.com/pippellia-btc/slicex v0.2.4/go.mod h1:fu7VjA9Cdk76wIUlkzWOYiMG8/VEs1fJiUhkKqEopd8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -82,8 +82,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/vertex-lab/relay v0.4.7 h1:VsOmkJNfMI9iz+UonD1StHemOvgt/33COgqpJA1COEk= -github.com/vertex-lab/relay v0.4.7/go.mod h1:K3Utw2y0FhhDigT86A2gc/ZqK6MrF06mw6p0CihD4I4= golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= diff --git a/pkg/pipe/engine.go b/pkg/pipe/engine.go index 78d0984..13fd7a0 100644 --- a/pkg/pipe/engine.go +++ b/pkg/pipe/engine.go @@ -8,13 +8,13 @@ import ( "log" "time" + "github.com/pippellia-btc/nastro" "github.com/pippellia-btc/slicex" "github.com/vertex-lab/crawler_v2/pkg/graph" "github.com/vertex-lab/crawler_v2/pkg/redb" "github.com/vertex-lab/crawler_v2/pkg/walks" "github.com/nbd-wtf/go-nostr" - "github.com/vertex-lab/relay/pkg/eventstore" ) type EngineConfig struct { @@ -44,7 +44,7 @@ func (c EngineConfig) Print() { func Engine( ctx context.Context, config EngineConfig, - store *eventstore.Store, + store nastro.Store, db redb.RedisDB, events chan *nostr.Event) { @@ -72,7 +72,7 @@ func Engine( func Archiver( ctx context.Context, config EngineConfig, - store *eventstore.Store, + store nastro.Store, events chan *nostr.Event, onReplace func(*nostr.Event) error) { @@ -204,7 +204,6 @@ func computeDelta(ctx context.Context, db redb.RedisDB, cache *walks.CachedWalke if err != nil { return graph.Delta{}, fmt.Errorf("failed to compute delta: %w", err) } - return graph.NewDelta(event.Kind, author.ID, oldFollows, newFollows), nil } diff --git a/pkg/pipe/intake.go b/pkg/pipe/intake.go index 17dfe48..a5ada29 100644 --- a/pkg/pipe/intake.go +++ b/pkg/pipe/intake.go @@ -8,7 +8,7 @@ import ( "time" "github.com/nbd-wtf/go-nostr" - "github.com/vertex-lab/relay/pkg/eventstore" + "github.com/pippellia-btc/nastro" ) var ( @@ -257,7 +257,7 @@ func fetch(ctx context.Context, pool *nostr.SimplePool, relays, pubkeys []string func FetcherDB( ctx context.Context, config FetcherConfig, - store *eventstore.Store, + store nastro.Store, pubkeys <-chan string, send func(*nostr.Event) error) { @@ -281,7 +281,7 @@ func FetcherDB( continue } - events, err := store.Query(ctx, &nostr.Filter{Kinds: Kinds, Authors: batch}) + events, err := store.Query(ctx, nostr.Filter{Kinds: Kinds, Authors: batch}) if err != nil { log.Printf("FetcherDB: %v", err) } @@ -300,7 +300,7 @@ func FetcherDB( continue } - events, err := store.Query(ctx, &nostr.Filter{Kinds: Kinds, Authors: batch}) + events, err := store.Query(ctx, nostr.Filter{Kinds: Kinds, Authors: batch}) if err != nil { log.Printf("FetcherDB: %v", err) } diff --git a/pkg/pipe/intake_test.go b/pkg/pipe/intake_test.go index 007f3bc..51b4329 100644 --- a/pkg/pipe/intake_test.go +++ b/pkg/pipe/intake_test.go @@ -24,7 +24,6 @@ func TestFirehose(t *testing.T) { checker := mockChecker{pubkey: pip} config := NewFirehoseConfig() - Firehose(ctx, config, checker, print) } diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index e51c4ab..fe703b1 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -106,17 +106,14 @@ func TestPagerank(t *testing.T) { walker := walks.NewSimpleWalker(followMap) store := test.NewWalkStore() - fmt.Println(" > db copied") - fmt.Printf(" > generating walks...\n") - fmt.Printf("---------------------------------\n\n") var active int for i, ID := range nodes { if i%1000 == 0 { fmt.Printf("\033[1A") fmt.Print("\033[J") - fmt.Printf("progress %d/%d...\n", i+1, len(nodes)) + fmt.Printf(" > generating walks %d/%d...\n", i+1, len(nodes)) } node, err := db.NodeByID(ctx, ID) @@ -142,8 +139,9 @@ func TestPagerank(t *testing.T) { expected := expectedDistance(active, len(nodes)) distance := test.Distance(original, recomputed) - fmt.Printf("expected distance %f, got %f\n", expected, distance) + fmt.Println("---------------------------------") + fmt.Printf("expected distance %f, got %f\n", expected, distance) if distance > expected { t.Fatalf("distance is higher than expected!") }