diff --git a/cmd/crawl/main.go b/cmd/crawl/main.go index 83177e7..9cd7de1 100644 --- a/cmd/crawl/main.go +++ b/cmd/crawl/main.go @@ -11,7 +11,6 @@ import ( "github.com/pippellia-btc/nastro/sqlite" "github.com/vertex-lab/crawler_v2/pkg/config" - "github.com/vertex-lab/crawler_v2/pkg/graph" "github.com/vertex-lab/crawler_v2/pkg/pipe" "github.com/vertex-lab/crawler_v2/pkg/redb" "github.com/vertex-lab/crawler_v2/pkg/store" @@ -61,13 +60,17 @@ func main() { } if nodes == 0 { - if err := initGraph(ctx, db, config.InitPubkeys); err != nil { + log.Println("initializing from empty database...") + + if err := pipe.InitGraph(ctx, db, config.InitPubkeys); err != nil { panic(err) } for _, pk := range config.InitPubkeys { fetcherQueue <- pk } + + log.Printf("correctly added %d pubkeys", len(config.InitPubkeys)) } if config.PrintStats { @@ -110,32 +113,6 @@ func main() { consumers.Wait() } -func initGraph(ctx context.Context, db redb.RedisDB, pubkeys []string) error { - if len(pubkeys) == 0 { - panic("init pubkeys are empty: impossible to initialize") - } - log.Println("initialize from empty database...") - - var initNodes = make([]graph.ID, len(pubkeys)) - var err error - - for i, pk := range pubkeys { - initNodes[i], err = db.AddNode(ctx, pk) - if err != nil { - panic(err) - } - } - - for _, node := range initNodes { - if err := pipe.Promote(db, node); err != nil { - panic(err) - } - } - - log.Printf("correctly added %d init pubkeys", len(pubkeys)) - return nil -} - func printStats( ctx context.Context, recorderQueue, engineQueue chan *nostr.Event, diff --git a/cmd/sync/main.go b/cmd/sync/main.go index e3572eb..bae47a1 100644 --- a/cmd/sync/main.go +++ b/cmd/sync/main.go @@ -10,7 +10,6 @@ import ( "time" "github.com/vertex-lab/crawler_v2/pkg/config" - "github.com/vertex-lab/crawler_v2/pkg/graph" "github.com/vertex-lab/crawler_v2/pkg/pipe" "github.com/vertex-lab/crawler_v2/pkg/redb" "github.com/vertex-lab/crawler_v2/pkg/store" @@ -57,30 +56,20 @@ func main() { } if nodes != 0 { - panic("refuse to run sync when redis is not empty") + panic("refuse to sync when redis is not empty") } - if len(config.InitPubkeys) == 0 { - panic("init fetcherQueue are empty: impossible to initialize") - } log.Println("initialize from empty database...") - initNodes := make([]graph.ID, len(config.InitPubkeys)) - for i, pk := range config.InitPubkeys { - initNodes[i], err = db.AddNode(ctx, pk) - if err != nil { - panic(err) - } - - fetcherQueue <- pk // add to queue + if err := pipe.InitGraph(ctx, db, config.InitPubkeys); err != nil { + panic(err) } - for _, node := range initNodes { - if err := pipe.Promote(db, node); err != nil { - panic(err) - } + for _, pk := range config.InitPubkeys { + fetcherQueue <- pk } - log.Printf("correctly added %d init fetcherQueue", len(config.InitPubkeys)) + + log.Printf("correctly added %d pubkeys", len(config.InitPubkeys)) if config.PrintStats { go printStats(ctx, builderQueue, fetcherQueue) diff --git a/pkg/pipe/utils.go b/pkg/pipe/utils.go index cebfd43..da13677 100644 --- a/pkg/pipe/utils.go +++ b/pkg/pipe/utils.go @@ -11,6 +11,8 @@ import ( "syscall" "github.com/nbd-wtf/go-nostr" + "github.com/vertex-lab/crawler_v2/pkg/graph" + "github.com/vertex-lab/crawler_v2/pkg/redb" ) var ( @@ -40,6 +42,30 @@ func HandleSignals(cancel context.CancelFunc) { cancel() } +// InitGraph by adding and promoting the provided pubkeys. +func InitGraph(ctx context.Context, db redb.RedisDB, pubkeys []string) error { + if len(pubkeys) == 0 { + return fmt.Errorf("InitGraph: init pubkeys are empty") + } + + var initNodes = make([]graph.ID, len(pubkeys)) + var err error + + for i, pk := range pubkeys { + initNodes[i], err = db.AddNode(ctx, pk) + if err != nil { + return fmt.Errorf("InitGraph: %v", err) + } + } + + for _, node := range initNodes { + if err := Promote(db, node); err != nil { + return fmt.Errorf("InitGraph: %v", err) + } + } + return nil +} + // Shutdown iterates over the relays in the pool and closes all connections. func shutdown(pool *nostr.SimplePool) { pool.Relays.Range(func(_ string, relay *nostr.Relay) bool {