crawler_v2/cmd/crawler.go

package main

import (
	"context"
	"github/pippellia-btc/crawler/pkg/graph"
	"github/pippellia-btc/crawler/pkg/pipe"
	"github/pippellia-btc/crawler/pkg/redb"
	"github/pippellia-btc/crawler/pkg/walks"
	"log"
	"os"
	"os/signal"
	"sync"
	"syscall"

	"github.com/nbd-wtf/go-nostr"
	"github.com/redis/go-redis/v9"
)

func main() {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go handleSignals(cancel)

	config, err := LoadConfig()
	if err != nil {
		panic(err)
	}

	events := make(chan *nostr.Event, config.EventsCapacity)
	pubkeys := make(chan string, config.PubkeysCapacity)

	db := redb.New(&redis.Options{Addr: config.RedisAddress})
	count, err := db.NodeCount(ctx)
	if err != nil {
		panic(err)
	}

	if count == 0 {
		log.Println("initializing crawler from empty database")

		nodes := make([]graph.ID, len(config.InitPubkeys))
		for i, pk := range config.InitPubkeys {
			nodes[i], err = db.AddNode(ctx, pk)
			if err != nil {
				panic(err)
			}

			pubkeys <- pk // add to queue
		}

		walks, err := walks.Generate(ctx, db, nodes...)
		if err != nil {
			panic(err)
		}

		if err := db.AddWalks(ctx, walks...); err != nil {
			panic(err)
		}

		log.Printf("correctly added %d init pubkeys", len(config.InitPubkeys))
	}

	_ = events

	// eventStore, err := eventstore.New(config.SQLiteURL)
	// if err != nil {
	// 	panic("failed to connect to the sqlite eventstore: " + err.Error())
	// }

	var wg sync.WaitGroup
	wg.Add(3)

	go func() {
		defer wg.Done()
		pipe.Firehose(ctx, config.Firehose, db, func(event *nostr.Event) error {
			select {
			case events <- event:
			default:
				log.Printf("Firehose: channel is full, dropping event ID %s by %s", event.ID, event.PubKey)
			}
			return nil
		})
	}()

	go func() {
		defer wg.Done()
		pipe.Fetcher(ctx, config.Fetcher, pubkeys, func(event *nostr.Event) error {
			select {
			case events <- event:
			default:
				log.Printf("Fetcher: channel is full, dropping event ID %s by %s", event.ID, event.PubKey)
			}
			return nil
		})
	}()

	go func() {
		defer wg.Done()
		pipe.Arbiter(ctx, config.Arbiter, db, func(pubkey string) error {
			select {
			case pubkeys <- pubkey:
			default:
				log.Printf("Arbiter: channel is full, dropping pubkey %s", pubkey)
			}
			return nil
		})
	}()

	log.Println("ready to process events")
	pipe.Processor(ctx, config.Processor, db, events)
	wg.Wait()
}

// handleSignals listens for OS signals and triggers context cancellation.
func handleSignals(cancel context.CancelFunc) {
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
	<-signals

	log.Println(" Signal received. Shutting down...")
	cancel()
}