added sync cmd

pkg/config/config.go

@@ -0,0 +1,204 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github/pippellia-btc/crawler/pkg/pipe"
+
+	_ "github.com/joho/godotenv/autoload" // autoloading .env
+	"github.com/nbd-wtf/go-nostr"
+)
+
+type SystemConfig struct {
+	RedisAddress string
+	SQLiteURL    string
+
+	EventsCapacity  int
+	PubkeysCapacity int
+
+	InitPubkeys []string // only used during initialization
+}
+
+func NewSystemConfig() SystemConfig {
+	return SystemConfig{
+		RedisAddress:    "localhost:6379",
+		SQLiteURL:       "events.sqlite",
+		EventsCapacity:  1000,
+		PubkeysCapacity: 1000,
+	}
+}
+
+func (c SystemConfig) Print() {
+	fmt.Println("System:")
+	fmt.Printf("  RedisAddress: %s\n", c.RedisAddress)
+	fmt.Printf("  SQLiteURL: %s\n", c.SQLiteURL)
+	fmt.Printf("  EventsCapacity: %d\n", c.EventsCapacity)
+	fmt.Printf("  PubkeysCapacity: %d\n", c.PubkeysCapacity)
+	fmt.Printf("  InitPubkeys: %v\n", c.InitPubkeys)
+}
+
+// The configuration parameters for the system and the main processes
+type Config struct {
+	SystemConfig
+	Firehose pipe.FirehoseConfig
+	Fetcher  pipe.FetcherConfig
+	Arbiter  pipe.ArbiterConfig
+	Engine   pipe.EngineConfig
+}
+
+// New returns a config with default parameters
+func New() *Config {
+	return &Config{
+		SystemConfig: NewSystemConfig(),
+		Firehose:     pipe.NewFirehoseConfig(),
+		Fetcher:      pipe.NewFetcherConfig(),
+		Arbiter:      pipe.NewArbiterConfig(),
+		Engine:       pipe.NewEngineConfig(),
+	}
+}
+
+func (c *Config) Print() {
+	c.SystemConfig.Print()
+	c.Firehose.Print()
+	c.Fetcher.Print()
+	c.Arbiter.Print()
+	c.Engine.Print()
+}
+
+// Load reads the enviroment variables and parses them into a [Config] struct
+func Load() (*Config, error) {
+	var config = New()
+	var err error
+
+	for _, item := range os.Environ() {
+		keyVal := strings.SplitN(item, "=", 2)
+		key, val := keyVal[0], keyVal[1]
+
+		switch key {
+		case "REDIS_ADDRESS":
+			config.RedisAddress = val
+
+		case "SQLITE_URL":
+			config.SQLiteURL = val
+
+		case "EVENTS_CAPACITY":
+			config.EventsCapacity, err = strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "PUBKEYS_CAPACITY":
+			config.PubkeysCapacity, err = strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "INIT_PUBKEYS":
+			pubkeys := strings.Split(val, ",")
+			for _, pk := range pubkeys {
+				if !nostr.IsValidPublicKey(pk) {
+					return nil, fmt.Errorf("pubkey %s is not valid", pk)
+				}
+			}
+
+			config.InitPubkeys = pubkeys
+
+		case "FIREHOSE_OFFSET":
+			offset, err := strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+			config.Fetcher.Interval = time.Duration(offset) * time.Second
+
+		case "RELAYS":
+			relays := strings.Split(val, ",")
+			if len(relays) == 0 {
+				return nil, fmt.Errorf("relay list is empty")
+			}
+
+			for _, relay := range relays {
+				if !nostr.IsValidRelayURL(relay) {
+					return nil, fmt.Errorf("relay \"%s\" is not a valid url", relay)
+				}
+			}
+
+			config.Firehose.Relays = relays
+			config.Fetcher.Relays = relays
+
+		case "FETCHER_BATCH":
+			config.Fetcher.Batch, err = strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "FETCHER_INTERVAL":
+			interval, err := strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+			config.Fetcher.Interval = time.Duration(interval) * time.Second
+
+		case "ARBITER_ACTIVATION":
+			config.Arbiter.Activation, err = strconv.ParseFloat(val, 64)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "ARBITER_PROMOTION":
+			config.Arbiter.Promotion, err = strconv.ParseFloat(val, 64)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "ARBITER_DEMOTION":
+			config.Arbiter.Demotion, err = strconv.ParseFloat(val, 64)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "ARBITER_PING_WAIT":
+			wait, err := strconv.ParseInt(val, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+			config.Arbiter.PingWait = time.Duration(wait) * time.Second
+
+		case "ARBITER_PROMOTION_WAIT":
+			wait, err := strconv.ParseInt(val, 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+			config.Arbiter.PromotionWait = time.Duration(wait) * time.Second
+
+		case "ENGINE_PRINT_EVERY":
+			config.Engine.PrintEvery, err = strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "ENGINE_UPDATER_CAPACITY":
+			config.Engine.UpdaterCapacity, err = strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "ENGINE_CACHE_CAPACITY":
+			config.Engine.CacheCapacity, err = strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+
+		case "ENGINE_ARCHIVE_CAPACITY":
+			config.Engine.ArchiverCapacity, err = strconv.Atoi(val)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing %v: %v", keyVal, err)
+			}
+		}
+	}
+
+	return config, nil
+}

pkg/pipe/intake.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	relevantKinds = []int{
+	Kinds = []int{
 		//nostr.KindProfileMetadata,
 		nostr.KindFollowList,
 	}
@@ -98,7 +98,7 @@ func (b *buffer) Contains(ID string) bool {
 	return slices.Contains(b.IDs, ID)
 }
 
-// Firehose connects to a list of relays and pulls [relevantKinds] events that are newer than [FirehoseConfig.Since].
+// Firehose connects to a list of relays and pulls [Kinds] events that are newer than [FirehoseConfig.Since].
 // It discards events from unknown pubkeys as an anti-spam mechanism.
 func Firehose(ctx context.Context, config FirehoseConfig, check PubkeyChecker, send func(*nostr.Event) error) {
 	defer log.Println("Firehose: shutting down...")
@@ -107,7 +107,7 @@ func Firehose(ctx context.Context, config FirehoseConfig, check PubkeyChecker, s
 	defer shutdown(pool)
 
 	filter := nostr.Filter{
-		Kinds: relevantKinds,
+		Kinds: Kinds,
 		Since: config.Since(),
 	}
 
@@ -218,7 +218,7 @@ func Fetcher(ctx context.Context, config FetcherConfig, pubkeys <-chan string, s
 	}
 }
 
-// fetch queries the [relevantKinds] of the specified pubkeys.
+// fetch queries the [Kinds] of the specified pubkeys.
 func fetch(ctx context.Context, pool *nostr.SimplePool, relays, pubkeys []string) ([]*nostr.Event, error) {
 	if len(pubkeys) == 0 {
 		return nil, nil
@@ -228,7 +228,7 @@ func fetch(ctx context.Context, pool *nostr.SimplePool, relays, pubkeys []string
 	defer cancel()
 
 	filter := nostr.Filter{
-		Kinds:   relevantKinds,
+		Kinds:   Kinds,
 		Authors: pubkeys,
 	}
 

pkg/pipe/intake_test.go

@@ -37,7 +37,7 @@ func TestFetch(t *testing.T) {
 		t.Fatalf("expected error nil, got %v", err)
 	}
 
-	expected := len(pubkeys) * len(relevantKinds)
+	expected := len(pubkeys) * len(Kinds)
 	if len(events) != expected {
 		t.Fatalf("expected %d events, got %d", expected, len(events))
 	}