aljaz/crawler_v2
1
0
Fork 0
mirror of https://github.com/aljazceru/crawler_v2.git synced 2025-12-16 23:14:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

removed events and pubkey capacities,

Browse Source 
favoring single channel capacity variable
This commit is contained in:
pippellia-btc
2025-09-19 15:56:37 +02:00
parent 1de9a84ba3
commit 758a37750b
4 changed files with 11 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/crawl/main.go
View File

@@ -50,9 +50,9 @@ func main() {
panic(err) panic(err)
} }
recorderQueue := make(chan *nostr.Event, config.EventsCapacity) recorderQueue := make(chan *nostr.Event, config.ChannelCapacity)
engineQueue := make(chan *nostr.Event, config.EventsCapacity) engineQueue := make(chan *nostr.Event, config.ChannelCapacity)
fetcherQueue := make(chan string, config.PubkeysCapacity) fetcherQueue := make(chan string, config.ChannelCapacity)
nodes, err := db.NodeCount(ctx) nodes, err := db.NodeCount(ctx)
if err != nil { if err != nil {

4
cmd/sync/main.go
View File

@@ -47,8 +47,8 @@ func main() {
panic(err) panic(err)
} }
builderQueue := make(chan *nostr.Event, config.EventsCapacity) builderQueue := make(chan *nostr.Event, config.ChannelCapacity)
fetcherQueue := make(chan string, config.PubkeysCapacity) fetcherQueue := make(chan string, config.ChannelCapacity)
nodes, err := db.NodeCount(ctx) nodes, err := db.NodeCount(ctx)
if err != nil { if err != nil {

13
pkg/config/config.go
View File

@@ -15,8 +15,7 @@ import (
type SystemConfig struct { type SystemConfig struct {
RedisAddress string `envconfig:"REDIS_ADDRESS"` RedisAddress string `envconfig:"REDIS_ADDRESS"`
SQLiteURL string `envconfig:"SQLITE_URL"` SQLiteURL string `envconfig:"SQLITE_URL"`
EventsCapacity int `envconfig:"EVENTS_CAPACITY"` ChannelCapacity int `envconfig:"CHANNEL_CAPACITY"`
PubkeysCapacity int `envconfig:"PUBKEYS_CAPACITY"`
InitPubkeys []string `envconfig:"INIT_PUBKEYS"` InitPubkeys []string `envconfig:"INIT_PUBKEYS"`
PrintStats bool `envconfig:"PRINT_STATS"` PrintStats bool `envconfig:"PRINT_STATS"`
} }
@@ -25,14 +24,13 @@ func NewSystemConfig() SystemConfig {
return SystemConfig{ return SystemConfig{
RedisAddress: "localhost:6379", RedisAddress: "localhost:6379",
SQLiteURL: "events.sqlite", SQLiteURL: "events.sqlite",
EventsCapacity: 1000, ChannelCapacity: 10_000,
PubkeysCapacity: 1000,
} }
} }
func (c SystemConfig) Validate() error { func (c SystemConfig) Validate() error {
if c.EventsCapacity < 0 { if c.ChannelCapacity < 0 {
return errors.New("events: value cannot be negative") return errors.New("channel capacity cannot be negative")
} }
for _, pk := range c.InitPubkeys { for _, pk := range c.InitPubkeys {
@@ -47,8 +45,7 @@ func (c SystemConfig) Print() {
fmt.Println("System:") fmt.Println("System:")
fmt.Printf(" RedisAddress: %s\n", c.RedisAddress) fmt.Printf(" RedisAddress: %s\n", c.RedisAddress)
fmt.Printf(" SQLiteURL: %s\n", c.SQLiteURL) fmt.Printf(" SQLiteURL: %s\n", c.SQLiteURL)
fmt.Printf(" EventsCapacity: %d\n", c.EventsCapacity) fmt.Printf(" ChannelCapacity: %d\n", c.ChannelCapacity)
fmt.Printf(" PubkeysCapacity: %d\n", c.PubkeysCapacity)
fmt.Printf(" InitPubkeys: %v\n", c.InitPubkeys) fmt.Printf(" InitPubkeys: %v\n", c.InitPubkeys)
fmt.Printf(" PrintStats: %v\n", c.PrintStats) fmt.Printf(" PrintStats: %v\n", c.PrintStats)
} }

2
pkg/pipe/engine.go
View File

@@ -21,7 +21,7 @@ import (
type EngineConfig struct { type EngineConfig struct {
Archiver ArchiverConfig Archiver ArchiverConfig
Builder GraphBuilderConfig Builder GraphBuilderConfig
ChannelCapacity int `envconfig:"ENGINE_BUILDER_CAPACITY"` ChannelCapacity int `envconfig:"CHANNEL_CAPACITY"`
} }
func NewEngineConfig() EngineConfig { func NewEngineConfig() EngineConfig {