aljaz/crawler_v2
1
0
Fork 0
mirror of https://github.com/aljazceru/crawler_v2.git synced 2025-12-16 23:14:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

added print stats to arbiter

Browse Source 
more validation for activation ratio
changed > to >= for activation
This commit is contained in:
pippellia-btc
2025-09-19 17:34:03 +02:00
parent ab9f946f1a
commit 3940a2462d
1 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/pipe/arbiter.go
View File

@@ -24,6 +24,7 @@ type ArbiterConfig struct {
Demotion float64 `envconfig:"ARBITER_DEMOTION"`
PromotionWait time.Duration `envconfig:"ARBITER_PROMOTION_WAIT"`
PingWait time.Duration `envconfig:"ARBITER_PING_WAIT"`
PrintStats bool `envconfig:"ARBITER_PRINT_STATS"`
}
func NewArbiterConfig() ArbiterConfig {
@@ -33,6 +34,7 @@ func NewArbiterConfig() ArbiterConfig {
Demotion: 1.05,
PromotionWait: time.Hour,
PingWait: time.Minute,
PrintStats: true,
}
}
@@ -41,6 +43,10 @@ func (c ArbiterConfig) Validate() error {
return errors.New("activation ratio cannot be negative")
}
if c.Activation > 1 {
return errors.New("activation ratio cannot be greater than 1")
}
if c.Promotion < 0 {
return errors.New("promotion multiplier cannot be negative")
}
@@ -73,6 +79,7 @@ func (c ArbiterConfig) Print() {
fmt.Printf(" Demotion: %f\n", c.Demotion)
fmt.Printf(" PromotionWait: %v\n", c.PromotionWait)
fmt.Printf(" PingWait: %v\n", c.PingWait)
fmt.Printf(" PrintStats: %v\n", c.PrintStats)
}
// Arbiter activates when the % of walks changed is greater than a threshold. Then it:
@@ -107,14 +114,17 @@ func Arbiter(
changed := WalksTracker.Load()
changeRatio := float64(changed) / float64(total)
if changeRatio > config.Activation {
if changeRatio >= config.Activation {
promoted, demoted, err := arbiterScan(ctx, config, db, forward)
if err != nil && ctx.Err() == nil {
log.Printf("Arbiter: %v", err)
}
if config.PrintStats {
log.Printf("Arbiter: promoted %d, demoted %d", promoted, demoted)
}
WalksTracker.Store(0) // resets tracker
log.Printf("Arbiter: promoted %d, demoted %d", promoted, demoted)
}
}
}