aljaz/crawler_v2
1
0
Fork 0
mirror of https://github.com/aljazceru/crawler_v2.git synced 2025-12-17 07:24:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

added print stats to arbiter

Browse Source 
more validation for activation ratio
changed > to >= for activation
This commit is contained in:
pippellia-btc
2025-09-19 17:34:03 +02:00
parent ab9f946f1a
commit 3940a2462d
1 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/pipe/arbiter.go
View File

@@ -24,6 +24,7 @@ type ArbiterConfig struct {
Demotion float64 `envconfig:"ARBITER_DEMOTION"` Demotion float64 `envconfig:"ARBITER_DEMOTION"`
PromotionWait time.Duration `envconfig:"ARBITER_PROMOTION_WAIT"` PromotionWait time.Duration `envconfig:"ARBITER_PROMOTION_WAIT"`
PingWait time.Duration `envconfig:"ARBITER_PING_WAIT"` PingWait time.Duration `envconfig:"ARBITER_PING_WAIT"`
PrintStats bool `envconfig:"ARBITER_PRINT_STATS"`
} }
func NewArbiterConfig() ArbiterConfig { func NewArbiterConfig() ArbiterConfig {
@@ -33,6 +34,7 @@ func NewArbiterConfig() ArbiterConfig {
Demotion: 1.05, Demotion: 1.05,
PromotionWait: time.Hour, PromotionWait: time.Hour,
PingWait: time.Minute, PingWait: time.Minute,
PrintStats: true,
} }
} }
@@ -41,6 +43,10 @@ func (c ArbiterConfig) Validate() error {
return errors.New("activation ratio cannot be negative") return errors.New("activation ratio cannot be negative")
} }
if c.Activation > 1 {
return errors.New("activation ratio cannot be greater than 1")
}
if c.Promotion < 0 { if c.Promotion < 0 {
return errors.New("promotion multiplier cannot be negative") return errors.New("promotion multiplier cannot be negative")
} }
@@ -73,6 +79,7 @@ func (c ArbiterConfig) Print() {
fmt.Printf(" Demotion: %f\n", c.Demotion) fmt.Printf(" Demotion: %f\n", c.Demotion)
fmt.Printf(" PromotionWait: %v\n", c.PromotionWait) fmt.Printf(" PromotionWait: %v\n", c.PromotionWait)
fmt.Printf(" PingWait: %v\n", c.PingWait) fmt.Printf(" PingWait: %v\n", c.PingWait)
fmt.Printf(" PrintStats: %v\n", c.PrintStats)
} }
// Arbiter activates when the % of walks changed is greater than a threshold. Then it: // Arbiter activates when the % of walks changed is greater than a threshold. Then it:
@@ -107,15 +114,18 @@ func Arbiter(
changed := WalksTracker.Load() changed := WalksTracker.Load()
changeRatio := float64(changed) / float64(total) changeRatio := float64(changed) / float64(total)
if changeRatio > config.Activation { if changeRatio >= config.Activation {
promoted, demoted, err := arbiterScan(ctx, config, db, forward) promoted, demoted, err := arbiterScan(ctx, config, db, forward)
if err != nil && ctx.Err() == nil { if err != nil && ctx.Err() == nil {
log.Printf("Arbiter: %v", err) log.Printf("Arbiter: %v", err)
} }
WalksTracker.Store(0) // resets tracker if config.PrintStats {
log.Printf("Arbiter: promoted %d, demoted %d", promoted, demoted) log.Printf("Arbiter: promoted %d, demoted %d", promoted, demoted)
} }
WalksTracker.Store(0) // resets tracker
}
} }
} }
} }