aljaz/crawler_v2
1
0
Fork 0
mirror of https://github.com/aljazceru/crawler_v2.git synced 2025-12-17 23:44:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

using cache for faster existance check in Firehose

Browse Source
This commit is contained in:
pippellia-btc
2025-09-15 17:54:01 +02:00
parent d176d8c81d
commit c489203932
3 changed files with 47 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/crawl/main.go
View File

@@ -83,7 +83,8 @@ func main() {
producers.Add(4)
go func() {
defer producers.Done()
pipe.Firehose(ctx, config.Firehose, db, pipe.Send(recorderQueue))
gate := pipe.NewExistenceGate(db)
pipe.Firehose(ctx, config.Firehose, gate, pipe.Send(recorderQueue))
}()
go func() {