aljaz/crawler_v2
1
0
Fork 0
mirror of https://github.com/aljazceru/crawler_v2.git synced 2025-12-17 07:24:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

implemented cached walker

Browse Source
This commit is contained in:
pippellia-btc
2025-06-03 15:35:59 +02:00
parent 7d82354540
commit c7b0d8ff94
7 changed files with 306 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/pagerank/pagerank.go
View File

@@ -117,19 +117,27 @@ func Personalized(
return map[graph.ID]float64{source: 1.0}, nil
}
followByNode, err := loader.BulkFollows(ctx, follows)
bulk, err := loader.BulkFollows(ctx, follows)
if err != nil {
return nil, fmt.Errorf("Personalized: failed to fetch the two-hop network of source: %w", err)
}
walker := walks.NewCachedWalker(follows, followByNode, loader)
targetWalks := int(float64(targetLenght) * (1 - walks.Alpha))
walker := walks.NewWalker(
walks.WithCapacity(10000),
walks.WithFallback(loader),
)
walks, err := loader.WalksVisitingAny(ctx, append(follows, source), targetWalks)
if err := walker.Load(follows, bulk); err != nil {
return nil, fmt.Errorf("Personalized: failed to load the two-hop network of source: %w", err)
}
targetWalks := int(float64(targetLenght) * (1 - walks.Alpha))
visiting, err := loader.WalksVisitingAny(ctx, append(follows, source), targetWalks)
if err != nil {
return nil, fmt.Errorf("Personalized: failed to fetch the walk: %w", err)
}
pool := newWalkPool(walks)
pool := newWalkPool(visiting)
walk, err := personalizedWalk(ctx, walker, pool, source, targetLenght)
if err != nil {