mirror of
https://github.com/aljazceru/crawler_v2.git
synced 2025-12-17 07:24:21 +01:00
database code dump
This commit is contained in:
pippellia-btc
37
go.mod
37
go.mod
@@ -1,3 +1,38 @@
|
||||
module github/pippellia-btc/crawler
|
||||
|
||||
go 1.23.1
|
||||
go 1.24.1
|
||||
|
||||
toolchain go1.24.3
|
||||
|
||||
require (
|
||||
github.com/nbd-wtf/go-nostr v0.51.12
|
||||
github.com/redis/go-redis/v9 v9.8.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
|
||||
github.com/bytedance/sonic v1.13.1 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
github.com/coder/websocket v1.8.12 // indirect
|
||||
github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/mailru/easyjson v0.9.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
|
||||
golang.org/x/sys v0.31.0 // indirect
|
||||
)
|
||||
|
||||
89
go.sum
Normal file
89
go.sum
Normal file
@@ -0,0 +1,89 @@
|
||||
github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 h1:ClzzXMDDuUbWfNNZqGeYq4PnYOlwlOVIvSyNaIy0ykg=
|
||||
github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3/go.mod h1:we0YA5CsBbH5+/NUzC/AlMmxaDtWlXeNsqrwXjTzmzA=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ=
|
||||
github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ=
|
||||
github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
|
||||
github.com/bytedance/sonic v1.13.1 h1:Jyd5CIvdFnkOWuKXr+wm4Nyk2h0yAFsr8ucJgEasO3g=
|
||||
github.com/bytedance/sonic v1.13.1/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
|
||||
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY=
|
||||
github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
|
||||
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo=
|
||||
github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
|
||||
github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
|
||||
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/nbd-wtf/go-nostr v0.51.12 h1:MRQcrShiW/cHhnYSVDQ4SIEc7DlYV7U7gg/l4H4gbbE=
|
||||
github.com/nbd-wtf/go-nostr v0.51.12/go.mod h1:IF30/Cm4AS90wd1GjsFJbBqq7oD1txo+2YUFYXqK3Nc=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg=
|
||||
github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA=
|
||||
github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI=
|
||||
github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
|
||||
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
|
||||
golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
|
||||
golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
|
||||
golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
|
||||
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
|
||||
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
@@ -1,33 +1,70 @@
|
||||
package graph
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// types of status
|
||||
StatusActive string = "active" // meaning, we generate random walks for this node
|
||||
StatusInactive string = "inactive"
|
||||
|
||||
// internal record kinds
|
||||
Addition int = -3
|
||||
Promotion int = -2
|
||||
Demotion int = -1
|
||||
)
|
||||
|
||||
type ID string
|
||||
|
||||
// Delta represent the changes a Node made to its follow list.
|
||||
// It Removed some nodes, and Added some others.
|
||||
// This means the old follow list is Removed + Common, while the new is Common + Added
|
||||
func (id ID) MarshalBinary() ([]byte, error) { return []byte(id), nil }
|
||||
|
||||
// Node contains the metadata about a node, including a collection of Records.
|
||||
type Node struct {
|
||||
ID ID
|
||||
Pubkey string
|
||||
Status string // either [StatusActive] or [StatusInactive]
|
||||
Records []Record
|
||||
}
|
||||
|
||||
// Record contains the timestamp of a node update.
|
||||
type Record struct {
|
||||
Kind int // either [Addition], [Promotion] or [Demotion]
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// Delta represents updates to apply to a Node.
|
||||
// Add and Remove contain node IDs to add to or remove from the node’s relationships (e.g., follow list).
|
||||
type Delta struct {
|
||||
Node ID
|
||||
Removed []ID
|
||||
Common []ID
|
||||
Added []ID
|
||||
Kind int
|
||||
Node ID
|
||||
Remove []ID
|
||||
Keep []ID
|
||||
Add []ID
|
||||
}
|
||||
|
||||
// Size returns the number of relationships changed by delta
|
||||
func (d Delta) Size() int {
|
||||
return len(d.Remove) + len(d.Add)
|
||||
}
|
||||
|
||||
// Old returns the old state of the delta
|
||||
func (d Delta) Old() []ID {
|
||||
return append(d.Common, d.Removed...)
|
||||
return append(d.Keep, d.Remove...)
|
||||
}
|
||||
|
||||
// New returns the new state of the delta
|
||||
func (d Delta) New() []ID {
|
||||
return append(d.Common, d.Added...)
|
||||
return append(d.Keep, d.Add...)
|
||||
}
|
||||
|
||||
// Inverse of the delta. If a delta and it's inverse are applied, the graph returns to its original state.
|
||||
func (d Delta) Inverse() Delta {
|
||||
return Delta{
|
||||
Node: d.Node,
|
||||
Common: d.Common,
|
||||
Removed: d.Added,
|
||||
Added: d.Removed,
|
||||
Kind: d.Kind,
|
||||
Node: d.Node,
|
||||
Keep: d.Keep,
|
||||
Remove: d.Add,
|
||||
Add: d.Remove,
|
||||
}
|
||||
}
|
||||
|
||||
359
pkg/redb/graph.go
Normal file
359
pkg/redb/graph.go
Normal file
@@ -0,0 +1,359 @@
|
||||
package redb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github/pippellia-btc/crawler/pkg/graph"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
// redis variable names
|
||||
KeyDatabase string = "database" // TODO: this can be removed
|
||||
KeyLastNodeID string = "lastNodeID" // TODO: change it to "next" inside "node" hash
|
||||
KeyKeyIndex string = "keyIndex" // TODO: change to key_index
|
||||
KeyNodePrefix string = "node:"
|
||||
KeyFollowsPrefix string = "follows:"
|
||||
KeyFollowersPrefix string = "followers:"
|
||||
|
||||
// redis node HASH fields
|
||||
NodeID string = "id"
|
||||
NodePubkey string = "pubkey"
|
||||
NodeStatus string = "status"
|
||||
NodePromotionTS string = "promotion_TS" // TODO: change to promotion
|
||||
NodeDemotionTS string = "demotion_TS" // TODO: change to demotion
|
||||
NodeAddedTS string = "added_TS" // TODO: change to addition
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNodeNotFound = errors.New("node not found")
|
||||
ErrNodeAlreadyExists = errors.New("node already exists")
|
||||
)
|
||||
|
||||
type RedisDB struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func New(opt *redis.Options) RedisDB {
|
||||
return RedisDB{client: redis.NewClient(opt)}
|
||||
}
|
||||
|
||||
// Size returns the DBSize of redis, which is the total number of keys
|
||||
func (r RedisDB) Size(ctx context.Context) (int, error) {
|
||||
size, err := r.client.DBSize(ctx).Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int(size), nil
|
||||
}
|
||||
|
||||
// NodeCount returns the number of nodes stored in redis (in the keyIndex)
|
||||
func (r RedisDB) NodeCount(ctx context.Context) (int, error) {
|
||||
nodes, err := r.client.HLen(ctx, KeyKeyIndex).Result()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int(nodes), nil
|
||||
}
|
||||
|
||||
// NodeByID fetches a node by its ID
|
||||
func (r RedisDB) NodeByID(ctx context.Context, ID graph.ID) (*graph.Node, error) {
|
||||
fields, err := r.client.HGetAll(ctx, node(ID)).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w", node(ID), err)
|
||||
}
|
||||
|
||||
if len(fields) == 0 {
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w", node(ID), ErrNodeNotFound)
|
||||
}
|
||||
|
||||
return parseNode(fields)
|
||||
}
|
||||
|
||||
// NodeByKey fetches a node by its pubkey
|
||||
func (r RedisDB) NodeByKey(ctx context.Context, pubkey string) (*graph.Node, error) {
|
||||
ID, err := r.client.HGet(ctx, KeyKeyIndex, pubkey).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch ID of node with pubkey %s: %w", pubkey, err)
|
||||
}
|
||||
|
||||
fields, err := r.client.HGetAll(ctx, node(ID)).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch node with pubkey %s: %w", pubkey, err)
|
||||
}
|
||||
|
||||
if len(fields) == 0 {
|
||||
return nil, fmt.Errorf("failed to fetch node with pubkey %s: %w", pubkey, ErrNodeNotFound)
|
||||
}
|
||||
|
||||
return parseNode(fields)
|
||||
}
|
||||
|
||||
func (r RedisDB) containsNode(ctx context.Context, ID graph.ID) (bool, error) {
|
||||
exists, err := r.client.Exists(ctx, node(ID)).Result()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check for the existence of %v: %w", node(ID), err)
|
||||
}
|
||||
return exists == 1, nil
|
||||
}
|
||||
|
||||
// AddNode adds a new inactive node to the database and returns its assigned ID
|
||||
func (r RedisDB) AddNode(ctx context.Context, pubkey string) (graph.ID, error) {
|
||||
exists, err := r.client.HExists(ctx, KeyKeyIndex, pubkey).Result()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to check for existence of pubkey %s: %w", pubkey, err)
|
||||
}
|
||||
|
||||
if exists {
|
||||
return "", fmt.Errorf("failed to add node with pubkey %s: %w", pubkey, ErrNodeAlreadyExists)
|
||||
}
|
||||
|
||||
// get the ID outside the transaction, which implies there might be "holes",
|
||||
// meaning IDs not associated with any node
|
||||
next, err := r.client.HIncrBy(ctx, KeyDatabase, KeyLastNodeID, 1).Result()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to add node with pubkey %s: failed to increment ID", pubkey)
|
||||
}
|
||||
ID := strconv.FormatInt(next-1, 10)
|
||||
|
||||
pipe := r.client.TxPipeline()
|
||||
pipe.HSetNX(ctx, KeyKeyIndex, pubkey, ID)
|
||||
pipe.HSet(ctx, node(ID), NodeID, ID, NodePubkey, pubkey, NodeStatus, graph.StatusInactive, NodeAddedTS, time.Now().Unix())
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return "", fmt.Errorf("failed to add node with pubkey %s: pipeline failed: %w", pubkey, err)
|
||||
}
|
||||
|
||||
return graph.ID(ID), nil
|
||||
}
|
||||
|
||||
// Promote changes the node status to active
|
||||
func (r RedisDB) Promote(ctx context.Context, ID graph.ID) error {
|
||||
err := r.client.HSet(ctx, node(ID), NodeStatus, graph.StatusActive, NodePromotionTS, time.Now().Unix()).Err()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to promote %s: %w", node(ID), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Demote changes the node status to inactive
|
||||
func (r RedisDB) Demote(ctx context.Context, ID graph.ID) error {
|
||||
err := r.client.HSet(ctx, node(ID), NodeStatus, graph.StatusInactive, NodeDemotionTS, time.Now().Unix()).Err()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to promote %s: %w", node(ID), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Follows returns the follow list of node. If node is not found, it returns [ErrNodeNotFound].
|
||||
func (r RedisDB) Follows(ctx context.Context, node graph.ID) ([]graph.ID, error) {
|
||||
return r.members(ctx, follows, node)
|
||||
}
|
||||
|
||||
// Followers returns the list of followers of node. If node is not found, it returns [ErrNodeNotFound].
|
||||
func (r RedisDB) Followers(ctx context.Context, node graph.ID) ([]graph.ID, error) {
|
||||
return r.members(ctx, followers, node)
|
||||
}
|
||||
|
||||
func (r RedisDB) members(ctx context.Context, key func(graph.ID) string, node graph.ID) ([]graph.ID, error) {
|
||||
members, err := r.client.SMembers(ctx, key(node)).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w", key(node), err)
|
||||
}
|
||||
|
||||
if len(members) == 0 {
|
||||
// check if there are no members because node doesn't exists
|
||||
ok, err := r.containsNode(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w", key(node), ErrNodeNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
return toIDs(members), nil
|
||||
}
|
||||
|
||||
// FollowCounts returns the number of follows each node has. If a node is not found, it returns 0.
|
||||
func (r RedisDB) FollowCounts(ctx context.Context, nodes ...graph.ID) ([]int, error) {
|
||||
return r.counts(ctx, follows, nodes...)
|
||||
}
|
||||
|
||||
// FollowerCounts returns the number of followers each node has. If a node is not found, it returns 0.
|
||||
func (r RedisDB) FollowerCounts(ctx context.Context, nodes ...graph.ID) ([]int, error) {
|
||||
return r.counts(ctx, followers, nodes...)
|
||||
}
|
||||
|
||||
func (r RedisDB) counts(ctx context.Context, key func(graph.ID) string, nodes ...graph.ID) ([]int, error) {
|
||||
if len(nodes) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
pipe := r.client.Pipeline()
|
||||
cmds := make([]*redis.IntCmd, len(nodes))
|
||||
keys := make([]string, len(nodes))
|
||||
|
||||
for i, node := range nodes {
|
||||
keys[i] = key(node)
|
||||
cmds[i] = pipe.SCard(ctx, key(node))
|
||||
}
|
||||
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return nil, fmt.Errorf("failed to count the elements of %v: %w", keys, err)
|
||||
}
|
||||
|
||||
counts := make([]int, len(nodes))
|
||||
for i, cmd := range cmds {
|
||||
counts[i] = int(cmd.Val())
|
||||
}
|
||||
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
// Update applies the delta to the graph.
|
||||
func (r RedisDB) Update(ctx context.Context, delta *graph.Delta) error {
|
||||
if delta.Size() == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ok, err := r.containsNode(ctx, delta.Node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update with delta %v: %w", delta, err)
|
||||
}
|
||||
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to update with delta %v: %w", delta, ErrNodeNotFound)
|
||||
}
|
||||
|
||||
switch delta.Kind {
|
||||
case nostr.KindFollowList:
|
||||
err = r.updateFollows(ctx, delta)
|
||||
|
||||
default:
|
||||
err = fmt.Errorf("unsupported kind %d", delta.Kind)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update with delta %v: %w", delta, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r RedisDB) updateFollows(ctx context.Context, delta *graph.Delta) error {
|
||||
pipe := r.client.TxPipeline()
|
||||
if len(delta.Add) > 0 {
|
||||
// add all node --> added
|
||||
pipe.SAdd(ctx, follows(delta.Node), toStrings(delta.Add))
|
||||
|
||||
for _, a := range delta.Add {
|
||||
pipe.SAdd(ctx, followers(a), delta.Node)
|
||||
}
|
||||
}
|
||||
|
||||
if len(delta.Remove) > 0 {
|
||||
// remove all node --> removed
|
||||
pipe.SRem(ctx, follows(delta.Node), toStrings(delta.Remove))
|
||||
|
||||
for _, r := range delta.Remove {
|
||||
pipe.SRem(ctx, followers(r), delta.Node)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
return fmt.Errorf("pipeline failed: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NodeIDs returns a slice of node IDs assosiated with the pubkeys.
|
||||
// If a pubkey is not found, an empty ID "" is returned
|
||||
func (r RedisDB) NodeIDs(ctx context.Context, pubkeys ...string) ([]graph.ID, error) {
|
||||
if len(pubkeys) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
IDs, err := r.client.HMGet(ctx, KeyKeyIndex, pubkeys...).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch the node IDs of %v: %w", pubkeys, err)
|
||||
}
|
||||
|
||||
nodes := make([]graph.ID, len(IDs))
|
||||
for i, ID := range IDs {
|
||||
switch ID {
|
||||
case nil:
|
||||
nodes[i] = "" // empty ID means missing pubkey
|
||||
|
||||
default:
|
||||
nodes[i] = graph.ID(ID.(string)) // no need to type-assert because everything in redis is a string
|
||||
}
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// Pubkeys returns a slice of pubkeys assosiated with the node IDs.
|
||||
// If a node ID is not found, an empty pubkey "" is returned
|
||||
func (r RedisDB) Pubkeys(ctx context.Context, nodes ...graph.ID) ([]string, error) {
|
||||
if len(nodes) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
pipe := r.client.Pipeline()
|
||||
cmds := make([]*redis.StringCmd, len(nodes))
|
||||
for i, ID := range nodes {
|
||||
cmds[i] = pipe.HGet(ctx, node(ID), NodePubkey)
|
||||
}
|
||||
|
||||
if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) {
|
||||
// deal later with redis.Nil, which means node(s) not found
|
||||
return nil, fmt.Errorf("failed to fetch the pubkeys of %v: pipeline failed: %w", nodes, err)
|
||||
}
|
||||
|
||||
pubkeys := make([]string, len(nodes))
|
||||
for i, cmd := range cmds {
|
||||
switch {
|
||||
case errors.Is(cmd.Err(), redis.Nil):
|
||||
pubkeys[i] = "" // empty pubkey means missing node
|
||||
|
||||
case cmd.Err() != nil:
|
||||
return nil, fmt.Errorf("failed to fetch the pubkeys of %v: %w", nodes, cmd.Err())
|
||||
|
||||
default:
|
||||
pubkeys[i] = cmd.Val()
|
||||
}
|
||||
}
|
||||
|
||||
return pubkeys, nil
|
||||
}
|
||||
|
||||
// ScanNodes to return a batch of node IDs of size roughly proportional to limit.
|
||||
// Limit controls how much "work" is invested in fetching the batch, hence it's not precise.
|
||||
// Learn more about scan: https://redis.io/docs/latest/commands/scan/
|
||||
func (r RedisDB) ScanNodes(ctx context.Context, cursor uint64, limit int) ([]graph.ID, uint64, error) {
|
||||
match := KeyNodePrefix + "*"
|
||||
keys, cursor, err := r.client.Scan(ctx, cursor, match, int64(limit)).Result()
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to scan for keys matching %s: %w", match, err)
|
||||
}
|
||||
|
||||
nodes := make([]graph.ID, len(keys))
|
||||
for i, key := range keys {
|
||||
node, found := strings.CutPrefix(key, KeyNodePrefix)
|
||||
if !found {
|
||||
return nil, 0, fmt.Errorf("failed to scan for keys matching %s: bad match %s", match, node)
|
||||
}
|
||||
|
||||
nodes[i] = graph.ID(node)
|
||||
}
|
||||
|
||||
return nodes, cursor, nil
|
||||
}
|
||||
357
pkg/redb/graph_test.go
Normal file
357
pkg/redb/graph_test.go
Normal file
@@ -0,0 +1,357 @@
|
||||
package redb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github/pippellia-btc/crawler/pkg/graph"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nbd-wtf/go-nostr"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var ctx = context.Background()
|
||||
|
||||
func TestParseNode(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fields map[string]string
|
||||
expected *graph.Node
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "nil map",
|
||||
},
|
||||
{
|
||||
name: "empty map",
|
||||
fields: map[string]string{},
|
||||
},
|
||||
{
|
||||
name: "valid no records",
|
||||
fields: map[string]string{
|
||||
NodeID: "19",
|
||||
NodePubkey: "nineteen",
|
||||
NodeStatus: graph.StatusActive,
|
||||
},
|
||||
expected: &graph.Node{
|
||||
ID: "19",
|
||||
Pubkey: "nineteen",
|
||||
Status: graph.StatusActive,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid with record",
|
||||
fields: map[string]string{
|
||||
NodeID: "19",
|
||||
NodePubkey: "nineteen",
|
||||
NodeStatus: graph.StatusActive,
|
||||
NodeAddedTS: "1",
|
||||
},
|
||||
expected: &graph.Node{
|
||||
ID: "19",
|
||||
Pubkey: "nineteen",
|
||||
Status: graph.StatusActive,
|
||||
Records: []graph.Record{
|
||||
{Kind: graph.Addition, Timestamp: time.Unix(1, 0)},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
node, err := parseNode(test.fields)
|
||||
if !errors.Is(err, test.err) {
|
||||
t.Fatalf("expected %v got %v", test.err, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(node, test.expected) {
|
||||
t.Fatalf("ParseNode(): expected node %v got %v", test.expected, node)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddNode(t *testing.T) {
|
||||
t.Run("node already exists", func(t *testing.T) {
|
||||
db, err := OneNode()
|
||||
if err != nil {
|
||||
t.Fatalf("setup failed: %v", err)
|
||||
}
|
||||
defer db.flushAll()
|
||||
|
||||
if _, err = db.AddNode(ctx, "0"); !errors.Is(err, ErrNodeAlreadyExists) {
|
||||
t.Fatalf("expected error %v, got %v", ErrNodeAlreadyExists, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
db, err := OneNode()
|
||||
if err != nil {
|
||||
t.Fatalf("setup failed: %v", err)
|
||||
}
|
||||
defer db.flushAll()
|
||||
|
||||
ID, err := db.AddNode(ctx, "xxx")
|
||||
if err != nil {
|
||||
t.Fatalf("expected nil, got %v", err)
|
||||
}
|
||||
|
||||
expected := &graph.Node{
|
||||
ID: "1",
|
||||
Pubkey: "xxx",
|
||||
Status: graph.StatusInactive,
|
||||
Records: []graph.Record{{Kind: graph.Addition, Timestamp: time.Unix(time.Now().Unix(), 0)}},
|
||||
}
|
||||
|
||||
if ID != expected.ID {
|
||||
t.Fatalf("expected ID %s, got %s", expected.ID, ID)
|
||||
}
|
||||
|
||||
node, err := db.NodeByKey(ctx, "xxx")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(node, expected) {
|
||||
t.Fatalf("expected node %v, got %v", expected, node)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestMembers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func() (RedisDB, error)
|
||||
node graph.ID
|
||||
expected []graph.ID
|
||||
err error
|
||||
}{
|
||||
{
|
||||
name: "empty database",
|
||||
setup: Empty,
|
||||
node: "0",
|
||||
err: ErrNodeNotFound,
|
||||
},
|
||||
{
|
||||
name: "node not found",
|
||||
setup: OneNode,
|
||||
node: "1",
|
||||
err: ErrNodeNotFound,
|
||||
},
|
||||
{
|
||||
name: "dandling node",
|
||||
setup: OneNode,
|
||||
node: "0",
|
||||
expected: []graph.ID{},
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
setup: Simple,
|
||||
node: "0",
|
||||
expected: []graph.ID{"1"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
db, err := test.setup()
|
||||
if err != nil {
|
||||
t.Fatalf("setup failed: %v", err)
|
||||
}
|
||||
defer db.flushAll()
|
||||
|
||||
follows, err := db.members(ctx, follows, test.node)
|
||||
if !errors.Is(err, test.err) {
|
||||
t.Fatalf("expected error %v, got %v", test.err, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(follows, test.expected) {
|
||||
t.Errorf("expected follows %v, got %v", test.expected, follows)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFollows(t *testing.T) {
|
||||
db, err := Simple()
|
||||
if err != nil {
|
||||
t.Fatalf("setup failed: %v", err)
|
||||
}
|
||||
defer db.flushAll()
|
||||
|
||||
delta := &graph.Delta{
|
||||
Kind: nostr.KindFollowList,
|
||||
Node: "0",
|
||||
Remove: []graph.ID{"1"},
|
||||
Add: []graph.ID{"2"},
|
||||
}
|
||||
|
||||
if err := db.Update(ctx, delta); err != nil {
|
||||
t.Fatalf("expected error nil, got %v", err)
|
||||
}
|
||||
|
||||
follows, err := db.Follows(ctx, "0")
|
||||
if err != nil {
|
||||
t.Fatalf("expected nil got %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(follows, []graph.ID{"2"}) {
|
||||
t.Fatalf("expected follows(0) %v, got %v", []graph.ID{"2"}, follows)
|
||||
}
|
||||
|
||||
followers, err := db.Followers(ctx, "1")
|
||||
if err != nil {
|
||||
t.Fatalf("expected nil got %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(followers, []graph.ID{}) {
|
||||
t.Fatalf("expected followers(1) %v, got %v", []graph.ID{}, followers)
|
||||
}
|
||||
|
||||
followers, err = db.Followers(ctx, "2")
|
||||
if err != nil {
|
||||
t.Fatalf("expected nil got %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(followers, []graph.ID{"0"}) {
|
||||
t.Fatalf("expected followers(2) %v, got %v", []graph.ID{"0"}, followers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeIDs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func() (RedisDB, error)
|
||||
pubkeys []string
|
||||
expected []graph.ID
|
||||
}{
|
||||
{
|
||||
name: "empty database",
|
||||
setup: Empty,
|
||||
pubkeys: []string{"0"},
|
||||
expected: []graph.ID{""},
|
||||
},
|
||||
{
|
||||
name: "node not found",
|
||||
setup: OneNode,
|
||||
pubkeys: []string{"1"},
|
||||
expected: []graph.ID{""},
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
setup: Simple,
|
||||
pubkeys: []string{"0", "1", "69"},
|
||||
expected: []graph.ID{"0", "1", ""}, // last is not found
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
db, err := test.setup()
|
||||
if err != nil {
|
||||
t.Fatalf("setup failed: %v", err)
|
||||
}
|
||||
defer db.flushAll()
|
||||
|
||||
nodes, err := db.NodeIDs(ctx, test.pubkeys...)
|
||||
if err != nil {
|
||||
t.Fatalf("expected error nil, got %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(nodes, test.expected) {
|
||||
t.Fatalf("expected nodes %v, got %v", test.expected, nodes)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPubkeys(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func() (RedisDB, error)
|
||||
nodes []graph.ID
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "empty database",
|
||||
setup: Empty,
|
||||
nodes: []graph.ID{"0"},
|
||||
expected: []string{""},
|
||||
},
|
||||
{
|
||||
name: "node not found",
|
||||
setup: OneNode,
|
||||
nodes: []graph.ID{"1"},
|
||||
expected: []string{""},
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
setup: Simple,
|
||||
nodes: []graph.ID{"0", "1", "69"},
|
||||
expected: []string{"0", "1", ""}, // last is not found
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
db, err := test.setup()
|
||||
if err != nil {
|
||||
t.Fatalf("setup failed: %v", err)
|
||||
}
|
||||
defer db.flushAll()
|
||||
|
||||
pubkeys, err := db.Pubkeys(ctx, test.nodes...)
|
||||
if err != nil {
|
||||
t.Fatalf("expected error nil, got %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(pubkeys, test.expected) {
|
||||
t.Fatalf("expected pubkeys %v, got %v", test.expected, pubkeys)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------- HELPERS -------------------------------
|
||||
|
||||
func Empty() (RedisDB, error) {
|
||||
return New(&redis.Options{Addr: testAddress}), nil
|
||||
}
|
||||
|
||||
func OneNode() (RedisDB, error) {
|
||||
db := New(&redis.Options{Addr: testAddress})
|
||||
if _, err := db.AddNode(context.Background(), "0"); err != nil {
|
||||
db.flushAll()
|
||||
return RedisDB{}, err
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func Simple() (RedisDB, error) {
|
||||
ctx := context.Background()
|
||||
db := New(&redis.Options{Addr: testAddress})
|
||||
|
||||
for _, pk := range []string{"0", "1", "2"} {
|
||||
if _, err := db.AddNode(ctx, pk); err != nil {
|
||||
db.flushAll()
|
||||
return RedisDB{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// 0 ---> 1
|
||||
if err := db.client.SAdd(ctx, follows("0"), "1").Err(); err != nil {
|
||||
db.flushAll()
|
||||
return RedisDB{}, err
|
||||
}
|
||||
|
||||
if err := db.client.SAdd(ctx, followers("1"), "0").Err(); err != nil {
|
||||
db.flushAll()
|
||||
return RedisDB{}, err
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
100
pkg/redb/utils.go
Normal file
100
pkg/redb/utils.go
Normal file
@@ -0,0 +1,100 @@
|
||||
package redb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github/pippellia-btc/crawler/pkg/graph"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
testAddress = "localhost:6380"
|
||||
)
|
||||
|
||||
// flushAll deletes all the keys of all existing databases. This command never fails.
|
||||
func (r RedisDB) flushAll() {
|
||||
r.client.FlushAll(context.Background())
|
||||
}
|
||||
|
||||
func node[ID string | graph.ID](id ID) string {
|
||||
return KeyNodePrefix + string(id)
|
||||
}
|
||||
|
||||
func follows[ID string | graph.ID](id ID) string {
|
||||
return KeyFollowsPrefix + string(id)
|
||||
}
|
||||
|
||||
func followers[ID string | graph.ID](id ID) string {
|
||||
return KeyFollowersPrefix + string(id)
|
||||
}
|
||||
|
||||
// ids converts a slice of strings to IDs
|
||||
func toIDs(s []string) []graph.ID {
|
||||
IDs := make([]graph.ID, len(s))
|
||||
for i, e := range s {
|
||||
IDs[i] = graph.ID(e)
|
||||
}
|
||||
return IDs
|
||||
}
|
||||
|
||||
// strings converts graph IDs to a slice of strings
|
||||
func toStrings(ids []graph.ID) []string {
|
||||
s := make([]string, len(ids))
|
||||
for i, id := range ids {
|
||||
s[i] = string(id)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// parseNode() parses the map into a node structure
|
||||
func parseNode(fields map[string]string) (*graph.Node, error) {
|
||||
if len(fields) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var node graph.Node
|
||||
for key, val := range fields {
|
||||
switch key {
|
||||
case NodeID:
|
||||
node.ID = graph.ID(val)
|
||||
|
||||
case NodePubkey:
|
||||
node.Pubkey = val
|
||||
|
||||
case NodeStatus:
|
||||
node.Status = val
|
||||
|
||||
case NodeAddedTS:
|
||||
ts, err := parseTimestamp(val)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node.Records = append(node.Records, graph.Record{Kind: graph.Addition, Timestamp: ts})
|
||||
|
||||
case NodePromotionTS:
|
||||
ts, err := parseTimestamp(val)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node.Records = append(node.Records, graph.Record{Kind: graph.Promotion, Timestamp: ts})
|
||||
|
||||
case NodeDemotionTS:
|
||||
ts, err := parseTimestamp(val)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node.Records = append(node.Records, graph.Record{Kind: graph.Demotion, Timestamp: ts})
|
||||
}
|
||||
}
|
||||
|
||||
return &node, nil
|
||||
}
|
||||
|
||||
// parseTimestamp() parses a unix timestamp string into a time.Time
|
||||
func parseTimestamp(unix string) (time.Time, error) {
|
||||
ts, err := strconv.ParseInt(unix, 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return time.Unix(ts, 0), nil
|
||||
}
|
||||
1
pkg/redb/walks.go
Normal file
1
pkg/redb/walks.go
Normal file
@@ -0,0 +1 @@
|
||||
package redb
|
||||
@@ -179,7 +179,7 @@ func ToUpdate(ctx context.Context, walker Walker, delta graph.Delta, walks []Wal
|
||||
}
|
||||
|
||||
shouldResample = rand.Float64() < resampleProbability
|
||||
isInvalid = (pos < walk.Len()-1) && slices.Contains(delta.Removed, walk.Path[pos+1])
|
||||
isInvalid = (pos < walk.Len()-1) && slices.Contains(delta.Remove, walk.Path[pos+1])
|
||||
|
||||
switch {
|
||||
case shouldResample:
|
||||
@@ -188,7 +188,7 @@ func ToUpdate(ctx context.Context, walker Walker, delta graph.Delta, walks []Wal
|
||||
updated.Prune(pos + 1)
|
||||
|
||||
if rand.Float64() < Alpha {
|
||||
new, err := generate(ctx, walker, delta.Added...)
|
||||
new, err := generate(ctx, walker, delta.Add...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ToUpdate: failed to generate new segment: %w", err)
|
||||
}
|
||||
@@ -203,7 +203,7 @@ func ToUpdate(ctx context.Context, walker Walker, delta graph.Delta, walks []Wal
|
||||
updated := walk.Copy()
|
||||
updated.Prune(pos + 1)
|
||||
|
||||
new, err := generate(ctx, walker, delta.Common...)
|
||||
new, err := generate(ctx, walker, delta.Keep...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ToUpdate: failed to generate new segment: %w", err)
|
||||
}
|
||||
@@ -223,24 +223,24 @@ func ToUpdate(ctx context.Context, walker Walker, delta graph.Delta, walks []Wal
|
||||
// Our goal is to have 1/3 of the walks that continue go to each of 1, 2 and 3.
|
||||
// This means we have to re-do 2/3 of the walks and make them continue towards 2 or 3.
|
||||
func resampleProbability(delta graph.Delta) float64 {
|
||||
if len(delta.Added) == 0 {
|
||||
if len(delta.Add) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
c := float64(len(delta.Common))
|
||||
a := float64(len(delta.Added))
|
||||
c := float64(len(delta.Keep))
|
||||
a := float64(len(delta.Add))
|
||||
return a / (a + c)
|
||||
}
|
||||
|
||||
func expectedUpdates(walks []Walk, delta graph.Delta) int {
|
||||
if len(delta.Common) == 0 {
|
||||
if len(delta.Keep) == 0 {
|
||||
// no nodes have remained, all walks must be re-computed
|
||||
return len(walks)
|
||||
}
|
||||
|
||||
r := float64(len(delta.Removed))
|
||||
c := float64(len(delta.Common))
|
||||
a := float64(len(delta.Added))
|
||||
r := float64(len(delta.Remove))
|
||||
c := float64(len(delta.Keep))
|
||||
a := float64(len(delta.Add))
|
||||
|
||||
invalidProbability := Alpha * r / (r + c)
|
||||
resampleProbability := a / (a + c)
|
||||
|
||||
@@ -62,9 +62,9 @@ func TestUpdateRemove(t *testing.T) {
|
||||
})
|
||||
|
||||
delta := graph.Delta{
|
||||
Node: "0",
|
||||
Removed: []graph.ID{"1"}, // the old follows were "1" and "3"
|
||||
Common: []graph.ID{"3"},
|
||||
Node: "0",
|
||||
Remove: []graph.ID{"1"}, // the old follows were "1" and "3"
|
||||
Keep: []graph.ID{"3"},
|
||||
}
|
||||
|
||||
walks := []Walk{
|
||||
|
||||
@@ -38,7 +38,7 @@ func Dandlings(n int) Setup {
|
||||
personalized := make([]float64, n)
|
||||
personalized[0] = 1
|
||||
|
||||
added := make([]graph.ID, 0, n-1)
|
||||
add := make([]graph.ID, 0, n-1)
|
||||
deltas := make([]graph.Delta, 0, n-1)
|
||||
|
||||
for i := range n {
|
||||
@@ -48,8 +48,8 @@ func Dandlings(n int) Setup {
|
||||
|
||||
if i > 0 {
|
||||
// all the possible deltas modulo graph isomorphism; 0 --> [1,2, ... k] for 1 <= k <= n
|
||||
added = append(added, node)
|
||||
deltas = append(deltas, graph.Delta{Node: "0", Added: added})
|
||||
add = append(add, node)
|
||||
deltas = append(deltas, graph.Delta{Node: "0", Add: add})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,9 +79,9 @@ func Cyclic(n int) Setup {
|
||||
return Setup{
|
||||
walker: walks.NewCyclicWalker(n),
|
||||
deltas: []graph.Delta{
|
||||
{Node: "0", Removed: []graph.ID{"1"}},
|
||||
{Node: "0", Common: []graph.ID{"1"}, Added: []graph.ID{mid}},
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Added: []graph.ID{mid}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}},
|
||||
{Node: "0", Keep: []graph.ID{"1"}, Add: []graph.ID{mid}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Add: []graph.ID{mid}},
|
||||
},
|
||||
|
||||
nodes: nodes,
|
||||
@@ -102,33 +102,33 @@ var Acyclic1 = Setup{
|
||||
}),
|
||||
deltas: []graph.Delta{
|
||||
// removals
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}},
|
||||
{Node: "0", Removed: []graph.ID{"2"}, Common: []graph.ID{"1"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}},
|
||||
{Node: "2", Removed: []graph.ID{"3"}},
|
||||
{Node: "3", Removed: []graph.ID{"1"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}},
|
||||
{Node: "0", Remove: []graph.ID{"2"}, Keep: []graph.ID{"1"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}},
|
||||
{Node: "2", Remove: []graph.ID{"3"}},
|
||||
{Node: "3", Remove: []graph.ID{"1"}},
|
||||
// additions
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"4"}},
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "4"}},
|
||||
{Node: "4", Added: []graph.ID{"0"}},
|
||||
{Node: "4", Added: []graph.ID{"1"}},
|
||||
{Node: "4", Added: []graph.ID{"2"}},
|
||||
{Node: "4", Added: []graph.ID{"3"}},
|
||||
{Node: "4", Added: []graph.ID{"1", "2"}},
|
||||
{Node: "4", Added: []graph.ID{"2", "3"}},
|
||||
{Node: "4", Added: []graph.ID{"3", "4"}},
|
||||
{Node: "4", Added: []graph.ID{"0", "1", "2"}},
|
||||
{Node: "4", Added: []graph.ID{"0", "1", "2", "3"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"4"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "4"}},
|
||||
{Node: "4", Add: []graph.ID{"0"}},
|
||||
{Node: "4", Add: []graph.ID{"1"}},
|
||||
{Node: "4", Add: []graph.ID{"2"}},
|
||||
{Node: "4", Add: []graph.ID{"3"}},
|
||||
{Node: "4", Add: []graph.ID{"1", "2"}},
|
||||
{Node: "4", Add: []graph.ID{"2", "3"}},
|
||||
{Node: "4", Add: []graph.ID{"3", "4"}},
|
||||
{Node: "4", Add: []graph.ID{"0", "1", "2"}},
|
||||
{Node: "4", Add: []graph.ID{"0", "1", "2", "3"}},
|
||||
// removals and additions
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"4"}},
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"4"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "4"}},
|
||||
{Node: "2", Removed: []graph.ID{"3"}, Added: []graph.ID{"1"}},
|
||||
{Node: "2", Removed: []graph.ID{"3"}, Added: []graph.ID{"4"}},
|
||||
{Node: "2", Removed: []graph.ID{"3"}, Added: []graph.ID{"1", "4"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"4"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"4"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "4"}},
|
||||
{Node: "2", Remove: []graph.ID{"3"}, Add: []graph.ID{"1"}},
|
||||
{Node: "2", Remove: []graph.ID{"3"}, Add: []graph.ID{"4"}},
|
||||
{Node: "2", Remove: []graph.ID{"3"}, Add: []graph.ID{"1", "4"}},
|
||||
},
|
||||
nodes: []graph.ID{"0", "1", "2", "3", "4"},
|
||||
global: []float64{0.11185, 0.36950, 0.15943, 0.24736, 0.11185},
|
||||
@@ -146,20 +146,20 @@ var Acyclic2 = Setup{
|
||||
}),
|
||||
deltas: []graph.Delta{
|
||||
// removals
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}},
|
||||
// additions
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"4"}},
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "4"}},
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "5"}},
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "4", "5"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"4"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "4"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "5"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "4", "5"}},
|
||||
// removals and additions
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"4"}},
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3", "4"}},
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3", "5"}},
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3", "4", "5"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"4"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3", "4"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3", "5"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3", "4", "5"}},
|
||||
},
|
||||
nodes: []graph.ID{"0", "1", "2", "3", "4", "5"},
|
||||
global: []float64{0.12987, 0.18506, 0.18506, 0.18506, 0.12987, 0.18506},
|
||||
@@ -175,14 +175,14 @@ var Acyclic3 = Setup{
|
||||
}),
|
||||
deltas: []graph.Delta{
|
||||
// removals
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}},
|
||||
// additions
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "2", Added: []graph.ID{"1"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "2", Add: []graph.ID{"1"}},
|
||||
// removals and additions
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}},
|
||||
},
|
||||
nodes: []graph.ID{"0", "1", "2", "3"},
|
||||
global: []float64{0.17544, 0.32456, 0.32456, 0.17544},
|
||||
@@ -198,19 +198,19 @@ var Acyclic4 = Setup{
|
||||
}),
|
||||
deltas: []graph.Delta{
|
||||
// removals
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}},
|
||||
{Node: "3", Removed: []graph.ID{"1"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}},
|
||||
{Node: "3", Remove: []graph.ID{"1"}},
|
||||
// additions
|
||||
{Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "2", Added: []graph.ID{"1"}},
|
||||
{Node: "2", Added: []graph.ID{"3"}},
|
||||
{Node: "3", Common: []graph.ID{"1"}, Added: []graph.ID{"0"}},
|
||||
{Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "2", Add: []graph.ID{"1"}},
|
||||
{Node: "2", Add: []graph.ID{"3"}},
|
||||
{Node: "3", Keep: []graph.ID{"1"}, Add: []graph.ID{"0"}},
|
||||
// removals and additions
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}},
|
||||
{Node: "3", Removed: []graph.ID{"1"}, Added: []graph.ID{"0"}},
|
||||
{Node: "3", Removed: []graph.ID{"1"}, Added: []graph.ID{"0", "2"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}},
|
||||
{Node: "3", Remove: []graph.ID{"1"}, Add: []graph.ID{"0"}},
|
||||
{Node: "3", Remove: []graph.ID{"1"}, Add: []graph.ID{"0", "2"}},
|
||||
},
|
||||
nodes: []graph.ID{"0", "1", "2", "3"},
|
||||
global: []float64{0.17544, 0.39912, 0.25, 0.17544},
|
||||
@@ -226,19 +226,19 @@ var Acyclic5 = Setup{
|
||||
}),
|
||||
deltas: []graph.Delta{
|
||||
// removals
|
||||
{Node: "0", Removed: []graph.ID{"3"}},
|
||||
{Node: "1", Removed: []graph.ID{"0"}},
|
||||
{Node: "3", Removed: []graph.ID{"2"}},
|
||||
{Node: "0", Remove: []graph.ID{"3"}},
|
||||
{Node: "1", Remove: []graph.ID{"0"}},
|
||||
{Node: "3", Remove: []graph.ID{"2"}},
|
||||
// additions
|
||||
{Node: "0", Common: []graph.ID{"3"}, Added: []graph.ID{"2"}},
|
||||
{Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"2"}},
|
||||
{Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"3"}},
|
||||
{Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"2", "3"}},
|
||||
{Node: "0", Keep: []graph.ID{"3"}, Add: []graph.ID{"2"}},
|
||||
{Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"2"}},
|
||||
{Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"3"}},
|
||||
{Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"2", "3"}},
|
||||
// removals and additions
|
||||
{Node: "0", Removed: []graph.ID{"3"}, Added: []graph.ID{"2"}},
|
||||
{Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"2"}},
|
||||
{Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"3"}},
|
||||
{Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"2", "3"}},
|
||||
{Node: "0", Remove: []graph.ID{"3"}, Add: []graph.ID{"2"}},
|
||||
{Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"2"}},
|
||||
{Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"3"}},
|
||||
{Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"2", "3"}},
|
||||
},
|
||||
nodes: []graph.ID{"0", "1", "2", "3"},
|
||||
global: []float64{0.21489, 0.11616, 0.37015, 0.29881},
|
||||
@@ -255,34 +255,34 @@ var Acyclic6 = Setup{
|
||||
}),
|
||||
deltas: []graph.Delta{
|
||||
// removals
|
||||
{Node: "0", Removed: []graph.ID{"4"}},
|
||||
{Node: "1", Removed: []graph.ID{"0"}},
|
||||
{Node: "3", Removed: []graph.ID{"1"}, Common: []graph.ID{"4"}},
|
||||
{Node: "3", Removed: []graph.ID{"4"}, Common: []graph.ID{"1"}},
|
||||
{Node: "3", Removed: []graph.ID{"1", "4"}},
|
||||
{Node: "4", Removed: []graph.ID{"2"}},
|
||||
{Node: "0", Remove: []graph.ID{"4"}},
|
||||
{Node: "1", Remove: []graph.ID{"0"}},
|
||||
{Node: "3", Remove: []graph.ID{"1"}, Keep: []graph.ID{"4"}},
|
||||
{Node: "3", Remove: []graph.ID{"4"}, Keep: []graph.ID{"1"}},
|
||||
{Node: "3", Remove: []graph.ID{"1", "4"}},
|
||||
{Node: "4", Remove: []graph.ID{"2"}},
|
||||
// additions
|
||||
{Node: "0", Common: []graph.ID{"4"}, Added: []graph.ID{"2"}},
|
||||
{Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"2"}},
|
||||
{Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"4"}},
|
||||
{Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"2", "4"}},
|
||||
{Node: "3", Common: []graph.ID{"1", "4"}, Added: []graph.ID{"0"}},
|
||||
{Node: "3", Common: []graph.ID{"1", "4"}, Added: []graph.ID{"2"}},
|
||||
{Node: "3", Common: []graph.ID{"1", "4"}, Added: []graph.ID{"0", "2"}},
|
||||
{Node: "0", Keep: []graph.ID{"4"}, Add: []graph.ID{"2"}},
|
||||
{Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"2"}},
|
||||
{Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"4"}},
|
||||
{Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"2", "4"}},
|
||||
{Node: "3", Keep: []graph.ID{"1", "4"}, Add: []graph.ID{"0"}},
|
||||
{Node: "3", Keep: []graph.ID{"1", "4"}, Add: []graph.ID{"2"}},
|
||||
{Node: "3", Keep: []graph.ID{"1", "4"}, Add: []graph.ID{"0", "2"}},
|
||||
// removals and additions
|
||||
{Node: "0", Removed: []graph.ID{"4"}, Added: []graph.ID{"2"}},
|
||||
{Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"2"}},
|
||||
{Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"4"}},
|
||||
{Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"2", "4"}},
|
||||
{Node: "3", Removed: []graph.ID{"1"}, Common: []graph.ID{"4"}, Added: []graph.ID{"0"}},
|
||||
{Node: "3", Removed: []graph.ID{"1"}, Common: []graph.ID{"4"}, Added: []graph.ID{"2"}},
|
||||
{Node: "3", Removed: []graph.ID{"1"}, Common: []graph.ID{"4"}, Added: []graph.ID{"0", "2"}},
|
||||
{Node: "3", Removed: []graph.ID{"4"}, Common: []graph.ID{"1"}, Added: []graph.ID{"0"}},
|
||||
{Node: "3", Removed: []graph.ID{"4"}, Common: []graph.ID{"1"}, Added: []graph.ID{"2"}},
|
||||
{Node: "3", Removed: []graph.ID{"4"}, Common: []graph.ID{"1"}, Added: []graph.ID{"0", "2"}},
|
||||
{Node: "3", Removed: []graph.ID{"1", "4"}, Added: []graph.ID{"0"}},
|
||||
{Node: "3", Removed: []graph.ID{"1", "4"}, Added: []graph.ID{"2"}},
|
||||
{Node: "3", Removed: []graph.ID{"1", "4"}, Added: []graph.ID{"0", "2"}},
|
||||
{Node: "0", Remove: []graph.ID{"4"}, Add: []graph.ID{"2"}},
|
||||
{Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"2"}},
|
||||
{Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"4"}},
|
||||
{Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"2", "4"}},
|
||||
{Node: "3", Remove: []graph.ID{"1"}, Keep: []graph.ID{"4"}, Add: []graph.ID{"0"}},
|
||||
{Node: "3", Remove: []graph.ID{"1"}, Keep: []graph.ID{"4"}, Add: []graph.ID{"2"}},
|
||||
{Node: "3", Remove: []graph.ID{"1"}, Keep: []graph.ID{"4"}, Add: []graph.ID{"0", "2"}},
|
||||
{Node: "3", Remove: []graph.ID{"4"}, Keep: []graph.ID{"1"}, Add: []graph.ID{"0"}},
|
||||
{Node: "3", Remove: []graph.ID{"4"}, Keep: []graph.ID{"1"}, Add: []graph.ID{"2"}},
|
||||
{Node: "3", Remove: []graph.ID{"4"}, Keep: []graph.ID{"1"}, Add: []graph.ID{"0", "2"}},
|
||||
{Node: "3", Remove: []graph.ID{"1", "4"}, Add: []graph.ID{"0"}},
|
||||
{Node: "3", Remove: []graph.ID{"1", "4"}, Add: []graph.ID{"2"}},
|
||||
{Node: "3", Remove: []graph.ID{"1", "4"}, Add: []graph.ID{"0", "2"}},
|
||||
},
|
||||
nodes: []graph.ID{"0", "1", "2", "3", "4"},
|
||||
global: []float64{0.18820, 0.12128, 0.32417, 0.08511, 0.28125},
|
||||
@@ -299,17 +299,17 @@ var Acyclic7 = Setup{
|
||||
}),
|
||||
deltas: []graph.Delta{
|
||||
// removals
|
||||
{Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2", "3"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2"}, Common: []graph.ID{"3"}},
|
||||
{Node: "0", Removed: []graph.ID{"1", "2", "3"}},
|
||||
{Node: "4", Removed: []graph.ID{"0"}, Common: []graph.ID{"1", "2", "3"}},
|
||||
{Node: "4", Removed: []graph.ID{"1"}, Common: []graph.ID{"0", "2", "3"}},
|
||||
{Node: "4", Removed: []graph.ID{"1", "2"}, Common: []graph.ID{"0", "3"}},
|
||||
{Node: "4", Removed: []graph.ID{"1", "2", "3"}, Common: []graph.ID{"0"}},
|
||||
{Node: "4", Removed: []graph.ID{"0", "1", "2", "3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2", "3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2"}, Keep: []graph.ID{"3"}},
|
||||
{Node: "0", Remove: []graph.ID{"1", "2", "3"}},
|
||||
{Node: "4", Remove: []graph.ID{"0"}, Keep: []graph.ID{"1", "2", "3"}},
|
||||
{Node: "4", Remove: []graph.ID{"1"}, Keep: []graph.ID{"0", "2", "3"}},
|
||||
{Node: "4", Remove: []graph.ID{"1", "2"}, Keep: []graph.ID{"0", "3"}},
|
||||
{Node: "4", Remove: []graph.ID{"1", "2", "3"}, Keep: []graph.ID{"0"}},
|
||||
{Node: "4", Remove: []graph.ID{"0", "1", "2", "3"}},
|
||||
// additions
|
||||
{Node: "1", Added: []graph.ID{"2"}},
|
||||
{Node: "1", Added: []graph.ID{"2", "3"}},
|
||||
{Node: "1", Add: []graph.ID{"2"}},
|
||||
{Node: "1", Add: []graph.ID{"2", "3"}},
|
||||
},
|
||||
nodes: []graph.ID{"0", "1", "2", "3", "4"},
|
||||
global: []float64{0.17622, 0.22615, 0.22615, 0.22615, 0.14534},
|
||||
|
||||
Reference in New Issue
Block a user