From 85a2eebc95d683d844119de514988ba9e035ea4a Mon Sep 17 00:00:00 2001 From: pippellia-btc Date: Tue, 27 May 2025 11:40:44 +0200 Subject: [PATCH] database code dump --- go.mod | 37 +++- go.sum | 89 +++++++++ pkg/graph/graph.go | 63 +++++-- pkg/redb/graph.go | 359 +++++++++++++++++++++++++++++++++++++ pkg/redb/graph_test.go | 357 ++++++++++++++++++++++++++++++++++++ pkg/redb/utils.go | 100 +++++++++++ pkg/redb/walks.go | 1 + pkg/walks/walks.go | 20 +-- pkg/walks/walks_test.go | 6 +- tests/random/utils_test.go | 214 +++++++++++----------- 10 files changed, 1112 insertions(+), 134 deletions(-) create mode 100644 go.sum create mode 100644 pkg/redb/graph.go create mode 100644 pkg/redb/graph_test.go create mode 100644 pkg/redb/utils.go create mode 100644 pkg/redb/walks.go diff --git a/go.mod b/go.mod index 0c8d1f7..e8e30d8 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,38 @@ module github/pippellia-btc/crawler -go 1.23.1 +go 1.24.1 + +toolchain go1.24.3 + +require ( + github.com/nbd-wtf/go-nostr v0.51.12 + github.com/redis/go-redis/v9 v9.8.0 +) + +require ( + github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect + github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect + github.com/bytedance/sonic v1.13.1 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect + github.com/coder/websocket v1.8.12 // indirect + github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + golang.org/x/arch v0.15.0 // indirect + golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect + golang.org/x/sys v0.31.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..337dfb1 --- /dev/null +++ b/go.sum @@ -0,0 +1,89 @@ +github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 h1:ClzzXMDDuUbWfNNZqGeYq4PnYOlwlOVIvSyNaIy0ykg= +github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3/go.mod h1:we0YA5CsBbH5+/NUzC/AlMmxaDtWlXeNsqrwXjTzmzA= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= +github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= +github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 h1:59Kx4K6lzOW5w6nFlA0v5+lk/6sjybR934QNHSJZPTQ= +github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/bytedance/sonic v1.13.1 h1:Jyd5CIvdFnkOWuKXr+wm4Nyk2h0yAFsr8ucJgEasO3g= +github.com/bytedance/sonic v1.13.1/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= +github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= +github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nbd-wtf/go-nostr v0.51.12 h1:MRQcrShiW/cHhnYSVDQ4SIEc7DlYV7U7gg/l4H4gbbE= +github.com/nbd-wtf/go-nostr v0.51.12/go.mod h1:IF30/Cm4AS90wd1GjsFJbBqq7oD1txo+2YUFYXqK3Nc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg= +github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= +github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI= +github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= +golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/pkg/graph/graph.go b/pkg/graph/graph.go index 1885a6b..9344974 100644 --- a/pkg/graph/graph.go +++ b/pkg/graph/graph.go @@ -1,33 +1,70 @@ package graph +import ( + "time" +) + +const ( + // types of status + StatusActive string = "active" // meaning, we generate random walks for this node + StatusInactive string = "inactive" + + // internal record kinds + Addition int = -3 + Promotion int = -2 + Demotion int = -1 +) + type ID string -// Delta represent the changes a Node made to its follow list. -// It Removed some nodes, and Added some others. -// This means the old follow list is Removed + Common, while the new is Common + Added +func (id ID) MarshalBinary() ([]byte, error) { return []byte(id), nil } + +// Node contains the metadata about a node, including a collection of Records. +type Node struct { + ID ID + Pubkey string + Status string // either [StatusActive] or [StatusInactive] + Records []Record +} + +// Record contains the timestamp of a node update. +type Record struct { + Kind int // either [Addition], [Promotion] or [Demotion] + Timestamp time.Time +} + +// Delta represents updates to apply to a Node. +// Add and Remove contain node IDs to add to or remove from the node’s relationships (e.g., follow list). type Delta struct { - Node ID - Removed []ID - Common []ID - Added []ID + Kind int + Node ID + Remove []ID + Keep []ID + Add []ID +} + +// Size returns the number of relationships changed by delta +func (d Delta) Size() int { + return len(d.Remove) + len(d.Add) } // Old returns the old state of the delta func (d Delta) Old() []ID { - return append(d.Common, d.Removed...) + return append(d.Keep, d.Remove...) } // New returns the new state of the delta func (d Delta) New() []ID { - return append(d.Common, d.Added...) + return append(d.Keep, d.Add...) } // Inverse of the delta. If a delta and it's inverse are applied, the graph returns to its original state. func (d Delta) Inverse() Delta { return Delta{ - Node: d.Node, - Common: d.Common, - Removed: d.Added, - Added: d.Removed, + Kind: d.Kind, + Node: d.Node, + Keep: d.Keep, + Remove: d.Add, + Add: d.Remove, } } diff --git a/pkg/redb/graph.go b/pkg/redb/graph.go new file mode 100644 index 0000000..0c2fc72 --- /dev/null +++ b/pkg/redb/graph.go @@ -0,0 +1,359 @@ +package redb + +import ( + "context" + "errors" + "fmt" + "github/pippellia-btc/crawler/pkg/graph" + "strconv" + "strings" + "time" + + "github.com/nbd-wtf/go-nostr" + "github.com/redis/go-redis/v9" +) + +const ( + // redis variable names + KeyDatabase string = "database" // TODO: this can be removed + KeyLastNodeID string = "lastNodeID" // TODO: change it to "next" inside "node" hash + KeyKeyIndex string = "keyIndex" // TODO: change to key_index + KeyNodePrefix string = "node:" + KeyFollowsPrefix string = "follows:" + KeyFollowersPrefix string = "followers:" + + // redis node HASH fields + NodeID string = "id" + NodePubkey string = "pubkey" + NodeStatus string = "status" + NodePromotionTS string = "promotion_TS" // TODO: change to promotion + NodeDemotionTS string = "demotion_TS" // TODO: change to demotion + NodeAddedTS string = "added_TS" // TODO: change to addition +) + +var ( + ErrNodeNotFound = errors.New("node not found") + ErrNodeAlreadyExists = errors.New("node already exists") +) + +type RedisDB struct { + client *redis.Client +} + +func New(opt *redis.Options) RedisDB { + return RedisDB{client: redis.NewClient(opt)} +} + +// Size returns the DBSize of redis, which is the total number of keys +func (r RedisDB) Size(ctx context.Context) (int, error) { + size, err := r.client.DBSize(ctx).Result() + if err != nil { + return 0, err + } + return int(size), nil +} + +// NodeCount returns the number of nodes stored in redis (in the keyIndex) +func (r RedisDB) NodeCount(ctx context.Context) (int, error) { + nodes, err := r.client.HLen(ctx, KeyKeyIndex).Result() + if err != nil { + return 0, err + } + return int(nodes), nil +} + +// NodeByID fetches a node by its ID +func (r RedisDB) NodeByID(ctx context.Context, ID graph.ID) (*graph.Node, error) { + fields, err := r.client.HGetAll(ctx, node(ID)).Result() + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %w", node(ID), err) + } + + if len(fields) == 0 { + return nil, fmt.Errorf("failed to fetch %s: %w", node(ID), ErrNodeNotFound) + } + + return parseNode(fields) +} + +// NodeByKey fetches a node by its pubkey +func (r RedisDB) NodeByKey(ctx context.Context, pubkey string) (*graph.Node, error) { + ID, err := r.client.HGet(ctx, KeyKeyIndex, pubkey).Result() + if err != nil { + return nil, fmt.Errorf("failed to fetch ID of node with pubkey %s: %w", pubkey, err) + } + + fields, err := r.client.HGetAll(ctx, node(ID)).Result() + if err != nil { + return nil, fmt.Errorf("failed to fetch node with pubkey %s: %w", pubkey, err) + } + + if len(fields) == 0 { + return nil, fmt.Errorf("failed to fetch node with pubkey %s: %w", pubkey, ErrNodeNotFound) + } + + return parseNode(fields) +} + +func (r RedisDB) containsNode(ctx context.Context, ID graph.ID) (bool, error) { + exists, err := r.client.Exists(ctx, node(ID)).Result() + if err != nil { + return false, fmt.Errorf("failed to check for the existence of %v: %w", node(ID), err) + } + return exists == 1, nil +} + +// AddNode adds a new inactive node to the database and returns its assigned ID +func (r RedisDB) AddNode(ctx context.Context, pubkey string) (graph.ID, error) { + exists, err := r.client.HExists(ctx, KeyKeyIndex, pubkey).Result() + if err != nil { + return "", fmt.Errorf("failed to check for existence of pubkey %s: %w", pubkey, err) + } + + if exists { + return "", fmt.Errorf("failed to add node with pubkey %s: %w", pubkey, ErrNodeAlreadyExists) + } + + // get the ID outside the transaction, which implies there might be "holes", + // meaning IDs not associated with any node + next, err := r.client.HIncrBy(ctx, KeyDatabase, KeyLastNodeID, 1).Result() + if err != nil { + return "", fmt.Errorf("failed to add node with pubkey %s: failed to increment ID", pubkey) + } + ID := strconv.FormatInt(next-1, 10) + + pipe := r.client.TxPipeline() + pipe.HSetNX(ctx, KeyKeyIndex, pubkey, ID) + pipe.HSet(ctx, node(ID), NodeID, ID, NodePubkey, pubkey, NodeStatus, graph.StatusInactive, NodeAddedTS, time.Now().Unix()) + if _, err := pipe.Exec(ctx); err != nil { + return "", fmt.Errorf("failed to add node with pubkey %s: pipeline failed: %w", pubkey, err) + } + + return graph.ID(ID), nil +} + +// Promote changes the node status to active +func (r RedisDB) Promote(ctx context.Context, ID graph.ID) error { + err := r.client.HSet(ctx, node(ID), NodeStatus, graph.StatusActive, NodePromotionTS, time.Now().Unix()).Err() + if err != nil { + return fmt.Errorf("failed to promote %s: %w", node(ID), err) + } + return nil +} + +// Demote changes the node status to inactive +func (r RedisDB) Demote(ctx context.Context, ID graph.ID) error { + err := r.client.HSet(ctx, node(ID), NodeStatus, graph.StatusInactive, NodeDemotionTS, time.Now().Unix()).Err() + if err != nil { + return fmt.Errorf("failed to promote %s: %w", node(ID), err) + } + return nil +} + +// Follows returns the follow list of node. If node is not found, it returns [ErrNodeNotFound]. +func (r RedisDB) Follows(ctx context.Context, node graph.ID) ([]graph.ID, error) { + return r.members(ctx, follows, node) +} + +// Followers returns the list of followers of node. If node is not found, it returns [ErrNodeNotFound]. +func (r RedisDB) Followers(ctx context.Context, node graph.ID) ([]graph.ID, error) { + return r.members(ctx, followers, node) +} + +func (r RedisDB) members(ctx context.Context, key func(graph.ID) string, node graph.ID) ([]graph.ID, error) { + members, err := r.client.SMembers(ctx, key(node)).Result() + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %w", key(node), err) + } + + if len(members) == 0 { + // check if there are no members because node doesn't exists + ok, err := r.containsNode(ctx, node) + if err != nil { + return nil, err + } + + if !ok { + return nil, fmt.Errorf("failed to fetch %s: %w", key(node), ErrNodeNotFound) + } + } + + return toIDs(members), nil +} + +// FollowCounts returns the number of follows each node has. If a node is not found, it returns 0. +func (r RedisDB) FollowCounts(ctx context.Context, nodes ...graph.ID) ([]int, error) { + return r.counts(ctx, follows, nodes...) +} + +// FollowerCounts returns the number of followers each node has. If a node is not found, it returns 0. +func (r RedisDB) FollowerCounts(ctx context.Context, nodes ...graph.ID) ([]int, error) { + return r.counts(ctx, followers, nodes...) +} + +func (r RedisDB) counts(ctx context.Context, key func(graph.ID) string, nodes ...graph.ID) ([]int, error) { + if len(nodes) == 0 { + return nil, nil + } + + pipe := r.client.Pipeline() + cmds := make([]*redis.IntCmd, len(nodes)) + keys := make([]string, len(nodes)) + + for i, node := range nodes { + keys[i] = key(node) + cmds[i] = pipe.SCard(ctx, key(node)) + } + + if _, err := pipe.Exec(ctx); err != nil { + return nil, fmt.Errorf("failed to count the elements of %v: %w", keys, err) + } + + counts := make([]int, len(nodes)) + for i, cmd := range cmds { + counts[i] = int(cmd.Val()) + } + + return counts, nil +} + +// Update applies the delta to the graph. +func (r RedisDB) Update(ctx context.Context, delta *graph.Delta) error { + if delta.Size() == 0 { + return nil + } + + ok, err := r.containsNode(ctx, delta.Node) + if err != nil { + return fmt.Errorf("failed to update with delta %v: %w", delta, err) + } + + if !ok { + return fmt.Errorf("failed to update with delta %v: %w", delta, ErrNodeNotFound) + } + + switch delta.Kind { + case nostr.KindFollowList: + err = r.updateFollows(ctx, delta) + + default: + err = fmt.Errorf("unsupported kind %d", delta.Kind) + } + + if err != nil { + return fmt.Errorf("failed to update with delta %v: %w", delta, err) + } + + return nil +} + +func (r RedisDB) updateFollows(ctx context.Context, delta *graph.Delta) error { + pipe := r.client.TxPipeline() + if len(delta.Add) > 0 { + // add all node --> added + pipe.SAdd(ctx, follows(delta.Node), toStrings(delta.Add)) + + for _, a := range delta.Add { + pipe.SAdd(ctx, followers(a), delta.Node) + } + } + + if len(delta.Remove) > 0 { + // remove all node --> removed + pipe.SRem(ctx, follows(delta.Node), toStrings(delta.Remove)) + + for _, r := range delta.Remove { + pipe.SRem(ctx, followers(r), delta.Node) + } + } + + if _, err := pipe.Exec(ctx); err != nil { + return fmt.Errorf("pipeline failed: %w", err) + } + + return nil +} + +// NodeIDs returns a slice of node IDs assosiated with the pubkeys. +// If a pubkey is not found, an empty ID "" is returned +func (r RedisDB) NodeIDs(ctx context.Context, pubkeys ...string) ([]graph.ID, error) { + if len(pubkeys) == 0 { + return nil, nil + } + + IDs, err := r.client.HMGet(ctx, KeyKeyIndex, pubkeys...).Result() + if err != nil { + return nil, fmt.Errorf("failed to fetch the node IDs of %v: %w", pubkeys, err) + } + + nodes := make([]graph.ID, len(IDs)) + for i, ID := range IDs { + switch ID { + case nil: + nodes[i] = "" // empty ID means missing pubkey + + default: + nodes[i] = graph.ID(ID.(string)) // no need to type-assert because everything in redis is a string + } + } + + return nodes, nil +} + +// Pubkeys returns a slice of pubkeys assosiated with the node IDs. +// If a node ID is not found, an empty pubkey "" is returned +func (r RedisDB) Pubkeys(ctx context.Context, nodes ...graph.ID) ([]string, error) { + if len(nodes) == 0 { + return nil, nil + } + + pipe := r.client.Pipeline() + cmds := make([]*redis.StringCmd, len(nodes)) + for i, ID := range nodes { + cmds[i] = pipe.HGet(ctx, node(ID), NodePubkey) + } + + if _, err := pipe.Exec(ctx); err != nil && !errors.Is(err, redis.Nil) { + // deal later with redis.Nil, which means node(s) not found + return nil, fmt.Errorf("failed to fetch the pubkeys of %v: pipeline failed: %w", nodes, err) + } + + pubkeys := make([]string, len(nodes)) + for i, cmd := range cmds { + switch { + case errors.Is(cmd.Err(), redis.Nil): + pubkeys[i] = "" // empty pubkey means missing node + + case cmd.Err() != nil: + return nil, fmt.Errorf("failed to fetch the pubkeys of %v: %w", nodes, cmd.Err()) + + default: + pubkeys[i] = cmd.Val() + } + } + + return pubkeys, nil +} + +// ScanNodes to return a batch of node IDs of size roughly proportional to limit. +// Limit controls how much "work" is invested in fetching the batch, hence it's not precise. +// Learn more about scan: https://redis.io/docs/latest/commands/scan/ +func (r RedisDB) ScanNodes(ctx context.Context, cursor uint64, limit int) ([]graph.ID, uint64, error) { + match := KeyNodePrefix + "*" + keys, cursor, err := r.client.Scan(ctx, cursor, match, int64(limit)).Result() + if err != nil { + return nil, 0, fmt.Errorf("failed to scan for keys matching %s: %w", match, err) + } + + nodes := make([]graph.ID, len(keys)) + for i, key := range keys { + node, found := strings.CutPrefix(key, KeyNodePrefix) + if !found { + return nil, 0, fmt.Errorf("failed to scan for keys matching %s: bad match %s", match, node) + } + + nodes[i] = graph.ID(node) + } + + return nodes, cursor, nil +} diff --git a/pkg/redb/graph_test.go b/pkg/redb/graph_test.go new file mode 100644 index 0000000..fc99475 --- /dev/null +++ b/pkg/redb/graph_test.go @@ -0,0 +1,357 @@ +package redb + +import ( + "context" + "errors" + "github/pippellia-btc/crawler/pkg/graph" + "reflect" + "testing" + "time" + + "github.com/nbd-wtf/go-nostr" + "github.com/redis/go-redis/v9" +) + +var ctx = context.Background() + +func TestParseNode(t *testing.T) { + tests := []struct { + name string + fields map[string]string + expected *graph.Node + err error + }{ + { + name: "nil map", + }, + { + name: "empty map", + fields: map[string]string{}, + }, + { + name: "valid no records", + fields: map[string]string{ + NodeID: "19", + NodePubkey: "nineteen", + NodeStatus: graph.StatusActive, + }, + expected: &graph.Node{ + ID: "19", + Pubkey: "nineteen", + Status: graph.StatusActive, + }, + }, + { + name: "valid with record", + fields: map[string]string{ + NodeID: "19", + NodePubkey: "nineteen", + NodeStatus: graph.StatusActive, + NodeAddedTS: "1", + }, + expected: &graph.Node{ + ID: "19", + Pubkey: "nineteen", + Status: graph.StatusActive, + Records: []graph.Record{ + {Kind: graph.Addition, Timestamp: time.Unix(1, 0)}, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + node, err := parseNode(test.fields) + if !errors.Is(err, test.err) { + t.Fatalf("expected %v got %v", test.err, err) + } + + if !reflect.DeepEqual(node, test.expected) { + t.Fatalf("ParseNode(): expected node %v got %v", test.expected, node) + } + }) + } +} + +func TestAddNode(t *testing.T) { + t.Run("node already exists", func(t *testing.T) { + db, err := OneNode() + if err != nil { + t.Fatalf("setup failed: %v", err) + } + defer db.flushAll() + + if _, err = db.AddNode(ctx, "0"); !errors.Is(err, ErrNodeAlreadyExists) { + t.Fatalf("expected error %v, got %v", ErrNodeAlreadyExists, err) + } + }) + + t.Run("valid", func(t *testing.T) { + db, err := OneNode() + if err != nil { + t.Fatalf("setup failed: %v", err) + } + defer db.flushAll() + + ID, err := db.AddNode(ctx, "xxx") + if err != nil { + t.Fatalf("expected nil, got %v", err) + } + + expected := &graph.Node{ + ID: "1", + Pubkey: "xxx", + Status: graph.StatusInactive, + Records: []graph.Record{{Kind: graph.Addition, Timestamp: time.Unix(time.Now().Unix(), 0)}}, + } + + if ID != expected.ID { + t.Fatalf("expected ID %s, got %s", expected.ID, ID) + } + + node, err := db.NodeByKey(ctx, "xxx") + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(node, expected) { + t.Fatalf("expected node %v, got %v", expected, node) + } + }) +} + +func TestMembers(t *testing.T) { + tests := []struct { + name string + setup func() (RedisDB, error) + node graph.ID + expected []graph.ID + err error + }{ + { + name: "empty database", + setup: Empty, + node: "0", + err: ErrNodeNotFound, + }, + { + name: "node not found", + setup: OneNode, + node: "1", + err: ErrNodeNotFound, + }, + { + name: "dandling node", + setup: OneNode, + node: "0", + expected: []graph.ID{}, + }, + { + name: "valid", + setup: Simple, + node: "0", + expected: []graph.ID{"1"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db, err := test.setup() + if err != nil { + t.Fatalf("setup failed: %v", err) + } + defer db.flushAll() + + follows, err := db.members(ctx, follows, test.node) + if !errors.Is(err, test.err) { + t.Fatalf("expected error %v, got %v", test.err, err) + } + + if !reflect.DeepEqual(follows, test.expected) { + t.Errorf("expected follows %v, got %v", test.expected, follows) + } + }) + } +} + +func TestUpdateFollows(t *testing.T) { + db, err := Simple() + if err != nil { + t.Fatalf("setup failed: %v", err) + } + defer db.flushAll() + + delta := &graph.Delta{ + Kind: nostr.KindFollowList, + Node: "0", + Remove: []graph.ID{"1"}, + Add: []graph.ID{"2"}, + } + + if err := db.Update(ctx, delta); err != nil { + t.Fatalf("expected error nil, got %v", err) + } + + follows, err := db.Follows(ctx, "0") + if err != nil { + t.Fatalf("expected nil got %v", err) + } + + if !reflect.DeepEqual(follows, []graph.ID{"2"}) { + t.Fatalf("expected follows(0) %v, got %v", []graph.ID{"2"}, follows) + } + + followers, err := db.Followers(ctx, "1") + if err != nil { + t.Fatalf("expected nil got %v", err) + } + + if !reflect.DeepEqual(followers, []graph.ID{}) { + t.Fatalf("expected followers(1) %v, got %v", []graph.ID{}, followers) + } + + followers, err = db.Followers(ctx, "2") + if err != nil { + t.Fatalf("expected nil got %v", err) + } + + if !reflect.DeepEqual(followers, []graph.ID{"0"}) { + t.Fatalf("expected followers(2) %v, got %v", []graph.ID{"0"}, followers) + } +} + +func TestNodeIDs(t *testing.T) { + tests := []struct { + name string + setup func() (RedisDB, error) + pubkeys []string + expected []graph.ID + }{ + { + name: "empty database", + setup: Empty, + pubkeys: []string{"0"}, + expected: []graph.ID{""}, + }, + { + name: "node not found", + setup: OneNode, + pubkeys: []string{"1"}, + expected: []graph.ID{""}, + }, + { + name: "valid", + setup: Simple, + pubkeys: []string{"0", "1", "69"}, + expected: []graph.ID{"0", "1", ""}, // last is not found + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db, err := test.setup() + if err != nil { + t.Fatalf("setup failed: %v", err) + } + defer db.flushAll() + + nodes, err := db.NodeIDs(ctx, test.pubkeys...) + if err != nil { + t.Fatalf("expected error nil, got %v", err) + } + + if !reflect.DeepEqual(nodes, test.expected) { + t.Fatalf("expected nodes %v, got %v", test.expected, nodes) + } + }) + } +} + +func TestPubkeys(t *testing.T) { + tests := []struct { + name string + setup func() (RedisDB, error) + nodes []graph.ID + expected []string + }{ + { + name: "empty database", + setup: Empty, + nodes: []graph.ID{"0"}, + expected: []string{""}, + }, + { + name: "node not found", + setup: OneNode, + nodes: []graph.ID{"1"}, + expected: []string{""}, + }, + { + name: "valid", + setup: Simple, + nodes: []graph.ID{"0", "1", "69"}, + expected: []string{"0", "1", ""}, // last is not found + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db, err := test.setup() + if err != nil { + t.Fatalf("setup failed: %v", err) + } + defer db.flushAll() + + pubkeys, err := db.Pubkeys(ctx, test.nodes...) + if err != nil { + t.Fatalf("expected error nil, got %v", err) + } + + if !reflect.DeepEqual(pubkeys, test.expected) { + t.Fatalf("expected pubkeys %v, got %v", test.expected, pubkeys) + } + }) + } +} + +// ------------------------------------- HELPERS ------------------------------- + +func Empty() (RedisDB, error) { + return New(&redis.Options{Addr: testAddress}), nil +} + +func OneNode() (RedisDB, error) { + db := New(&redis.Options{Addr: testAddress}) + if _, err := db.AddNode(context.Background(), "0"); err != nil { + db.flushAll() + return RedisDB{}, err + } + + return db, nil +} + +func Simple() (RedisDB, error) { + ctx := context.Background() + db := New(&redis.Options{Addr: testAddress}) + + for _, pk := range []string{"0", "1", "2"} { + if _, err := db.AddNode(ctx, pk); err != nil { + db.flushAll() + return RedisDB{}, err + } + } + + // 0 ---> 1 + if err := db.client.SAdd(ctx, follows("0"), "1").Err(); err != nil { + db.flushAll() + return RedisDB{}, err + } + + if err := db.client.SAdd(ctx, followers("1"), "0").Err(); err != nil { + db.flushAll() + return RedisDB{}, err + } + + return db, nil +} diff --git a/pkg/redb/utils.go b/pkg/redb/utils.go new file mode 100644 index 0000000..74f4bc5 --- /dev/null +++ b/pkg/redb/utils.go @@ -0,0 +1,100 @@ +package redb + +import ( + "context" + "github/pippellia-btc/crawler/pkg/graph" + "strconv" + "time" +) + +var ( + testAddress = "localhost:6380" +) + +// flushAll deletes all the keys of all existing databases. This command never fails. +func (r RedisDB) flushAll() { + r.client.FlushAll(context.Background()) +} + +func node[ID string | graph.ID](id ID) string { + return KeyNodePrefix + string(id) +} + +func follows[ID string | graph.ID](id ID) string { + return KeyFollowsPrefix + string(id) +} + +func followers[ID string | graph.ID](id ID) string { + return KeyFollowersPrefix + string(id) +} + +// ids converts a slice of strings to IDs +func toIDs(s []string) []graph.ID { + IDs := make([]graph.ID, len(s)) + for i, e := range s { + IDs[i] = graph.ID(e) + } + return IDs +} + +// strings converts graph IDs to a slice of strings +func toStrings(ids []graph.ID) []string { + s := make([]string, len(ids)) + for i, id := range ids { + s[i] = string(id) + } + return s +} + +// parseNode() parses the map into a node structure +func parseNode(fields map[string]string) (*graph.Node, error) { + if len(fields) == 0 { + return nil, nil + } + + var node graph.Node + for key, val := range fields { + switch key { + case NodeID: + node.ID = graph.ID(val) + + case NodePubkey: + node.Pubkey = val + + case NodeStatus: + node.Status = val + + case NodeAddedTS: + ts, err := parseTimestamp(val) + if err != nil { + return nil, err + } + node.Records = append(node.Records, graph.Record{Kind: graph.Addition, Timestamp: ts}) + + case NodePromotionTS: + ts, err := parseTimestamp(val) + if err != nil { + return nil, err + } + node.Records = append(node.Records, graph.Record{Kind: graph.Promotion, Timestamp: ts}) + + case NodeDemotionTS: + ts, err := parseTimestamp(val) + if err != nil { + return nil, err + } + node.Records = append(node.Records, graph.Record{Kind: graph.Demotion, Timestamp: ts}) + } + } + + return &node, nil +} + +// parseTimestamp() parses a unix timestamp string into a time.Time +func parseTimestamp(unix string) (time.Time, error) { + ts, err := strconv.ParseInt(unix, 10, 64) + if err != nil { + return time.Time{}, err + } + return time.Unix(ts, 0), nil +} diff --git a/pkg/redb/walks.go b/pkg/redb/walks.go new file mode 100644 index 0000000..9195bd4 --- /dev/null +++ b/pkg/redb/walks.go @@ -0,0 +1 @@ +package redb diff --git a/pkg/walks/walks.go b/pkg/walks/walks.go index c6d3ea1..66f302d 100644 --- a/pkg/walks/walks.go +++ b/pkg/walks/walks.go @@ -179,7 +179,7 @@ func ToUpdate(ctx context.Context, walker Walker, delta graph.Delta, walks []Wal } shouldResample = rand.Float64() < resampleProbability - isInvalid = (pos < walk.Len()-1) && slices.Contains(delta.Removed, walk.Path[pos+1]) + isInvalid = (pos < walk.Len()-1) && slices.Contains(delta.Remove, walk.Path[pos+1]) switch { case shouldResample: @@ -188,7 +188,7 @@ func ToUpdate(ctx context.Context, walker Walker, delta graph.Delta, walks []Wal updated.Prune(pos + 1) if rand.Float64() < Alpha { - new, err := generate(ctx, walker, delta.Added...) + new, err := generate(ctx, walker, delta.Add...) if err != nil { return nil, fmt.Errorf("ToUpdate: failed to generate new segment: %w", err) } @@ -203,7 +203,7 @@ func ToUpdate(ctx context.Context, walker Walker, delta graph.Delta, walks []Wal updated := walk.Copy() updated.Prune(pos + 1) - new, err := generate(ctx, walker, delta.Common...) + new, err := generate(ctx, walker, delta.Keep...) if err != nil { return nil, fmt.Errorf("ToUpdate: failed to generate new segment: %w", err) } @@ -223,24 +223,24 @@ func ToUpdate(ctx context.Context, walker Walker, delta graph.Delta, walks []Wal // Our goal is to have 1/3 of the walks that continue go to each of 1, 2 and 3. // This means we have to re-do 2/3 of the walks and make them continue towards 2 or 3. func resampleProbability(delta graph.Delta) float64 { - if len(delta.Added) == 0 { + if len(delta.Add) == 0 { return 0 } - c := float64(len(delta.Common)) - a := float64(len(delta.Added)) + c := float64(len(delta.Keep)) + a := float64(len(delta.Add)) return a / (a + c) } func expectedUpdates(walks []Walk, delta graph.Delta) int { - if len(delta.Common) == 0 { + if len(delta.Keep) == 0 { // no nodes have remained, all walks must be re-computed return len(walks) } - r := float64(len(delta.Removed)) - c := float64(len(delta.Common)) - a := float64(len(delta.Added)) + r := float64(len(delta.Remove)) + c := float64(len(delta.Keep)) + a := float64(len(delta.Add)) invalidProbability := Alpha * r / (r + c) resampleProbability := a / (a + c) diff --git a/pkg/walks/walks_test.go b/pkg/walks/walks_test.go index 701b5ff..c1a2c61 100644 --- a/pkg/walks/walks_test.go +++ b/pkg/walks/walks_test.go @@ -62,9 +62,9 @@ func TestUpdateRemove(t *testing.T) { }) delta := graph.Delta{ - Node: "0", - Removed: []graph.ID{"1"}, // the old follows were "1" and "3" - Common: []graph.ID{"3"}, + Node: "0", + Remove: []graph.ID{"1"}, // the old follows were "1" and "3" + Keep: []graph.ID{"3"}, } walks := []Walk{ diff --git a/tests/random/utils_test.go b/tests/random/utils_test.go index 6c99f56..120f215 100644 --- a/tests/random/utils_test.go +++ b/tests/random/utils_test.go @@ -38,7 +38,7 @@ func Dandlings(n int) Setup { personalized := make([]float64, n) personalized[0] = 1 - added := make([]graph.ID, 0, n-1) + add := make([]graph.ID, 0, n-1) deltas := make([]graph.Delta, 0, n-1) for i := range n { @@ -48,8 +48,8 @@ func Dandlings(n int) Setup { if i > 0 { // all the possible deltas modulo graph isomorphism; 0 --> [1,2, ... k] for 1 <= k <= n - added = append(added, node) - deltas = append(deltas, graph.Delta{Node: "0", Added: added}) + add = append(add, node) + deltas = append(deltas, graph.Delta{Node: "0", Add: add}) } } @@ -79,9 +79,9 @@ func Cyclic(n int) Setup { return Setup{ walker: walks.NewCyclicWalker(n), deltas: []graph.Delta{ - {Node: "0", Removed: []graph.ID{"1"}}, - {Node: "0", Common: []graph.ID{"1"}, Added: []graph.ID{mid}}, - {Node: "0", Removed: []graph.ID{"1"}, Added: []graph.ID{mid}}, + {Node: "0", Remove: []graph.ID{"1"}}, + {Node: "0", Keep: []graph.ID{"1"}, Add: []graph.ID{mid}}, + {Node: "0", Remove: []graph.ID{"1"}, Add: []graph.ID{mid}}, }, nodes: nodes, @@ -102,33 +102,33 @@ var Acyclic1 = Setup{ }), deltas: []graph.Delta{ // removals - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}}, - {Node: "0", Removed: []graph.ID{"2"}, Common: []graph.ID{"1"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}}, - {Node: "2", Removed: []graph.ID{"3"}}, - {Node: "3", Removed: []graph.ID{"1"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}}, + {Node: "0", Remove: []graph.ID{"2"}, Keep: []graph.ID{"1"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}}, + {Node: "2", Remove: []graph.ID{"3"}}, + {Node: "3", Remove: []graph.ID{"1"}}, // additions - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}}, - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"4"}}, - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "4"}}, - {Node: "4", Added: []graph.ID{"0"}}, - {Node: "4", Added: []graph.ID{"1"}}, - {Node: "4", Added: []graph.ID{"2"}}, - {Node: "4", Added: []graph.ID{"3"}}, - {Node: "4", Added: []graph.ID{"1", "2"}}, - {Node: "4", Added: []graph.ID{"2", "3"}}, - {Node: "4", Added: []graph.ID{"3", "4"}}, - {Node: "4", Added: []graph.ID{"0", "1", "2"}}, - {Node: "4", Added: []graph.ID{"0", "1", "2", "3"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"4"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "4"}}, + {Node: "4", Add: []graph.ID{"0"}}, + {Node: "4", Add: []graph.ID{"1"}}, + {Node: "4", Add: []graph.ID{"2"}}, + {Node: "4", Add: []graph.ID{"3"}}, + {Node: "4", Add: []graph.ID{"1", "2"}}, + {Node: "4", Add: []graph.ID{"2", "3"}}, + {Node: "4", Add: []graph.ID{"3", "4"}}, + {Node: "4", Add: []graph.ID{"0", "1", "2"}}, + {Node: "4", Add: []graph.ID{"0", "1", "2", "3"}}, // removals and additions - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"4"}}, - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"4"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "4"}}, - {Node: "2", Removed: []graph.ID{"3"}, Added: []graph.ID{"1"}}, - {Node: "2", Removed: []graph.ID{"3"}, Added: []graph.ID{"4"}}, - {Node: "2", Removed: []graph.ID{"3"}, Added: []graph.ID{"1", "4"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"4"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"4"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "4"}}, + {Node: "2", Remove: []graph.ID{"3"}, Add: []graph.ID{"1"}}, + {Node: "2", Remove: []graph.ID{"3"}, Add: []graph.ID{"4"}}, + {Node: "2", Remove: []graph.ID{"3"}, Add: []graph.ID{"1", "4"}}, }, nodes: []graph.ID{"0", "1", "2", "3", "4"}, global: []float64{0.11185, 0.36950, 0.15943, 0.24736, 0.11185}, @@ -146,20 +146,20 @@ var Acyclic2 = Setup{ }), deltas: []graph.Delta{ // removals - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}}, // additions - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}}, - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"4"}}, - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "4"}}, - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "5"}}, - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3", "4", "5"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"4"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "4"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "5"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3", "4", "5"}}, // removals and additions - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3"}}, - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"4"}}, - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3", "4"}}, - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3", "5"}}, - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3", "4", "5"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"4"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3", "4"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3", "5"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3", "4", "5"}}, }, nodes: []graph.ID{"0", "1", "2", "3", "4", "5"}, global: []float64{0.12987, 0.18506, 0.18506, 0.18506, 0.12987, 0.18506}, @@ -175,14 +175,14 @@ var Acyclic3 = Setup{ }), deltas: []graph.Delta{ // removals - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}}, // additions - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}}, - {Node: "2", Added: []graph.ID{"1"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}}, + {Node: "2", Add: []graph.ID{"1"}}, // removals and additions - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}}, }, nodes: []graph.ID{"0", "1", "2", "3"}, global: []float64{0.17544, 0.32456, 0.32456, 0.17544}, @@ -198,19 +198,19 @@ var Acyclic4 = Setup{ }), deltas: []graph.Delta{ // removals - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}}, - {Node: "3", Removed: []graph.ID{"1"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}}, + {Node: "3", Remove: []graph.ID{"1"}}, // additions - {Node: "0", Common: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}}, - {Node: "2", Added: []graph.ID{"1"}}, - {Node: "2", Added: []graph.ID{"3"}}, - {Node: "3", Common: []graph.ID{"1"}, Added: []graph.ID{"0"}}, + {Node: "0", Keep: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}}, + {Node: "2", Add: []graph.ID{"1"}}, + {Node: "2", Add: []graph.ID{"3"}}, + {Node: "3", Keep: []graph.ID{"1"}, Add: []graph.ID{"0"}}, // removals and additions - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2"}, Added: []graph.ID{"3"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}, Added: []graph.ID{"3"}}, - {Node: "3", Removed: []graph.ID{"1"}, Added: []graph.ID{"0"}}, - {Node: "3", Removed: []graph.ID{"1"}, Added: []graph.ID{"0", "2"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2"}, Add: []graph.ID{"3"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}, Add: []graph.ID{"3"}}, + {Node: "3", Remove: []graph.ID{"1"}, Add: []graph.ID{"0"}}, + {Node: "3", Remove: []graph.ID{"1"}, Add: []graph.ID{"0", "2"}}, }, nodes: []graph.ID{"0", "1", "2", "3"}, global: []float64{0.17544, 0.39912, 0.25, 0.17544}, @@ -226,19 +226,19 @@ var Acyclic5 = Setup{ }), deltas: []graph.Delta{ // removals - {Node: "0", Removed: []graph.ID{"3"}}, - {Node: "1", Removed: []graph.ID{"0"}}, - {Node: "3", Removed: []graph.ID{"2"}}, + {Node: "0", Remove: []graph.ID{"3"}}, + {Node: "1", Remove: []graph.ID{"0"}}, + {Node: "3", Remove: []graph.ID{"2"}}, // additions - {Node: "0", Common: []graph.ID{"3"}, Added: []graph.ID{"2"}}, - {Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"2"}}, - {Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"3"}}, - {Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"2", "3"}}, + {Node: "0", Keep: []graph.ID{"3"}, Add: []graph.ID{"2"}}, + {Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"2"}}, + {Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"3"}}, + {Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"2", "3"}}, // removals and additions - {Node: "0", Removed: []graph.ID{"3"}, Added: []graph.ID{"2"}}, - {Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"2"}}, - {Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"3"}}, - {Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"2", "3"}}, + {Node: "0", Remove: []graph.ID{"3"}, Add: []graph.ID{"2"}}, + {Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"2"}}, + {Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"3"}}, + {Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"2", "3"}}, }, nodes: []graph.ID{"0", "1", "2", "3"}, global: []float64{0.21489, 0.11616, 0.37015, 0.29881}, @@ -255,34 +255,34 @@ var Acyclic6 = Setup{ }), deltas: []graph.Delta{ // removals - {Node: "0", Removed: []graph.ID{"4"}}, - {Node: "1", Removed: []graph.ID{"0"}}, - {Node: "3", Removed: []graph.ID{"1"}, Common: []graph.ID{"4"}}, - {Node: "3", Removed: []graph.ID{"4"}, Common: []graph.ID{"1"}}, - {Node: "3", Removed: []graph.ID{"1", "4"}}, - {Node: "4", Removed: []graph.ID{"2"}}, + {Node: "0", Remove: []graph.ID{"4"}}, + {Node: "1", Remove: []graph.ID{"0"}}, + {Node: "3", Remove: []graph.ID{"1"}, Keep: []graph.ID{"4"}}, + {Node: "3", Remove: []graph.ID{"4"}, Keep: []graph.ID{"1"}}, + {Node: "3", Remove: []graph.ID{"1", "4"}}, + {Node: "4", Remove: []graph.ID{"2"}}, // additions - {Node: "0", Common: []graph.ID{"4"}, Added: []graph.ID{"2"}}, - {Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"2"}}, - {Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"4"}}, - {Node: "1", Common: []graph.ID{"0"}, Added: []graph.ID{"2", "4"}}, - {Node: "3", Common: []graph.ID{"1", "4"}, Added: []graph.ID{"0"}}, - {Node: "3", Common: []graph.ID{"1", "4"}, Added: []graph.ID{"2"}}, - {Node: "3", Common: []graph.ID{"1", "4"}, Added: []graph.ID{"0", "2"}}, + {Node: "0", Keep: []graph.ID{"4"}, Add: []graph.ID{"2"}}, + {Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"2"}}, + {Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"4"}}, + {Node: "1", Keep: []graph.ID{"0"}, Add: []graph.ID{"2", "4"}}, + {Node: "3", Keep: []graph.ID{"1", "4"}, Add: []graph.ID{"0"}}, + {Node: "3", Keep: []graph.ID{"1", "4"}, Add: []graph.ID{"2"}}, + {Node: "3", Keep: []graph.ID{"1", "4"}, Add: []graph.ID{"0", "2"}}, // removals and additions - {Node: "0", Removed: []graph.ID{"4"}, Added: []graph.ID{"2"}}, - {Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"2"}}, - {Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"4"}}, - {Node: "1", Removed: []graph.ID{"0"}, Added: []graph.ID{"2", "4"}}, - {Node: "3", Removed: []graph.ID{"1"}, Common: []graph.ID{"4"}, Added: []graph.ID{"0"}}, - {Node: "3", Removed: []graph.ID{"1"}, Common: []graph.ID{"4"}, Added: []graph.ID{"2"}}, - {Node: "3", Removed: []graph.ID{"1"}, Common: []graph.ID{"4"}, Added: []graph.ID{"0", "2"}}, - {Node: "3", Removed: []graph.ID{"4"}, Common: []graph.ID{"1"}, Added: []graph.ID{"0"}}, - {Node: "3", Removed: []graph.ID{"4"}, Common: []graph.ID{"1"}, Added: []graph.ID{"2"}}, - {Node: "3", Removed: []graph.ID{"4"}, Common: []graph.ID{"1"}, Added: []graph.ID{"0", "2"}}, - {Node: "3", Removed: []graph.ID{"1", "4"}, Added: []graph.ID{"0"}}, - {Node: "3", Removed: []graph.ID{"1", "4"}, Added: []graph.ID{"2"}}, - {Node: "3", Removed: []graph.ID{"1", "4"}, Added: []graph.ID{"0", "2"}}, + {Node: "0", Remove: []graph.ID{"4"}, Add: []graph.ID{"2"}}, + {Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"2"}}, + {Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"4"}}, + {Node: "1", Remove: []graph.ID{"0"}, Add: []graph.ID{"2", "4"}}, + {Node: "3", Remove: []graph.ID{"1"}, Keep: []graph.ID{"4"}, Add: []graph.ID{"0"}}, + {Node: "3", Remove: []graph.ID{"1"}, Keep: []graph.ID{"4"}, Add: []graph.ID{"2"}}, + {Node: "3", Remove: []graph.ID{"1"}, Keep: []graph.ID{"4"}, Add: []graph.ID{"0", "2"}}, + {Node: "3", Remove: []graph.ID{"4"}, Keep: []graph.ID{"1"}, Add: []graph.ID{"0"}}, + {Node: "3", Remove: []graph.ID{"4"}, Keep: []graph.ID{"1"}, Add: []graph.ID{"2"}}, + {Node: "3", Remove: []graph.ID{"4"}, Keep: []graph.ID{"1"}, Add: []graph.ID{"0", "2"}}, + {Node: "3", Remove: []graph.ID{"1", "4"}, Add: []graph.ID{"0"}}, + {Node: "3", Remove: []graph.ID{"1", "4"}, Add: []graph.ID{"2"}}, + {Node: "3", Remove: []graph.ID{"1", "4"}, Add: []graph.ID{"0", "2"}}, }, nodes: []graph.ID{"0", "1", "2", "3", "4"}, global: []float64{0.18820, 0.12128, 0.32417, 0.08511, 0.28125}, @@ -299,17 +299,17 @@ var Acyclic7 = Setup{ }), deltas: []graph.Delta{ // removals - {Node: "0", Removed: []graph.ID{"1"}, Common: []graph.ID{"2", "3"}}, - {Node: "0", Removed: []graph.ID{"1", "2"}, Common: []graph.ID{"3"}}, - {Node: "0", Removed: []graph.ID{"1", "2", "3"}}, - {Node: "4", Removed: []graph.ID{"0"}, Common: []graph.ID{"1", "2", "3"}}, - {Node: "4", Removed: []graph.ID{"1"}, Common: []graph.ID{"0", "2", "3"}}, - {Node: "4", Removed: []graph.ID{"1", "2"}, Common: []graph.ID{"0", "3"}}, - {Node: "4", Removed: []graph.ID{"1", "2", "3"}, Common: []graph.ID{"0"}}, - {Node: "4", Removed: []graph.ID{"0", "1", "2", "3"}}, + {Node: "0", Remove: []graph.ID{"1"}, Keep: []graph.ID{"2", "3"}}, + {Node: "0", Remove: []graph.ID{"1", "2"}, Keep: []graph.ID{"3"}}, + {Node: "0", Remove: []graph.ID{"1", "2", "3"}}, + {Node: "4", Remove: []graph.ID{"0"}, Keep: []graph.ID{"1", "2", "3"}}, + {Node: "4", Remove: []graph.ID{"1"}, Keep: []graph.ID{"0", "2", "3"}}, + {Node: "4", Remove: []graph.ID{"1", "2"}, Keep: []graph.ID{"0", "3"}}, + {Node: "4", Remove: []graph.ID{"1", "2", "3"}, Keep: []graph.ID{"0"}}, + {Node: "4", Remove: []graph.ID{"0", "1", "2", "3"}}, // additions - {Node: "1", Added: []graph.ID{"2"}}, - {Node: "1", Added: []graph.ID{"2", "3"}}, + {Node: "1", Add: []graph.ID{"2"}}, + {Node: "1", Add: []graph.ID{"2", "3"}}, }, nodes: []graph.ID{"0", "1", "2", "3", "4"}, global: []float64{0.17622, 0.22615, 0.22615, 0.22615, 0.14534},