From e16541c77dea319db0d2acd8f812fda947847073 Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Mon, 19 Sep 2022 17:39:09 +0300 Subject: [PATCH 1/4] Use initial_chanid and confirmed_chanid to handle zerconf channels --- db.go | 29 +++++++++++------- go.mod | 92 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 106 insertions(+), 15 deletions(-) diff --git a/db.go b/db.go index acc3202..131a019 100644 --- a/db.go +++ b/db.go @@ -75,12 +75,22 @@ func registerPayment(destination, paymentHash, paymentSecret []byte, incomingAmo } func insertChannel(chanID uint64, channelPoint string, nodeID []byte, lastUpdate time.Time) error { - _, err := pgxPool.Exec(context.Background(), - `INSERT INTO - channels (chanid, channel_point, nodeid, last_update) + var query string + sid := lnwire.NewShortChanIDFromInt(chanID) + if sid.IsFake() { + query = `INSERT INTO + channels (initial_chanid, channel_point, nodeid, last_update) VALUES ($1, $2, $3, $4) - ON CONFLICT (chanid) DO UPDATE SET last_update=$4`, - chanID, channelPoint, nodeID, lastUpdate) + ON CONFLICT (channel_point) DO UPDATE SET last_update=$4` + } else { + query = `INSERT INTO + channels (initial_chanid, confirmed_chanid, channel_point, nodeid, last_update) + VALUES ($1, $1, $2, $3, $4) + ON CONFLICT (channel_point) DO UPDATE SET confirmed_chanid=$1, last_update=$4` + } + + _, err := pgxPool.Exec(context.Background(), + query, chanID, channelPoint, nodeID, lastUpdate) if err != nil { return fmt.Errorf("insertChannel(%v, %s, %x) error: %w", chanID, channelPoint, nodeID, err) @@ -94,9 +104,9 @@ func confirmedChannels(sNodeID string) (map[string]uint64, error) { return nil, fmt.Errorf("hex.DecodeString(%v) error: %w", sNodeID, err) } rows, err := pgxPool.Query(context.Background(), - `SELECT chanid, channel_point + `SELECT confirmed_chanid, channel_point FROM channels - WHERE nodeid=$1`, + WHERE nodeid=$1 AND confirmed_chanid IS NOT NULL`, nodeID) if err != nil { return nil, fmt.Errorf("channels(%x) error: %w", nodeID, err) @@ -112,10 +122,7 @@ func confirmedChannels(sNodeID string) (map[string]uint64, error) { if err != nil { return nil, fmt.Errorf("channels(%x) rows.Scan error: %w", nodeID, err) } - sid := lnwire.NewShortChanIDFromInt(chanID) - if !sid.IsFake() { - chans[channelPoint] = chanID - } + chans[channelPoint] = chanID } return chans, rows.Err() } diff --git a/go.mod b/go.mod index 7573a3a..aafcf5f 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,11 @@ module github.com/breez/lspd -go 1.14 +go 1.19 require ( github.com/aws/aws-sdk-go v1.30.20 github.com/btcsuite/btcd v0.20.1-beta.0.20200730232343-1db1b6f8217f github.com/caddyserver/certmagic v0.11.2 - github.com/coreos/etcd v3.3.25+incompatible // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/golang/protobuf v1.4.2 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 github.com/jackc/pgtype v1.4.2 @@ -19,4 +16,91 @@ require ( google.golang.org/grpc v1.29.1 ) +require ( + github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect + github.com/aead/siphash v1.0.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect + github.com/btcsuite/btcutil v1.0.2 // indirect + github.com/btcsuite/btcutil/psbt v1.0.2 // indirect + github.com/btcsuite/btcwallet v0.11.1-0.20200814001439-1d31f4ea6fc5 // indirect + github.com/btcsuite/btcwallet/wallet/txauthor v1.0.0 // indirect + github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 // indirect + github.com/btcsuite/btcwallet/wallet/txsizes v1.0.0 // indirect + github.com/btcsuite/btcwallet/walletdb v1.3.3 // indirect + github.com/btcsuite/btcwallet/wtxmgr v1.2.0 // indirect + github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect + github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect + github.com/cenkalti/backoff/v4 v4.0.0 // indirect + github.com/coreos/bbolt v1.3.3 // indirect + github.com/coreos/etcd v3.3.25+incompatible // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/decred/dcrd/lru v1.0.0 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/go-acme/lego/v3 v3.7.0 // indirect + github.com/go-errors/errors v1.0.1 // indirect + github.com/gogo/protobuf v1.2.0 // indirect + github.com/google/btree v1.0.0 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.14.3 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.6.4 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.0.2 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/puddle v1.1.1 // indirect + github.com/jmespath/go-jmespath v0.3.0 // indirect + github.com/jonboulle/clockwork v0.1.0 // indirect + github.com/jrick/logrotate v1.0.0 // indirect + github.com/json-iterator/go v1.1.9 // indirect + github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect + github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec // indirect + github.com/klauspost/cpuid v1.2.3 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/lightninglabs/gozmq v0.0.0-20191113021534-d20a764486bf // indirect + github.com/lightninglabs/neutrino v0.11.1-0.20200316235139-bffc52e8f200 // indirect + github.com/lightningnetwork/lnd/clock v1.0.1 // indirect + github.com/lightningnetwork/lnd/queue v1.0.4 // indirect + github.com/lightningnetwork/lnd/ticker v1.0.0 // indirect + github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/miekg/dns v1.1.27 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/prometheus/client_golang v1.1.0 // indirect + github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect + github.com/prometheus/common v0.6.0 // indirect + github.com/prometheus/procfs v0.0.3 // indirect + github.com/rogpeppe/fastuuid v1.2.0 // indirect + github.com/sirupsen/logrus v1.4.2 // indirect + github.com/soheilhy/cmux v0.1.4 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/bbolt v1.3.5-0.20200615073812-232d8fc87f50 // indirect + go.uber.org/atomic v1.6.0 // indirect + go.uber.org/multierr v1.5.0 // indirect + go.uber.org/zap v1.14.1 // indirect + golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 // indirect + golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect + golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect + golang.org/x/text v0.3.3 // indirect + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect + google.golang.org/genproto v0.0.0-20200305110556-506484158171 // indirect + google.golang.org/protobuf v1.23.0 // indirect + gopkg.in/errgo.v1 v1.0.1 // indirect + gopkg.in/macaroon-bakery.v2 v2.0.1 // indirect + gopkg.in/macaroon.v2 v2.0.0 // indirect + gopkg.in/square/go-jose.v2 v2.3.1 // indirect + gopkg.in/yaml.v2 v2.2.8 // indirect + sigs.k8s.io/yaml v1.1.0 // indirect +) + replace github.com/lightningnetwork/lnd v0.11.0-beta => github.com/breez/lnd v0.11.0-beta.rc4.0.20210125150416-0c10146b223c From 6594e3de273c32a603ba347b2945d003e4435e49 Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Mon, 19 Sep 2022 20:47:21 +0300 Subject: [PATCH 2/4] Convert between int64 (in postgresql) and uint64 (short channel id) --- db.go | 6 +++--- forwarding_history.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db.go b/db.go index 131a019..b4b8ea6 100644 --- a/db.go +++ b/db.go @@ -90,7 +90,7 @@ func insertChannel(chanID uint64, channelPoint string, nodeID []byte, lastUpdate } _, err := pgxPool.Exec(context.Background(), - query, chanID, channelPoint, nodeID, lastUpdate) + query, int64(chanID), channelPoint, nodeID, lastUpdate) if err != nil { return fmt.Errorf("insertChannel(%v, %s, %x) error: %w", chanID, channelPoint, nodeID, err) @@ -115,14 +115,14 @@ func confirmedChannels(sNodeID string) (map[string]uint64, error) { chans := make(map[string]uint64) for rows.Next() { var ( - chanID uint64 + chanID int64 channelPoint string ) err = rows.Scan(&chanID, &channelPoint) if err != nil { return nil, fmt.Errorf("channels(%x) rows.Scan error: %w", nodeID, err) } - chans[channelPoint] = chanID + chans[channelPoint] = uint64(chanID) } return chans, rows.Err() } diff --git a/forwarding_history.go b/forwarding_history.go index 4a98caa..9ad9066 100644 --- a/forwarding_history.go +++ b/forwarding_history.go @@ -28,7 +28,7 @@ func (cfe *copyFromEvents) Values() ([]interface{}, error) { event := cfe.events[cfe.idx] values := []interface{}{ event.TimestampNs, - event.ChanIdIn, event.ChanIdOut, + int64(event.ChanIdIn), int64(event.ChanIdOut), event.AmtInMsat, event.AmtOutMsat} return values, nil } From 3a4892ce8a9d8c4837f32d6020763debdeff9f19 Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Mon, 19 Sep 2022 22:32:19 +0300 Subject: [PATCH 3/4] Add data migrations --- .../000008_one_record_per_channel.down.sql | 21 ++++++++++++++++++ .../000008_one_record_per_channel.up.sql | 22 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 postgresql/migrations/000008_one_record_per_channel.down.sql create mode 100644 postgresql/migrations/000008_one_record_per_channel.up.sql diff --git a/postgresql/migrations/000008_one_record_per_channel.down.sql b/postgresql/migrations/000008_one_record_per_channel.down.sql new file mode 100644 index 0000000..4ba53f6 --- /dev/null +++ b/postgresql/migrations/000008_one_record_per_channel.down.sql @@ -0,0 +1,21 @@ +ALTER INDEX public.channels_nodeid_idx RENAME TO channels_new_nodeid_idx; +ALTER INDEX public.channels_channel_point_pkey RENAME TO channels_new_channel_point_pkey +ALTER TABLE public.channels RENAME TO channels_new; + +CREATE TABLE public.channels ( + chanid int8 NOT NULL, + channel_point varchar NULL, + nodeid bytea NULL, + last_update timestamp NULL, + CONSTRAINT chanid_pkey PRIMARY KEY (chanid) +); +CREATE INDEX channels_nodeid_idx ON public.channels USING btree (nodeid); + +INSERT INTO public.channels +SELECT initial_chanid chanid, channel_point, nodeid, last_update FROM channels_new; + +INSERT INTO public.channels +SELECT confirmed_chanid chanid, channel_point, nodeid, last_update FROM channels_new + WHERE confirmed_chanid IS NOT NULL AND confirmed_chanid <> initial_chanid; + +DROP TABLE channels_new; diff --git a/postgresql/migrations/000008_one_record_per_channel.up.sql b/postgresql/migrations/000008_one_record_per_channel.up.sql new file mode 100644 index 0000000..2261a23 --- /dev/null +++ b/postgresql/migrations/000008_one_record_per_channel.up.sql @@ -0,0 +1,22 @@ +ALTER INDEX public.channels_nodeid_idx RENAME TO channels_old_nodeid_idx; +ALTER INDEX public.chanid_pkey RENAME TO channels_old_chanid_pkey; +ALTER TABLE public.channels RENAME TO channels_old; + +CREATE TABLE public.channels ( + initial_chanid int8 NOT NULL, + confirmed_chanid int8 NULL, + channel_point varchar NOT NULL, + nodeid bytea NOT NULL, + last_update timestamp NULL, + CONSTRAINT channels_channel_point_pkey PRIMARY KEY (channel_point) +); +CREATE INDEX channels_nodeid_idx ON public.channels USING btree (nodeid); + +INSERT INTO public.channels +SELECT + min(chanid) initial_chanid, + CASE WHEN (max(chanid) >> 40) < (3 << 17) THEN NULL ELSE max(chanid) END confirmed_chanid, + channel_point, nodeid, max(last_update) last_update +FROM channels_old GROUP BY channel_point, nodeid; + +DROP TABLE public.channels_old; From 7ceb5bf98866c249c4960e0d9d8e32e159032d7b Mon Sep 17 00:00:00 2001 From: Yaacov Akiba Slama Date: Wed, 21 Sep 2022 12:18:37 +0300 Subject: [PATCH 4/4] Add some logging in insertChannel --- db.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/db.go b/db.go index b4b8ea6..81ee7b4 100644 --- a/db.go +++ b/db.go @@ -89,12 +89,16 @@ func insertChannel(chanID uint64, channelPoint string, nodeID []byte, lastUpdate ON CONFLICT (channel_point) DO UPDATE SET confirmed_chanid=$1, last_update=$4` } - _, err := pgxPool.Exec(context.Background(), + c, err := pgxPool.Exec(context.Background(), query, int64(chanID), channelPoint, nodeID, lastUpdate) if err != nil { + log.Printf("insertChannel(%v, %s, %x) error: %v", + chanID, channelPoint, nodeID, err) return fmt.Errorf("insertChannel(%v, %s, %x) error: %w", chanID, channelPoint, nodeID, err) } + log.Printf("insertChannel(%v, %s, %x) result: %v", + chanID, channelPoint, nodeID, c.String()) return nil }