From 2074820d85c7d658cba5e9aa6712b6d32bae301e Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:33:53 +0200 Subject: [PATCH 01/10] htlcswitch: add htlcnotifier This commit adds a HTLCNotifier to htlcswitch which HTLC events will be piped through to provide clients with subscriptions to HTLC level events. The event types added are forward events (which occur for sends from and forwards through our node), forward failues (when a send or forward fails down the line), settles for forwards or receives to our node and link failures which occur when a htlc is failed at our node (which may occur for a send, receive or foreward). --- htlcswitch/htlcnotifier.go | 365 +++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 htlcswitch/htlcnotifier.go diff --git a/htlcswitch/htlcnotifier.go b/htlcswitch/htlcnotifier.go new file mode 100644 index 00000000..1fb523b8 --- /dev/null +++ b/htlcswitch/htlcnotifier.go @@ -0,0 +1,365 @@ +package htlcswitch + +import ( + "fmt" + "strings" + "sync" + "time" + + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/htlcswitch/hop" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/subscribe" +) + +// HtlcNotifier notifies clients of htlc forwards, failures and settles for +// htlcs that the switch handles. It takes subscriptions for its events and +// notifies them when htlc events occur. These are served on a best-effort +// basis; events are not persisted, delivery is not guaranteed (in the event +// of a crash in the switch, forward events may be lost) and some events may +// be replayed upon restart. Events consumed from this package should be +// de-duplicated by the htlc's unique combination of incoming+outgoing circuit +// and not relied upon for critical operations. +// +// The htlc notifier sends the following kinds of events: +// Forwarding Event: +// - Represents a htlc which is forwarded onward from our node. +// - Present for htlc forwards through our node and local sends. +// +// Link Failure Event: +// - Indicates that a htlc has failed on our incoming or outgoing link, +// with an incoming boolean which indicates where the failure occurred. +// - Incoming link failures are present for failed attempts to pay one of +// our invoices (insufficient amount or mpp timeout, for example) and for +// forwards that we cannot decode to forward onwards. +// - Outgoing link failures are present for forwards or local payments that +// do not meet our outgoing link's policy (insufficient fees, for example) +// and when we fail to forward the payment on (insufficient outgoing +// capacity, or an unknown outgoing link). +// +// Forwarding Failure Event: +// - Forwarding failures indicate that a htlc we forwarded has failed at +// another node down the route. +// - Present for local sends and htlc forwards which fail after they left +// our node. +// +// Settle event: +// - Settle events are present when a htlc which we added is settled through +// the release of a preimage. +// - Present for local receives, and successful local sends or forwards. +// +// Each htlc is identified by its incoming and outgoing circuit key. Htlcs, +// and their subsequent settles or fails, can be identified by the combination +// of incoming and outgoing circuits. Note that receives to our node will +// have a zero outgoing circuit key because the htlc terminates at our +// node, and sends from our node will have a zero incoming circuit key because +// the send originates at our node. +type HtlcNotifier struct { + started sync.Once + stopped sync.Once + + // now returns the current time, it is set in the htlcnotifier to allow + // for timestamp mocking in tests. + now func() time.Time + + ntfnServer *subscribe.Server +} + +// NewHtlcNotifier creates a new HtlcNotifier which gets htlc forwarded, +// failed and settled events from links our node has established with peers +// and sends notifications to subscribing clients. +func NewHtlcNotifier(now func() time.Time) *HtlcNotifier { + return &HtlcNotifier{ + now: now, + ntfnServer: subscribe.NewServer(), + } +} + +// Start starts the HtlcNotifier and all goroutines it needs to consume +// events and provide subscriptions to clients. +func (h *HtlcNotifier) Start() error { + var err error + h.started.Do(func() { + log.Trace("HtlcNotifier starting") + err = h.ntfnServer.Start() + }) + return err +} + +// Stop signals the notifier for a graceful shutdown. +func (h *HtlcNotifier) Stop() { + h.stopped.Do(func() { + if err := h.ntfnServer.Stop(); err != nil { + log.Warnf("error stopping htlc notifier: %v", err) + } + }) +} + +// SubscribeHtlcEvents returns a subscribe.Client that will receive updates +// any time the server is made aware of a new event. +func (h *HtlcNotifier) SubscribeHtlcEvents() (*subscribe.Client, error) { + return h.ntfnServer.Subscribe() +} + +// HtlcKey uniquely identifies the htlc. +type HtlcKey struct { + // IncomingCircuit is the channel an htlc id of the incoming htlc. + IncomingCircuit channeldb.CircuitKey + + // OutgoingCircuit is the channel and htlc id of the outgoing htlc. + OutgoingCircuit channeldb.CircuitKey +} + +// String returns a string representation of a htlc key. +func (k HtlcKey) String() string { + switch { + case k.IncomingCircuit.ChanID == hop.Source: + return k.OutgoingCircuit.String() + + case k.OutgoingCircuit.ChanID == hop.Exit: + return k.IncomingCircuit.String() + + default: + return fmt.Sprintf("%v -> %v", k.IncomingCircuit, + k.OutgoingCircuit) + } +} + +// HtlcInfo provides the details of a htlc that our node has processed. For +// forwards, incoming and outgoing values are set, whereas sends and receives +// will only have outgoing or incoming details set. +type HtlcInfo struct { + // IncomingTimelock is the time lock of the htlc on our incoming + // channel. + IncomingTimeLock uint32 + + // OutgoingTimelock is the time lock the htlc on our outgoing channel. + OutgoingTimeLock uint32 + + // IncomingAmt is the amount of the htlc on our incoming channel. + IncomingAmt lnwire.MilliSatoshi + + // OutgoingAmt is the amount of the htlc on our outgoing channel. + OutgoingAmt lnwire.MilliSatoshi +} + +// String returns a string representation of a htlc. +func (h HtlcInfo) String() string { + var details []string + + // If the incoming information is not zero, as is the case for a send, + // we include the incoming amount and timelock. + if h.IncomingAmt != 0 || h.IncomingTimeLock != 0 { + str := fmt.Sprintf("incoming amount: %v, "+ + "incoming timelock: %v", h.IncomingAmt, + h.IncomingTimeLock) + + details = append(details, str) + } + + // If the outgoing information is not zero, as is the case for a + // receive, we include the outgoing amount and timelock. + if h.OutgoingAmt != 0 || h.OutgoingTimeLock != 0 { + str := fmt.Sprintf("outgoing amount: %v, "+ + "outgoing timelock: %v", h.OutgoingAmt, + h.OutgoingTimeLock) + + details = append(details, str) + } + + return strings.Join(details, ", ") +} + +// HtlcEventType represents the type of event that a htlc was part of. +type HtlcEventType int + +const ( + // HtlcEventTypeSend represents a htlc that was part of a send from + // our node. + HtlcEventTypeSend HtlcEventType = iota + + // HtlcEventTypeReceive represents a htlc that was part of a receive + // to our node. + HtlcEventTypeReceive + + // HtlcEventTypeForward represents a htlc that was forwarded through + // our node. + HtlcEventTypeForward +) + +// String returns a string representation of a htlc event type. +func (h HtlcEventType) String() string { + switch h { + case HtlcEventTypeSend: + return "send" + + case HtlcEventTypeReceive: + return "receive" + + case HtlcEventTypeForward: + return "forward" + + default: + return "unknown" + } +} + +// ForwardingEvent represents a htlc that was forwarded onwards from our node. +// Sends which originate from our node will report forward events with zero +// incoming circuits in their htlc key. +type ForwardingEvent struct { + // HtlcKey uniquely identifies the htlc, and can be used to match the + // forwarding event with subsequent settle/fail events. + HtlcKey + + // HtlcInfo contains details about the htlc. + HtlcInfo + + // HtlcEventType classifies the event as part of a local send or + // receive, or as part of a forward. + HtlcEventType + + // Timestamp is the time when this htlc was forwarded. + Timestamp time.Time +} + +// LinkFailEvent describes a htlc that failed on our incoming or outgoing +// link. The incoming bool is true for failures on incoming links, and false +// for failures on outgoing links. The failure reason is provided by a lnwire +// failure message which is enriched with a failure detail in the cases where +// the wire failure message does not contain full information about the +// failure. +type LinkFailEvent struct { + // HtlcKey uniquely identifies the htlc. + HtlcKey + + // HtlcInfo contains details about the htlc. + HtlcInfo + + // HtlcEventType classifies the event as part of a local send or + // receive, or as part of a forward. + HtlcEventType + + // LinkError is the reason that we failed the htlc. + LinkError *LinkError + + // Incoming is true if the htlc was failed on an incoming link. + // If it failed on the outgoing link, it is false. + Incoming bool + + // Timestamp is the time when the link failure occurred. + Timestamp time.Time +} + +// ForwardingFailEvent represents a htlc failure which occurred down the line +// after we forwarded a htlc onwards. An error is not included in this event +// because errors returned down the route are encrypted. HtlcInfo is not +// reliably available for forwarding failures, so it is omitted. These events +// should be matched with their corresponding forward event to obtain this +// information. +type ForwardingFailEvent struct { + // HtlcKey uniquely identifies the htlc, and can be used to match the + // htlc with its corresponding forwarding event. + HtlcKey + + // HtlcEventType classifies the event as part of a local send or + // receive, or as part of a forward. + HtlcEventType + + // Timestamp is the time when the forwarding failure was received. + Timestamp time.Time +} + +// SettleEvent represents a htlc that was settled. HtlcInfo is not reliably +// available for forwarding failures, so it is omitted. These events should +// be matched with corresponding forward events or invoices (for receives) +// to obtain additional information about the htlc. +type SettleEvent struct { + // HtlcKey uniquely identifies the htlc, and can be used to match + // forwards with their corresponding forwarding event. + HtlcKey + + // HtlcEventType classifies the event as part of a local send or + // receive, or as part of a forward. + HtlcEventType + + // Timestamp is the time when this htlc was settled. + Timestamp time.Time +} + +// NotifyForwardingEvent notifies the HtlcNotifier than a htlc has been +// forwarded. +func (h *HtlcNotifier) NotifyForwardingEvent(key HtlcKey, info HtlcInfo, + eventType HtlcEventType) { + + event := &ForwardingEvent{ + HtlcKey: key, + HtlcInfo: info, + HtlcEventType: eventType, + Timestamp: h.now(), + } + + log.Tracef("Notifying forward event: %v over %v, %v", eventType, key, + info) + + if err := h.ntfnServer.SendUpdate(event); err != nil { + log.Warnf("Unable to send forwarding event: %v", err) + } +} + +// NotifyLinkFailEvent notifies that a htlc has failed on our incoming +// or outgoing link. +func (h *HtlcNotifier) NotifyLinkFailEvent(key HtlcKey, info HtlcInfo, + eventType HtlcEventType, linkErr *LinkError, incoming bool) { + + event := &LinkFailEvent{ + HtlcKey: key, + HtlcInfo: info, + HtlcEventType: eventType, + LinkError: linkErr, + Incoming: incoming, + Timestamp: h.now(), + } + + log.Tracef("Notifying link failure event: %v over %v, %v", eventType, + key, info) + + if err := h.ntfnServer.SendUpdate(event); err != nil { + log.Warnf("Unable to send link fail event: %v", err) + } +} + +// NotifyForwardingFailEvent notifies the HtlcNotifier that a htlc we +// forwarded has failed down the line. +func (h *HtlcNotifier) NotifyForwardingFailEvent(key HtlcKey, + eventType HtlcEventType) { + + event := &ForwardingFailEvent{ + HtlcKey: key, + HtlcEventType: eventType, + Timestamp: h.now(), + } + + log.Tracef("Notifying forwarding failure event: %v over %v", eventType, + key) + + if err := h.ntfnServer.SendUpdate(event); err != nil { + log.Warnf("Unable to send forwarding fail event: %v", err) + } +} + +// NotifySettleEvent notifies the HtlcNotifier that a htlc that we committed +// to as part of a forward or a receive to our node has been settled. +func (h *HtlcNotifier) NotifySettleEvent(key HtlcKey, eventType HtlcEventType) { + event := &SettleEvent{ + HtlcKey: key, + HtlcEventType: eventType, + Timestamp: h.now(), + } + + log.Tracef("Notifying settle event: %v over %v", eventType, key) + + if err := h.ntfnServer.SendUpdate(event); err != nil { + log.Warnf("Unable to send settle event: %v", err) + } +} From abf780bf03c8893c0d4bdcc01704306caa4a5c82 Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:34:47 +0200 Subject: [PATCH 02/10] multi: add htlcNotifier interface to switch and link In this commit, a htlcNotifier interface is added to allow for easy unit testing. Instances of the HtlcNotifier are added to the server, switch and link. --- htlcswitch/htlcnotifier.go | 8 ++++++++ htlcswitch/interfaces.go | 26 ++++++++++++++++++++++++++ htlcswitch/link.go | 4 ++++ htlcswitch/link_test.go | 3 +++ htlcswitch/mock.go | 20 ++++++++++++++++++++ htlcswitch/switch.go | 4 ++++ htlcswitch/test_utils.go | 1 + peer.go | 1 + server.go | 10 ++++++++++ 9 files changed, 77 insertions(+) diff --git a/htlcswitch/htlcnotifier.go b/htlcswitch/htlcnotifier.go index 1fb523b8..dd71f3da 100644 --- a/htlcswitch/htlcnotifier.go +++ b/htlcswitch/htlcnotifier.go @@ -289,6 +289,8 @@ type SettleEvent struct { // NotifyForwardingEvent notifies the HtlcNotifier than a htlc has been // forwarded. +// +// Note this is part of the htlcNotifier interface. func (h *HtlcNotifier) NotifyForwardingEvent(key HtlcKey, info HtlcInfo, eventType HtlcEventType) { @@ -309,6 +311,8 @@ func (h *HtlcNotifier) NotifyForwardingEvent(key HtlcKey, info HtlcInfo, // NotifyLinkFailEvent notifies that a htlc has failed on our incoming // or outgoing link. +// +// Note this is part of the htlcNotifier interface. func (h *HtlcNotifier) NotifyLinkFailEvent(key HtlcKey, info HtlcInfo, eventType HtlcEventType, linkErr *LinkError, incoming bool) { @@ -331,6 +335,8 @@ func (h *HtlcNotifier) NotifyLinkFailEvent(key HtlcKey, info HtlcInfo, // NotifyForwardingFailEvent notifies the HtlcNotifier that a htlc we // forwarded has failed down the line. +// +// Note this is part of the htlcNotifier interface. func (h *HtlcNotifier) NotifyForwardingFailEvent(key HtlcKey, eventType HtlcEventType) { @@ -350,6 +356,8 @@ func (h *HtlcNotifier) NotifyForwardingFailEvent(key HtlcKey, // NotifySettleEvent notifies the HtlcNotifier that a htlc that we committed // to as part of a forward or a receive to our node has been settled. +// +// Note this is part of the htlcNotifier interface. func (h *HtlcNotifier) NotifySettleEvent(key HtlcKey, eventType HtlcEventType) { event := &SettleEvent{ HtlcKey: key, diff --git a/htlcswitch/interfaces.go b/htlcswitch/interfaces.go index f0eae99d..b28d137f 100644 --- a/htlcswitch/interfaces.go +++ b/htlcswitch/interfaces.go @@ -180,3 +180,29 @@ type TowerClient interface { // isTweakless should be true. BackupState(*lnwire.ChannelID, *lnwallet.BreachRetribution, bool) error } + +// htlcNotifier is an interface which represents the input side of the +// HtlcNotifier which htlc events are piped through. This interface is intended +// to allow for mocking of the htlcNotifier in tests, so is unexported because +// it is not needed outside of the htlcSwitch package. +type htlcNotifier interface { + // NotifyForwardingEvent notifies the HtlcNotifier than a htlc has been + // forwarded. + NotifyForwardingEvent(key HtlcKey, info HtlcInfo, + eventType HtlcEventType) + + // NotifyIncomingLinkFailEvent notifies that a htlc has failed on our + // incoming link. It takes an isReceive bool to differentiate between + // our node's receives and forwards. + NotifyLinkFailEvent(key HtlcKey, info HtlcInfo, + eventType HtlcEventType, linkErr *LinkError, incoming bool) + + // NotifyForwardingFailEvent notifies the HtlcNotifier that a htlc we + // forwarded has failed down the line. + NotifyForwardingFailEvent(key HtlcKey, eventType HtlcEventType) + + // NotifySettleEvent notifies the HtlcNotifier that a htlc that we + // committed to as part of a forward or a receive to our node has been + // settled. + NotifySettleEvent(key HtlcKey, eventType HtlcEventType) +} diff --git a/htlcswitch/link.go b/htlcswitch/link.go index 80b90f48..d8da0f13 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -272,6 +272,10 @@ type ChannelLinkConfig struct { // NotifyInactiveChannel allows the switch to tell the ChannelNotifier // when channels become inactive. NotifyInactiveChannel func(wire.OutPoint) + + // HtlcNotifier is an instance of a htlcNotifier which we will pipe htlc + // events through. + HtlcNotifier htlcNotifier } // channelLink is the service which drives a channel's commitment update diff --git a/htlcswitch/link_test.go b/htlcswitch/link_test.go index 427322ee..3b48c157 100644 --- a/htlcswitch/link_test.go +++ b/htlcswitch/link_test.go @@ -1750,6 +1750,7 @@ func newSingleLinkTestHarness(chanAmt, chanReserve btcutil.Amount) ( MaxFeeAllocation: DefaultMaxLinkFeeAllocation, NotifyActiveChannel: func(wire.OutPoint) {}, NotifyInactiveChannel: func(wire.OutPoint) {}, + HtlcNotifier: aliceSwitch.cfg.HtlcNotifier, } aliceLink := NewChannelLink(aliceCfg, aliceLc.channel) @@ -4313,6 +4314,7 @@ func (h *persistentLinkHarness) restartLink( MaxFeeAllocation: DefaultMaxLinkFeeAllocation, NotifyActiveChannel: func(wire.OutPoint) {}, NotifyInactiveChannel: func(wire.OutPoint) {}, + HtlcNotifier: aliceSwitch.cfg.HtlcNotifier, } aliceLink := NewChannelLink(aliceCfg, aliceChannel) @@ -5523,6 +5525,7 @@ func TestCheckHtlcForward(t *testing.T) { }, FetchLastChannelUpdate: fetchLastChannelUpdate, MaxOutgoingCltvExpiry: DefaultMaxOutgoingCltvExpiry, + HtlcNotifier: &mockHTLCNotifier{}, }, log: log, channel: testChannel.channel, diff --git a/htlcswitch/mock.go b/htlcswitch/mock.go index 7cc9fb04..0c7f1ed8 100644 --- a/htlcswitch/mock.go +++ b/htlcswitch/mock.go @@ -176,6 +176,7 @@ func initSwitchWithDB(startingHeight uint32, db *channeldb.DB) (*Switch, error) FwdEventTicker: ticker.NewForce(DefaultFwdEventInterval), LogEventTicker: ticker.NewForce(DefaultLogInterval), AckEventTicker: ticker.NewForce(DefaultAckInterval), + HtlcNotifier: &mockHTLCNotifier{}, } return New(cfg, startingHeight) @@ -1009,3 +1010,22 @@ func (m *mockOnionErrorDecryptor) DecryptError(encryptedData []byte) ( Message: m.message, }, m.err } + +var _ htlcNotifier = (*mockHTLCNotifier)(nil) + +type mockHTLCNotifier struct{} + +func (h *mockHTLCNotifier) NotifyForwardingEvent(key HtlcKey, info HtlcInfo, + eventType HtlcEventType) { +} + +func (h *mockHTLCNotifier) NotifyLinkFailEvent(key HtlcKey, info HtlcInfo, + eventType HtlcEventType, linkErr *LinkError, incoming bool) { +} + +func (h *mockHTLCNotifier) NotifyForwardingFailEvent(key HtlcKey, + eventType HtlcEventType) { +} + +func (h *mockHTLCNotifier) NotifySettleEvent(key HtlcKey, eventType HtlcEventType) { +} diff --git a/htlcswitch/switch.go b/htlcswitch/switch.go index b70e3662..f3ca0208 100644 --- a/htlcswitch/switch.go +++ b/htlcswitch/switch.go @@ -150,6 +150,10 @@ type Config struct { // the switch when a new block has arrived. Notifier chainntnfs.ChainNotifier + // HtlcNotifier is an instance of a htlcNotifier which we will pipe htlc + // events through. + HtlcNotifier htlcNotifier + // FwdEventTicker is a signal that instructs the htlcswitch to flush any // pending forwarding events. FwdEventTicker ticker.Ticker diff --git a/htlcswitch/test_utils.go b/htlcswitch/test_utils.go index da6b07ed..51020150 100644 --- a/htlcswitch/test_utils.go +++ b/htlcswitch/test_utils.go @@ -1139,6 +1139,7 @@ func (h *hopNetwork) createChannelLink(server, peer *mockServer, MaxFeeAllocation: DefaultMaxLinkFeeAllocation, NotifyActiveChannel: func(wire.OutPoint) {}, NotifyInactiveChannel: func(wire.OutPoint) {}, + HtlcNotifier: server.htlcSwitch.cfg.HtlcNotifier, }, channel, ) diff --git a/peer.go b/peer.go index a9568904..ea482d1f 100644 --- a/peer.go +++ b/peer.go @@ -636,6 +636,7 @@ func (p *peer) addLink(chanPoint *wire.OutPoint, MaxFeeAllocation: cfg.MaxChannelFeeAllocation, NotifyActiveChannel: p.server.channelNotifier.NotifyActiveChannelEvent, NotifyInactiveChannel: p.server.channelNotifier.NotifyInactiveChannelEvent, + HtlcNotifier: p.server.htlcNotifier, } link := htlcswitch.NewChannelLink(linkCfg, lnChan) diff --git a/server.go b/server.go index 7d2d7470..bde65671 100644 --- a/server.go +++ b/server.go @@ -203,6 +203,8 @@ type server struct { peerNotifier *peernotifier.PeerNotifier + htlcNotifier *htlcswitch.HtlcNotifier + witnessBeacon contractcourt.WitnessBeacon breachArbiter *breachArbiter @@ -438,6 +440,8 @@ func newServer(listenAddrs []net.Addr, chanDB *channeldb.DB, return nil, err } + s.htlcNotifier = htlcswitch.NewHtlcNotifier(time.Now) + s.htlcSwitch, err = htlcswitch.New(htlcswitch.Config{ DB: chanDB, LocalChannelClose: func(pubKey []byte, @@ -467,6 +471,7 @@ func newServer(listenAddrs []net.Addr, chanDB *channeldb.DB, ExtractErrorEncrypter: s.sphinx.ExtractErrorEncrypter, FetchLastChannelUpdate: s.fetchLastChanUpdate(), Notifier: s.cc.chainNotifier, + HtlcNotifier: s.htlcNotifier, FwdEventTicker: ticker.New(htlcswitch.DefaultFwdEventInterval), LogEventTicker: ticker.New(htlcswitch.DefaultLogInterval), AckEventTicker: ticker.New(htlcswitch.DefaultAckInterval), @@ -1265,6 +1270,10 @@ func (s *server) Start() error { startErr = err return } + if err := s.htlcNotifier.Start(); err != nil { + startErr = err + return + } if err := s.sphinx.Start(); err != nil { startErr = err return @@ -1429,6 +1438,7 @@ func (s *server) Stop() error { s.sweeper.Stop() s.channelNotifier.Stop() s.peerNotifier.Stop() + s.htlcNotifier.Stop() s.cc.wallet.Shutdown() s.cc.chainView.Stop() s.connMgr.Stop() From b70080a267c49b340d4b4cb64704f62402ccba9d Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:51:53 +0200 Subject: [PATCH 03/10] htlcswitch: failAddPacket with full packet information This commit sets more fields on the htlcPacket created to fail adding a htlc packet to the switch for notification purposes. This new data is copied by value from the original packet. The packet is then failed back to the peer that forwarded us the packet, which is handled by handledownstream packet. The values added to the packet are not used in the handling of a failed packet. --- htlcswitch/switch.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/htlcswitch/switch.go b/htlcswitch/switch.go index f3ca0208..52c47f3b 100644 --- a/htlcswitch/switch.go +++ b/htlcswitch/switch.go @@ -1256,12 +1256,21 @@ func (s *Switch) failAddPacket(packet *htlcPacket, failure *LinkError) error { log.Error(failure.Error()) + // Create a failure packet for this htlc. The the full set of + // information about the htlc failure is included so that they can + // be included in link failure notifications. failPkt := &htlcPacket{ - sourceRef: packet.sourceRef, - incomingChanID: packet.incomingChanID, - incomingHTLCID: packet.incomingHTLCID, - circuit: packet.circuit, - linkFailure: failure, + sourceRef: packet.sourceRef, + incomingChanID: packet.incomingChanID, + incomingHTLCID: packet.incomingHTLCID, + outgoingChanID: packet.outgoingChanID, + outgoingHTLCID: packet.outgoingHTLCID, + incomingAmount: packet.incomingAmount, + amount: packet.amount, + incomingTimeout: packet.incomingTimeout, + outgoingTimeout: packet.outgoingTimeout, + circuit: packet.circuit, + linkFailure: failure, htlc: &lnwire.UpdateFailHTLC{ Reason: reason, }, From fc0ee06a9906e2908ee4c04fc8afdbafbe473a42 Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 18:03:22 +0200 Subject: [PATCH 04/10] htlcswitch: add notifications for forwards This commit adds notifications for htlcs which are forwarded through our node. Forwards are notified when the htlc is added on our ougoing link, settles when we send a settle message to the downstream peer. If a failure occurs, we check whether it occurred at our node, then notify a link or forwarding failure accordingly. Note that this change also adds forward event notifications for sends which are initiated by our node because the handling code for adding a htlc which originates from our node is the same as that for handling forwards. Htlcs for our locally initiated sends have our internal pid set in the incoming htlcs id field, so we extract this value and notify with a zero htlc id to be consistent with receives (which have zero outgoing circuits). Subsequent settles or failures are not noitfied for local sends in this commit, and will be handled in a follow up. --- htlcswitch/htlcnotifier.go | 56 ++++++++++++++++++++++++++++++++++++++ htlcswitch/link.go | 40 +++++++++++++++++++++++++-- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/htlcswitch/htlcnotifier.go b/htlcswitch/htlcnotifier.go index dd71f3da..25953e65 100644 --- a/htlcswitch/htlcnotifier.go +++ b/htlcswitch/htlcnotifier.go @@ -371,3 +371,59 @@ func (h *HtlcNotifier) NotifySettleEvent(key HtlcKey, eventType HtlcEventType) { log.Warnf("Unable to send settle event: %v", err) } } + +// newHtlc key returns a htlc key for the packet provided. If the packet +// has a zero incoming channel ID, the packet is for one of our own sends, +// which has the payment id stashed in the incoming htlc id. If this is the +// case, we replace the incoming htlc id with zero so that the notifier +// consistently reports zero circuit keys for events that terminate or +// originate at our node. +func newHtlcKey(pkt *htlcPacket) HtlcKey { + htlcKey := HtlcKey{ + IncomingCircuit: channeldb.CircuitKey{ + ChanID: pkt.incomingChanID, + HtlcID: pkt.incomingHTLCID, + }, + OutgoingCircuit: CircuitKey{ + ChanID: pkt.outgoingChanID, + HtlcID: pkt.outgoingHTLCID, + }, + } + + // If the packet has a zero incoming channel ID, it is a send that was + // initiated at our node. If this is the case, our internal pid is in + // the incoming htlc ID, so we overwrite it with 0 for notification + // purposes. + if pkt.incomingChanID == hop.Source { + htlcKey.IncomingCircuit.HtlcID = 0 + } + + return htlcKey +} + +// newHtlcInfo returns HtlcInfo for the packet provided. +func newHtlcInfo(pkt *htlcPacket) HtlcInfo { + return HtlcInfo{ + IncomingTimeLock: pkt.incomingTimeout, + OutgoingTimeLock: pkt.outgoingTimeout, + IncomingAmt: pkt.incomingAmount, + OutgoingAmt: pkt.amount, + } +} + +// getEventType returns the htlc type based on the fields set in the htlc +// packet. Sends that originate at our node have the source (zero) incoming +// channel ID. Receives to our node have the exit (zero) outgoing channel ID +// and forwards have both fields set. +func getEventType(pkt *htlcPacket) HtlcEventType { + switch { + case pkt.incomingChanID == hop.Source: + return HtlcEventTypeSend + + case pkt.outgoingChanID == hop.Exit: + return HtlcEventTypeReceive + + default: + return HtlcEventTypeForward + } +} diff --git a/htlcswitch/link.go b/htlcswitch/link.go index d8da0f13..c386ad34 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -1418,6 +1418,18 @@ func (l *channelLink) handleDownStreamPkt(pkt *htlcPacket, isReProcess bool) { l.cfg.Peer.SendMessage(false, htlc) + // Send a forward event notification to htlcNotifier. + l.cfg.HtlcNotifier.NotifyForwardingEvent( + newHtlcKey(pkt), + HtlcInfo{ + IncomingTimeLock: pkt.incomingTimeout, + IncomingAmt: pkt.incomingAmount, + OutgoingTimeLock: htlc.Expiry, + OutgoingAmt: htlc.Amount, + }, + getEventType(pkt), + ) + case *lnwire.UpdateFulfillHTLC: // If hodl.SettleOutgoing mode is active, we exit early to // simulate arbitrary delays between the switch adding the @@ -1476,6 +1488,12 @@ func (l *channelLink) handleDownStreamPkt(pkt *htlcPacket, isReProcess bool) { l.cfg.Peer.SendMessage(false, htlc) isSettle = true + // Send a settle event notification to htlcNotifier. + l.cfg.HtlcNotifier.NotifySettleEvent( + newHtlcKey(pkt), + getEventType(pkt), + ) + case *lnwire.UpdateFailHTLC: // If hodl.FailOutgoing mode is active, we exit early to // simulate arbitrary delays between the switch adding a FAIL to @@ -1529,10 +1547,28 @@ func (l *channelLink) handleDownStreamPkt(pkt *htlcPacket, isReProcess bool) { htlc.ChanID = l.ChanID() htlc.ID = pkt.incomingHTLCID - // Finally, we send the HTLC message to the peer which - // initially created the HTLC. + // We send the HTLC message to the peer which initially created + // the HTLC. l.cfg.Peer.SendMessage(false, htlc) isSettle = true + + // If the packet does not have a link failure set, it failed + // further down the route so we notify a forwarding failure. + // Otherwise, we notify a link failure because it failed at our + // node. + if pkt.linkFailure != nil { + l.cfg.HtlcNotifier.NotifyLinkFailEvent( + newHtlcKey(pkt), + newHtlcInfo(pkt), + getEventType(pkt), + pkt.linkFailure, + false, + ) + } else { + l.cfg.HtlcNotifier.NotifyForwardingFailEvent( + newHtlcKey(pkt), getEventType(pkt), + ) + } } // If this newly added update exceeds the min batch size for adds, or From 937062b6d313831f428e6ed4fb500d467e5fc674 Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:34:47 +0200 Subject: [PATCH 05/10] htlcswitch: refactor handleLocalDispatch Split handleLocalDispatch into an extra handleLocalAddHTLC function so we can easily notify an error should one occur adding the htlc. --- htlcswitch/switch.go | 74 +++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/htlcswitch/switch.go b/htlcswitch/switch.go index 52c47f3b..84e5aa15 100644 --- a/htlcswitch/switch.go +++ b/htlcswitch/switch.go @@ -768,38 +768,9 @@ func (s *Switch) handleLocalDispatch(pkt *htlcPacket) error { // User have created the htlc update therefore we should find the // appropriate channel link and send the payment over this link. if htlc, ok := pkt.htlc.(*lnwire.UpdateAddHTLC); ok { - // Try to find links by node destination. - s.indexMtx.RLock() - link, err := s.getLinkByShortID(pkt.outgoingChanID) - s.indexMtx.RUnlock() + link, err := s.handleLocalAddHTLC(pkt, htlc) if err != nil { - log.Errorf("Link %v not found", pkt.outgoingChanID) - return NewLinkError(&lnwire.FailUnknownNextPeer{}) - } - - if !link.EligibleToForward() { - log.Errorf("Link %v is not available to forward", - pkt.outgoingChanID) - - // The update does not need to be populated as the error - // will be returned back to the router. - return NewDetailedLinkError( - lnwire.NewTemporaryChannelFailure(nil), - OutgoingFailureLinkNotEligible, - ) - } - - // Ensure that the htlc satisfies the outgoing channel policy. - currentHeight := atomic.LoadUint32(&s.bestHeight) - htlcErr := link.CheckHtlcTransit( - htlc.PaymentHash, - htlc.Amount, - htlc.Expiry, currentHeight, - ) - if htlcErr != nil { - log.Errorf("Link %v policy for local forward not "+ - "satisfied", pkt.outgoingChanID) - return htlcErr + return err } return link.HandleSwitchPacket(pkt) @@ -811,6 +782,47 @@ func (s *Switch) handleLocalDispatch(pkt *htlcPacket) error { return nil } +// handleLocalAddHTLC handles the addition of a htlc for a send that +// originates from our node. It returns the link that the htlc should +// be forwarded outwards on, and a link error if the htlc cannot be +// forwarded. +func (s *Switch) handleLocalAddHTLC(pkt *htlcPacket, + htlc *lnwire.UpdateAddHTLC) (ChannelLink, *LinkError) { + + // Try to find links by node destination. + s.indexMtx.RLock() + link, err := s.getLinkByShortID(pkt.outgoingChanID) + s.indexMtx.RUnlock() + if err != nil { + log.Errorf("Link %v not found", pkt.outgoingChanID) + return nil, NewLinkError(&lnwire.FailUnknownNextPeer{}) + } + + if !link.EligibleToForward() { + log.Errorf("Link %v is not available to forward", + pkt.outgoingChanID) + + // The update does not need to be populated as the error + // will be returned back to the router. + return nil, NewDetailedLinkError( + lnwire.NewTemporaryChannelFailure(nil), + OutgoingFailureLinkNotEligible, + ) + } + + // Ensure that the htlc satisfies the outgoing channel policy. + currentHeight := atomic.LoadUint32(&s.bestHeight) + htlcErr := link.CheckHtlcTransit( + htlc.PaymentHash, htlc.Amount, htlc.Expiry, currentHeight, + ) + if htlcErr != nil { + log.Errorf("Link %v policy for local forward not "+ + "satisfied", pkt.outgoingChanID) + return nil, htlcErr + } + return link, nil +} + // handleLocalResponse processes a Settle or Fail responding to a // locally-initiated payment. This is handled asynchronously to avoid blocking // the main event loop within the switch, as these operations can require From 79a890fa487a891577fd4154073c464b88634036 Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:34:47 +0200 Subject: [PATCH 06/10] htlcswitch: notify outgoing link failures for local sends Notify link failures for our own payments. Separate handling code is required for local payment link failures because we do not pass these failures back through the switch (like we do for link failures for forwards), but rather send them straight back to the router. Our own sends have the payment ID saved in the incoming htlc ID of the packet's incoming circuit. This change replaces that value with for the sake of consistent notifying of sends and receives from our node. --- htlcswitch/switch.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/htlcswitch/switch.go b/htlcswitch/switch.go index 84e5aa15..a30eb584 100644 --- a/htlcswitch/switch.go +++ b/htlcswitch/switch.go @@ -770,6 +770,20 @@ func (s *Switch) handleLocalDispatch(pkt *htlcPacket) error { if htlc, ok := pkt.htlc.(*lnwire.UpdateAddHTLC); ok { link, err := s.handleLocalAddHTLC(pkt, htlc) if err != nil { + // Notify the htlc notifier of a link failure on our + // outgoing link. Incoming timelock/amount values are + // not set because they are not present for local sends. + s.cfg.HtlcNotifier.NotifyLinkFailEvent( + newHtlcKey(pkt), + HtlcInfo{ + OutgoingTimeLock: htlc.Expiry, + OutgoingAmt: htlc.Amount, + }, + HtlcEventTypeSend, + err, + false, + ) + return err } From 22d370aa5155b5dba43eeaf1a9818c40724147dd Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:34:47 +0200 Subject: [PATCH 07/10] htlcswitch: notify send forward failures and settles Add notifications for local initiated sends settles and forwarding failures. As with link failures, local send settles and forwarding failures are reported directly to the router so must have their own notification handling. --- htlcswitch/switch.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/htlcswitch/switch.go b/htlcswitch/switch.go index a30eb584..602dc55b 100644 --- a/htlcswitch/switch.go +++ b/htlcswitch/switch.go @@ -898,6 +898,18 @@ func (s *Switch) handleLocalResponse(pkt *htlcPacket) { pkt.inKey(), err) return } + + // Finally, notify on the htlc failure or success that has been handled. + key := newHtlcKey(pkt) + eventType := getEventType(pkt) + + switch pkt.htlc.(type) { + case *lnwire.UpdateFulfillHTLC: + s.cfg.HtlcNotifier.NotifySettleEvent(key, eventType) + + case *lnwire.UpdateFailHTLC: + s.cfg.HtlcNotifier.NotifyForwardingFailEvent(key, eventType) + } } // extractResult uses the given deobfuscator to extract the payment result from From 6bd0de257a70997065d51eee96c590610a7179ce Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:34:47 +0200 Subject: [PATCH 08/10] htlcswitch: notify incoming link failures This commit adds link failure notifications for failures which occur on our incoming link. These failures may be receives which we failed or forwards which we could not parse. --- htlcswitch/link.go | 50 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index c386ad34..5d5a3040 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -1224,8 +1224,7 @@ func (l *channelLink) processHtlcResolution(resolution invoices.HtlcResolution, failure := getResolutionFailure(res, htlc.pd.Amount) l.sendHTLCError( - htlc.pd.HtlcIndex, failure, - htlc.obfuscator, htlc.pd.SourceRef, + htlc.pd, failure, htlc.obfuscator, true, ) return nil @@ -2696,9 +2695,7 @@ func (l *channelLink) processRemoteAdds(fwdPkg *channeldb.FwdPkg, // later date failure := lnwire.NewInvalidOnionPayload(failedType, 0) l.sendHTLCError( - pd.HtlcIndex, - NewLinkError(failure), - obfuscator, pd.SourceRef, + pd, NewLinkError(failure), obfuscator, false, ) l.log.Errorf("unable to decode forwarding "+ @@ -2811,10 +2808,7 @@ func (l *channelLink) processRemoteAdds(fwdPkg *channeldb.FwdPkg, ) l.sendHTLCError( - pd.HtlcIndex, - NewLinkError(failure), - obfuscator, - pd.SourceRef, + pd, NewLinkError(failure), obfuscator, false, ) continue } @@ -2900,7 +2894,7 @@ func (l *channelLink) processExitHop(pd *lnwallet.PaymentDescriptor, failure := NewLinkError( lnwire.NewFinalIncorrectHtlcAmount(pd.Amount), ) - l.sendHTLCError(pd.HtlcIndex, failure, obfuscator, pd.SourceRef) + l.sendHTLCError(pd, failure, obfuscator, true) return nil } @@ -2915,7 +2909,7 @@ func (l *channelLink) processExitHop(pd *lnwallet.PaymentDescriptor, failure := NewLinkError( lnwire.NewFinalIncorrectCltvExpiry(pd.Timeout), ) - l.sendHTLCError(pd.HtlcIndex, failure, obfuscator, pd.SourceRef) + l.sendHTLCError(pd, failure, obfuscator, true) return nil } @@ -3031,8 +3025,8 @@ func (l *channelLink) handleBatchFwdErrs(errChan chan error) { // sendHTLCError functions cancels HTLC and send cancel message back to the // peer from which HTLC was received. -func (l *channelLink) sendHTLCError(htlcIndex uint64, failure *LinkError, - e hop.ErrorEncrypter, sourceRef *channeldb.AddRef) { +func (l *channelLink) sendHTLCError(pd *lnwallet.PaymentDescriptor, + failure *LinkError, e hop.ErrorEncrypter, isReceive bool) { reason, err := e.EncryptFirstHop(failure.WireMessage()) if err != nil { @@ -3040,7 +3034,7 @@ func (l *channelLink) sendHTLCError(htlcIndex uint64, failure *LinkError, return } - err = l.channel.FailHTLC(htlcIndex, reason, sourceRef, nil, nil) + err = l.channel.FailHTLC(pd.HtlcIndex, reason, pd.SourceRef, nil, nil) if err != nil { l.log.Errorf("unable cancel htlc: %v", err) return @@ -3048,9 +3042,35 @@ func (l *channelLink) sendHTLCError(htlcIndex uint64, failure *LinkError, l.cfg.Peer.SendMessage(false, &lnwire.UpdateFailHTLC{ ChanID: l.ChanID(), - ID: htlcIndex, + ID: pd.HtlcIndex, Reason: reason, }) + + // Notify a link failure on our incoming link. Outgoing htlc information + // is not available at this point, because we have not decrypted the + // onion, so it is excluded. + var eventType HtlcEventType + if isReceive { + eventType = HtlcEventTypeReceive + } else { + eventType = HtlcEventTypeForward + } + + l.cfg.HtlcNotifier.NotifyLinkFailEvent( + HtlcKey{ + IncomingCircuit: channeldb.CircuitKey{ + ChanID: l.ShortChanID(), + HtlcID: pd.HtlcIndex, + }, + }, + HtlcInfo{ + IncomingTimeLock: pd.Timeout, + IncomingAmt: pd.Amount, + }, + eventType, + failure, + true, + ) } // sendMalformedHTLCError helper function which sends the malformed HTLC update From 71fdd755b44f709a43a32ef0ddb0b1a0b7a33cc0 Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:34:48 +0200 Subject: [PATCH 09/10] htlcswitch: notify local receive settles --- htlcswitch/link.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/htlcswitch/link.go b/htlcswitch/link.go index 5d5a3040..797c3c72 100644 --- a/htlcswitch/link.go +++ b/htlcswitch/link.go @@ -1208,10 +1208,7 @@ func (l *channelLink) processHtlcResolution(resolution invoices.HtlcResolution, l.log.Debugf("received settle resolution for %v"+ "with outcome: %v", circuitKey, res.Outcome) - return l.settleHTLC( - res.Preimage, htlc.pd.HtlcIndex, - htlc.pd.SourceRef, - ) + return l.settleHTLC(res.Preimage, htlc.pd) // For htlc failures, we get the relevant failure message based // on the failure resolution and then fail the htlc. @@ -2950,15 +2947,15 @@ func (l *channelLink) processExitHop(pd *lnwallet.PaymentDescriptor, } // settleHTLC settles the HTLC on the channel. -func (l *channelLink) settleHTLC(preimage lntypes.Preimage, htlcIndex uint64, - sourceRef *channeldb.AddRef) error { +func (l *channelLink) settleHTLC(preimage lntypes.Preimage, + pd *lnwallet.PaymentDescriptor) error { hash := preimage.Hash() l.log.Infof("settling htlc %v as exit hop", hash) err := l.channel.SettleHTLC( - preimage, htlcIndex, sourceRef, nil, nil, + preimage, pd.HtlcIndex, pd.SourceRef, nil, nil, ) if err != nil { return fmt.Errorf("unable to settle htlc: %v", err) @@ -2976,10 +2973,21 @@ func (l *channelLink) settleHTLC(preimage lntypes.Preimage, htlcIndex uint64, // remote peer. l.cfg.Peer.SendMessage(false, &lnwire.UpdateFulfillHTLC{ ChanID: l.ChanID(), - ID: htlcIndex, + ID: pd.HtlcIndex, PaymentPreimage: preimage, }) + // Once we have successfully settled the htlc, notify a settle event. + l.cfg.HtlcNotifier.NotifySettleEvent( + HtlcKey{ + IncomingCircuit: channeldb.CircuitKey{ + ChanID: l.ShortChanID(), + HtlcID: pd.HtlcIndex, + }, + }, + HtlcEventTypeReceive, + ) + return nil } From c0a4923dc1f24b7b83f075bb420c25e226de2aa1 Mon Sep 17 00:00:00 2001 From: carla Date: Wed, 19 Feb 2020 17:34:48 +0200 Subject: [PATCH 10/10] htlcswitch: add htlc notifier test --- htlcswitch/switch_test.go | 361 ++++++++++++++++++++++++++++++++++++++ htlcswitch/test_utils.go | 38 +++- 2 files changed, 398 insertions(+), 1 deletion(-) diff --git a/htlcswitch/switch_test.go b/htlcswitch/switch_test.go index 14032ed2..9cfed117 100644 --- a/htlcswitch/switch_test.go +++ b/htlcswitch/switch_test.go @@ -14,11 +14,14 @@ import ( "github.com/btcsuite/fastsha256" "github.com/davecgh/go-spew/spew" "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/htlcswitch/hop" "github.com/lightningnetwork/lnd/lntypes" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/ticker" ) +var zeroCircuit = channeldb.CircuitKey{} + func genPreimage() ([32]byte, error) { var preimage [32]byte if _, err := io.ReadFull(rand.Reader, preimage[:]); err != nil { @@ -2697,3 +2700,361 @@ func TestInvalidFailure(t *testing.T) { t.Fatal("err wasn't received") } } + +// htlcNotifierEvents is a function that generates a set of expected htlc +// notifier evetns for each node in a three hop network with the dynamic +// values provided. These functions take dynamic values so that changes to +// external systems (such as our default timelock delta) do not break +// these tests. +type htlcNotifierEvents func(channels *clusterChannels, htlcID uint64, + ts time.Time, htlc *lnwire.UpdateAddHTLC, + hops []*hop.Payload) ([]interface{}, []interface{}, []interface{}) + +// TestHtlcNotifier tests the notifying of htlc events that are routed over a +// three hop network. It sets up an Alice -> Bob -> Carol network and routes +// payments from Alice -> Carol to test events from the perspective of a +// sending (Alice), forwarding (Bob) and receiving (Carol) node. Test cases +// are present for saduccessful and failed payments. +func TestHtlcNotifier(t *testing.T) { + tests := []struct { + name string + + // Options is a set of options to apply to the three hop + // network's servers. + options []serverOption + + // expectedEvents is a function which returns an expected set + // of events for the test. + expectedEvents htlcNotifierEvents + + // iterations is the number of times we will send a payment, + // this is used to send more than one payment to force non- + // zero htlc indexes to make sure we aren't just checking + // default values. + iterations int + }{ + { + name: "successful three hop payment", + options: nil, + expectedEvents: func(channels *clusterChannels, + htlcID uint64, ts time.Time, + htlc *lnwire.UpdateAddHTLC, + hops []*hop.Payload) ([]interface{}, + []interface{}, []interface{}) { + + return getThreeHopEvents( + channels, htlcID, ts, htlc, hops, nil, + ) + }, + iterations: 2, + }, + { + name: "failed at forwarding link", + // Set a functional option which disables bob as a + // forwarding node to force a payment error. + options: []serverOption{ + serverOptionRejectHtlc(false, true, false), + }, + expectedEvents: func(channels *clusterChannels, + htlcID uint64, ts time.Time, + htlc *lnwire.UpdateAddHTLC, + hops []*hop.Payload) ([]interface{}, + []interface{}, []interface{}) { + + return getThreeHopEvents( + channels, htlcID, ts, htlc, hops, + &LinkError{ + msg: &lnwire.FailChannelDisabled{}, + FailureDetail: OutgoingFailureForwardsDisabled, + }, + ) + }, + iterations: 1, + }, + } + + for _, test := range tests { + test := test + + t.Run(test.name, func(t *testing.T) { + testHtcNotifier( + t, test.options, test.iterations, + test.expectedEvents, + ) + }) + } +} + +// testHtcNotifier runs a htlc notifier test. +func testHtcNotifier(t *testing.T, testOpts []serverOption, iterations int, + getEvents htlcNotifierEvents) { + + t.Parallel() + + // First, we'll create our traditional three hop + // network. + channels, cleanUp, _, err := createClusterChannels( + btcutil.SatoshiPerBitcoin*3, + btcutil.SatoshiPerBitcoin*5) + if err != nil { + t.Fatalf("unable to create channel: %v", err) + } + defer cleanUp() + + // Mock time so that all events are reported with a static timestamp. + now := time.Now() + mockTime := func() time.Time { + return now + } + + // Create htlc notifiers for each server in the three hop network and + // start them. + aliceNotifier := NewHtlcNotifier(mockTime) + if err := aliceNotifier.Start(); err != nil { + t.Fatalf("could not start alice notifier") + } + defer aliceNotifier.Stop() + + bobNotifier := NewHtlcNotifier(mockTime) + if err := bobNotifier.Start(); err != nil { + t.Fatalf("could not start bob notifier") + } + defer bobNotifier.Stop() + + carolNotifier := NewHtlcNotifier(mockTime) + if err := carolNotifier.Start(); err != nil { + t.Fatalf("could not start carol notifier") + } + defer carolNotifier.Stop() + + // Create a notifier server option which will set our htlc notifiers + // for the three hop network. + notifierOption := serverOptionWithHtlcNotifier( + aliceNotifier, bobNotifier, carolNotifier, + ) + + // Add the htlcNotifier option to any other options + // set in the test. + options := append(testOpts, notifierOption) + + n := newThreeHopNetwork( + t, channels.aliceToBob, + channels.bobToAlice, channels.bobToCarol, + channels.carolToBob, testStartingHeight, + options..., + ) + if err := n.start(); err != nil { + t.Fatalf("unable to start three hop "+ + "network: %v", err) + } + defer n.stop() + + // Before we forward anything, subscribe to htlc events + // from each notifier. + aliceEvents, err := aliceNotifier.SubscribeHtlcEvents() + if err != nil { + t.Fatalf("could not subscribe to alice's"+ + " events: %v", err) + } + defer aliceEvents.Cancel() + + bobEvents, err := bobNotifier.SubscribeHtlcEvents() + if err != nil { + t.Fatalf("could not subscribe to bob's"+ + " events: %v", err) + } + defer bobEvents.Cancel() + + carolEvents, err := carolNotifier.SubscribeHtlcEvents() + if err != nil { + t.Fatalf("could not subscribe to carol's"+ + " events: %v", err) + } + defer carolEvents.Cancel() + + // Send multiple payments, as specified by the test to test incrementing + // of htlc ids. + for i := 0; i < iterations; i++ { + // We'll start off by making a payment from + // Alice -> Bob -> Carol. + htlc, hops := n.sendThreeHopPayment(t) + + alice, bob, carol := getEvents( + channels, uint64(i), now, htlc, hops, + ) + + checkHtlcEvents(t, aliceEvents.Updates(), alice) + checkHtlcEvents(t, bobEvents.Updates(), bob) + checkHtlcEvents(t, carolEvents.Updates(), carol) + + } +} + +// checkHtlcEvents checks that a subscription has the set of htlc events +// we expect it to have. +func checkHtlcEvents(t *testing.T, events <-chan interface{}, + expectedEvents []interface{}) { + + for _, expected := range expectedEvents { + select { + case event := <-events: + if !reflect.DeepEqual(event, expected) { + t.Fatalf("expected %v, got: %v", expected, + event) + } + + case <-time.After(time.Second): + t.Fatalf("expected event: %v", expected) + } + } +} + +// sendThreeHopPayment is a helper function which sends a payment over +// Alice -> Bob -> Carol in a three hop network and returns Alice's first htlc +// and the remainder of the hops. +func (n *threeHopNetwork) sendThreeHopPayment(t *testing.T) (*lnwire.UpdateAddHTLC, + []*hop.Payload) { + + amount := lnwire.NewMSatFromSatoshis(btcutil.SatoshiPerBitcoin) + + htlcAmt, totalTimelock, hops := generateHops(amount, testStartingHeight, + n.firstBobChannelLink, n.carolChannelLink) + blob, err := generateRoute(hops...) + if err != nil { + t.Fatal(err) + } + invoice, htlc, pid, err := generatePayment( + amount, htlcAmt, totalTimelock, blob, + ) + if err != nil { + t.Fatal(err) + } + + err = n.carolServer.registry.AddInvoice(*invoice, htlc.PaymentHash) + if err != nil { + t.Fatalf("unable to add invoice in carol registry: %v", err) + } + + if err := n.aliceServer.htlcSwitch.SendHTLC( + n.firstBobChannelLink.ShortChanID(), pid, htlc, + ); err != nil { + t.Fatalf("could not send htlc") + } + + return htlc, hops +} + +// getThreeHopEvents gets the set of htlc events that we expect for a payment +// from Alice -> Bob -> Carol. If a non-nil link error is provided, the set +// of events will fail on Bob's outgoing link. +func getThreeHopEvents(channels *clusterChannels, htlcID uint64, + ts time.Time, htlc *lnwire.UpdateAddHTLC, hops []*hop.Payload, + linkError *LinkError) ([]interface{}, []interface{}, []interface{}) { + + aliceKey := HtlcKey{ + IncomingCircuit: zeroCircuit, + OutgoingCircuit: channeldb.CircuitKey{ + ChanID: channels.aliceToBob.ShortChanID(), + HtlcID: htlcID, + }, + } + + // Alice always needs a forwarding event because she initiates the + // send. + aliceEvents := []interface{}{ + &ForwardingEvent{ + HtlcKey: aliceKey, + HtlcInfo: HtlcInfo{ + OutgoingTimeLock: htlc.Expiry, + OutgoingAmt: htlc.Amount, + }, + HtlcEventType: HtlcEventTypeSend, + Timestamp: ts, + }, + } + + bobKey := HtlcKey{ + IncomingCircuit: channeldb.CircuitKey{ + ChanID: channels.bobToAlice.ShortChanID(), + HtlcID: htlcID, + }, + OutgoingCircuit: channeldb.CircuitKey{ + ChanID: channels.bobToCarol.ShortChanID(), + HtlcID: htlcID, + }, + } + + bobInfo := HtlcInfo{ + IncomingTimeLock: htlc.Expiry, + IncomingAmt: htlc.Amount, + OutgoingTimeLock: hops[1].FwdInfo.OutgoingCTLV, + OutgoingAmt: hops[1].FwdInfo.AmountToForward, + } + + // If we expect the payment to fail, we add failures for alice and + // bob, and no events for carol because the payment never reaches her. + if linkError != nil { + aliceEvents = append(aliceEvents, + &ForwardingFailEvent{ + HtlcKey: aliceKey, + HtlcEventType: HtlcEventTypeSend, + Timestamp: ts, + }, + ) + + bobEvents := []interface{}{ + &LinkFailEvent{ + HtlcKey: bobKey, + HtlcInfo: bobInfo, + HtlcEventType: HtlcEventTypeForward, + LinkError: linkError, + Incoming: false, + Timestamp: ts, + }, + } + + return aliceEvents, bobEvents, nil + } + + // If we want to get events for a successful payment, we add a settle + // for alice, a forward and settle for bob and a receive settle for + // carol. + aliceEvents = append( + aliceEvents, + &SettleEvent{ + HtlcKey: aliceKey, + HtlcEventType: HtlcEventTypeSend, + Timestamp: ts, + }, + ) + + bobEvents := []interface{}{ + &ForwardingEvent{ + HtlcKey: bobKey, + HtlcInfo: bobInfo, + HtlcEventType: HtlcEventTypeForward, + Timestamp: ts, + }, + &SettleEvent{ + HtlcKey: bobKey, + HtlcEventType: HtlcEventTypeForward, + Timestamp: ts, + }, + } + + carolEvents := []interface{}{ + &SettleEvent{ + HtlcKey: HtlcKey{ + IncomingCircuit: channeldb.CircuitKey{ + ChanID: channels.carolToBob.ShortChanID(), + HtlcID: htlcID, + }, + OutgoingCircuit: zeroCircuit, + }, + HtlcEventType: HtlcEventTypeReceive, + Timestamp: ts, + }, + } + + return aliceEvents, bobEvents, carolEvents +} diff --git a/htlcswitch/test_utils.go b/htlcswitch/test_utils.go index 51020150..26b8dad5 100644 --- a/htlcswitch/test_utils.go +++ b/htlcswitch/test_utils.go @@ -966,9 +966,11 @@ func createClusterChannels(aliceToBob, bobToCarol btcutil.Amount) ( // alice first bob second bob carol // channel link channel link channel link channel link // +// This function takes server options which can be used to apply custom +// settings to alice, bob and carol. func newThreeHopNetwork(t testing.TB, aliceChannel, firstBobChannel, secondBobChannel, carolChannel *lnwallet.LightningChannel, - startingHeight uint32) *threeHopNetwork { + startingHeight uint32, opts ...serverOption) *threeHopNetwork { aliceDb := aliceChannel.State().Db bobDb := firstBobChannel.State().Db @@ -996,6 +998,12 @@ func newThreeHopNetwork(t testing.TB, aliceChannel, firstBobChannel, t.Fatalf("unable to create carol server: %v", err) } + // Apply all additional functional options to the servers before + // creating any links. + for _, option := range opts { + option(aliceServer, bobServer, carolServer) + } + // Create mock decoder instead of sphinx one in order to mock the route // which htlc should follow. aliceDecoder := newMockIteratorDecoder() @@ -1045,6 +1053,34 @@ func newThreeHopNetwork(t testing.TB, aliceChannel, firstBobChannel, } } +// serverOption is a function which alters the three servers created for +// a three hop network to allow custom settings on each server. +type serverOption func(aliceServer, bobServer, carolServer *mockServer) + +// serverOptionWithHtlcNotifier is a functional option for the creation of +// three hop network servers which allows setting of htlc notifiers. +// Note that these notifiers should be started and stopped by the calling +// function. +func serverOptionWithHtlcNotifier(alice, bob, + carol *HtlcNotifier) serverOption { + + return func(aliceServer, bobServer, carolServer *mockServer) { + aliceServer.htlcSwitch.cfg.HtlcNotifier = alice + bobServer.htlcSwitch.cfg.HtlcNotifier = bob + carolServer.htlcSwitch.cfg.HtlcNotifier = carol + } +} + +// serverOptionRejectHtlc is the functional option for setting the reject +// htlc config option in each server's switch. +func serverOptionRejectHtlc(alice, bob, carol bool) serverOption { + return func(aliceServer, bobServer, carolServer *mockServer) { + aliceServer.htlcSwitch.cfg.RejectHTLC = alice + bobServer.htlcSwitch.cfg.RejectHTLC = bob + carolServer.htlcSwitch.cfg.RejectHTLC = carol + } +} + // createTwoClusterChannels creates lightning channels which are needed for // a 2 hop network cluster to be initialized. func createTwoClusterChannels(aliceToBob, bobToCarol btcutil.Amount) (