lightningd: add in_htlc_id / out_htlc_id to listforwards.

And document that we never know payment_hash.

Changelog-Added: JSON-RPC: `listforwards` now shows `in_htlc_id` and `out_htlc_id`
Changelog-Changed: JSON-RPC: `listforwards` now never shows `payment_hash`; use `listhtlcs`.

.msggen.json

@@ -574,8 +574,10 @@
         "ListforwardsForwards": {
             "ListForwards.forwards[].fee_msat": 7,
             "ListForwards.forwards[].in_channel": 1,
+            "ListForwards.forwards[].in_htlc_id": 10,
             "ListForwards.forwards[].in_msat": 2,
             "ListForwards.forwards[].out_channel": 5,
+            "ListForwards.forwards[].out_htlc_id": 11,
             "ListForwards.forwards[].out_msat": 8,
             "ListForwards.forwards[].payment_hash": 6,
             "ListForwards.forwards[].received_time": 4,

cln-grpc/proto/node.proto

@@ -1229,11 +1229,12 @@ message ListforwardsForwards {
 		TLV = 1;
 	}
 	string in_channel = 1;
+	uint64 in_htlc_id = 10;
 	Amount in_msat = 2;
 	ListforwardsForwardsStatus status = 3;
 	double received_time = 4;
 	optional string out_channel = 5;
-	optional bytes payment_hash = 6;
+	optional uint64 out_htlc_id = 11;
 	optional ListforwardsForwardsStyle style = 9;
 	optional Amount fee_msat = 7;
 	optional Amount out_msat = 8;

cln-grpc/src/convert.rs

@@ -897,11 +897,12 @@ impl From<&responses::ListforwardsForwards> for pb::ListforwardsForwards {
     fn from(c: &responses::ListforwardsForwards) -> Self {
         Self {
             in_channel: c.in_channel.to_string(), // Rule #2 for type short_channel_id
+            in_htlc_id: c.in_htlc_id.clone(), // Rule #2 for type u64
             in_msat: Some(c.in_msat.into()), // Rule #2 for type msat
             status: c.status as i32,
             received_time: c.received_time.clone(), // Rule #2 for type number
             out_channel: c.out_channel.as_ref().map(|v| v.to_string()), // Rule #2 for type short_channel_id?
-            payment_hash: c.payment_hash.as_ref().map(|v| hex::decode(&v).unwrap()), // Rule #2 for type hex?
+            out_htlc_id: c.out_htlc_id.clone(), // Rule #2 for type u64?
             style: c.style.map(|v| v as i32),
             fee_msat: c.fee_msat.map(|f| f.into()), // Rule #2 for type msat?
             out_msat: c.out_msat.map(|f| f.into()), // Rule #2 for type msat?

cln-rpc/src/model.rs

@@ -2673,6 +2673,8 @@ pub mod responses {
 	pub struct ListforwardsForwards {
 	    #[serde(alias = "in_channel")]
 	    pub in_channel: ShortChannelId,
+	    #[serde(alias = "in_htlc_id")]
+	    pub in_htlc_id: u64,
 	    #[serde(alias = "in_msat")]
 	    pub in_msat: Amount,
 	    // Path `ListForwards.forwards[].status`
@@ -2682,8 +2684,8 @@ pub mod responses {
 	    pub received_time: f64,
 	    #[serde(alias = "out_channel", skip_serializing_if = "Option::is_none")]
 	    pub out_channel: Option<ShortChannelId>,
-	    #[serde(alias = "payment_hash", skip_serializing_if = "Option::is_none")]
+	    #[serde(alias = "out_htlc_id", skip_serializing_if = "Option::is_none")]
-	    pub payment_hash: Option<String>,
+	    pub out_htlc_id: Option<u64>,
 	    #[serde(skip_serializing_if = "Option::is_none")]
 	    pub style: Option<ListforwardsForwardsStyle>,
 	    #[serde(alias = "fee_msat", skip_serializing_if = "Option::is_none")]

contrib/pyln-testing/pyln/testing/grpc2py.py

@@ -802,11 +802,12 @@ def getroute2py(m):
 def listforwards_forwards2py(m):
     return remove_default({
         "in_channel": m.in_channel,  # PrimitiveField in generate_composite
+        "in_htlc_id": m.in_htlc_id,  # PrimitiveField in generate_composite
         "in_msat": amount2msat(m.in_msat),  # PrimitiveField in generate_composite
         "status": str(m.status),  # EnumField in generate_composite
         "received_time": m.received_time,  # PrimitiveField in generate_composite
         "out_channel": m.out_channel,  # PrimitiveField in generate_composite
-        "payment_hash": hexlify(m.payment_hash),  # PrimitiveField in generate_composite
+        "out_htlc_id": m.out_htlc_id,  # PrimitiveField in generate_composite
         "style": str(m.style),  # EnumField in generate_composite
         "fee_msat": amount2msat(m.fee_msat),  # PrimitiveField in generate_composite
         "out_msat": amount2msat(m.out_msat),  # PrimitiveField in generate_composite

doc/lightning-listforwards.7.md

@@ -25,11 +25,12 @@ RETURN VALUE
 On success, an object containing **forwards** is returned.  It is an array of objects, where each object contains:
 
 - **in\_channel** (short\_channel\_id): the channel that received the HTLC
+- **in\_htlc\_id** (u64): the unique HTLC id the sender gave this
 - **in\_msat** (msat): the value of the incoming HTLC
 - **status** (string): still ongoing, completed, failed locally, or failed after forwarding (one of "offered", "settled", "local_failed", "failed")
 - **received\_time** (number): the UNIX timestamp when this was received
 - **out\_channel** (short\_channel\_id, optional): the channel that the HTLC (trying to) forward to
-- **payment\_hash** (hex, optional): payment hash sought by HTLC (always 64 characters)
+- **out\_htlc\_id** (u64, optional): the unique HTLC id we gave this when sending
 - **style** (string, optional): Either a legacy onion format or a modern tlv format (one of "legacy", "tlv")
 
 If **out\_msat** is present:
@@ -63,4 +64,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:39c71b957590f6a9b321120e7f337216833efd94f0144560da5cd55c91fee35c)
+[comment]: # ( SHA256STAMP:5b2da52b7f3a28563d0103d3853b9d8f717dc41a9e9c6b395ff19f1b975ca5fd)

doc/schemas/listforwards.schema.json

@@ -14,6 +14,7 @@
         "required": [
           "in_channel",
           "in_msat",
+          "in_htlc_id",
           "status",
           "received_time"
         ],
@@ -22,6 +23,10 @@
             "type": "short_channel_id",
             "description": "the channel that received the HTLC"
           },
+          "in_htlc_id": {
+            "type": "u64",
+            "description": "the unique HTLC id the sender gave this"
+          },
           "in_msatoshi": {
             "deprecated": true
           },
@@ -47,11 +52,9 @@
             "type": "short_channel_id",
             "description": "the channel that the HTLC (trying to) forward to"
           },
-          "payment_hash": {
+          "out_htlc_id": {
-            "type": "hex",
+            "type": "u64",
-            "description": "payment hash sought by HTLC",
+            "description": "the unique HTLC id we gave this when sending"
-            "maxLength": 64,
-            "minLength": 64
           },
           "style": {
             "type": "string",
@@ -74,10 +77,12 @@
               "required": [
                 "fee_msat",
                 "out_msat",
+                "out_htlc_id",
                 "out_channel"
               ],
               "properties": {
                 "in_channel": {},
+                "in_htlc_id": {},
                 "in_msatoshi": {},
                 "in_msat": {},
                 "status": {},
@@ -85,7 +90,7 @@
                 "received_time": {},
                 "resolved_time": {},
                 "out_channel": {},
-                "payment_hash": {},
+                "out_htlc_id": {},
                 "failcode": {},
                 "failreason": {},
                 "fee": {
@@ -109,13 +114,13 @@
               "required": [],
               "properties": {
                 "in_channel": {},
+                "in_htlc_id": {},
                 "in_msatoshi": {},
                 "in_msat": {},
                 "status": {},
                 "style": {},
                 "received_time": {},
                 "resolved_time": {},
-                "payment_hash": {},
                 "failcode": {},
                 "failreason": {},
                 "out_channel": {}
@@ -141,13 +146,14 @@
               ],
               "properties": {
                 "in_channel": {},
+                "in_htlc_id": {},
                 "in_msatoshi": {},
                 "in_msat": {},
                 "status": {},
                 "style": {},
                 "received_time": {},
                 "out_channel": {},
-                "payment_hash": {},
+                "out_htlc_id": {},
                 "fee": {},
                 "fee_msat": {},
                 "out_msatoshi": {},
@@ -164,13 +170,14 @@
               "additionalProperties": false,
               "properties": {
                 "in_channel": {},
+                "in_htlc_id": {},
                 "in_msatoshi": {},
                 "in_msat": {},
                 "status": {},
                 "style": {},
                 "received_time": {},
                 "out_channel": {},
-                "payment_hash": {},
+                "out_htlc_id": {},
                 "fee": {},
                 "fee_msat": {},
                 "failcode": {},
@@ -197,13 +204,14 @@
               "required": [],
               "properties": {
                 "in_channel": {},
+                "in_htlc_id": {},
                 "in_msatoshi": {},
                 "in_msat": {},
                 "status": {},
                 "style": {},
                 "received_time": {},
                 "out_channel": {},
-                "payment_hash": {},
+                "out_htlc_id": {},
                 "fee": {},
                 "fee_msat": {},
                 "out_msatoshi": {},
@@ -224,13 +232,14 @@
               "required": [],
               "properties": {
                 "in_channel": {},
+                "in_htlc_id": {},
                 "in_msatoshi": {},
                 "in_msat": {},
                 "status": {},
                 "style": {},
                 "received_time": {},
                 "out_channel": {},
-                "payment_hash": {},
+                "out_htlc_id": {},
                 "fee": {},
                 "fee_msat": {},
                 "out_msatoshi": {},

lightningd/notification.c

@@ -328,14 +328,16 @@ static void forward_event_notification_serialize(struct json_stream *stream,
 		cur->msat_out = AMOUNT_MSAT(0);
 		cur->fee = AMOUNT_MSAT(0);
 	}
-	cur->payment_hash = tal_dup(cur, struct sha256, &in->payment_hash);
+	cur->htlc_id_out = NULL;
 	cur->status = state;
 	cur->failcode = failcode;
 	cur->received_time = in->received_time;
 	cur->resolved_time = tal_steal(cur, resolved_time);
 	cur->forward_style = forward_style;
+	cur->htlc_id_in = in->key.id;
 
-	json_format_forwarding_object(stream, "forward_event", cur);
+	json_add_forwarding_object(stream, "forward_event",
+				   cur, &in->payment_hash);
 }
 
 REGISTER_NOTIFICATION(forward_event,

lightningd/peer_htlcs.c

@@ -2765,21 +2765,26 @@ AUTODATA(json_command, &dev_ignore_htlcs);
 
 /* Warp this process to ensure the consistent json object structure
  * between 'listforwards' API and 'forward_event' notification. */
-void json_format_forwarding_object(struct json_stream *response,
+void json_add_forwarding_object(struct json_stream *response,
-				   const char *fieldname,
+				const char *fieldname,
-				   const struct forwarding *cur)
+				const struct forwarding *cur,
+				const struct sha256 *payment_hash)
 {
 	json_object_start(response, fieldname);
 
-	/* See 6d333f16cc0f3aac7097269bf0985b5fa06d59b4: we may have deleted HTLC. */
+	/* Only for forward_event */
-	if (cur->payment_hash)
+	if (payment_hash)
-		json_add_sha256(response, "payment_hash", cur->payment_hash);
+		json_add_sha256(response, "payment_hash", payment_hash);
 	json_add_short_channel_id(response, "in_channel", &cur->channel_in);
+	json_add_u64(response, "in_htlc_id", cur->htlc_id_in);
 
 	/* This can be unknown if we failed before channel lookup */
-	if (cur->channel_out.u64 != 0)
+	if (cur->channel_out.u64 != 0) {
 		json_add_short_channel_id(response, "out_channel",
 					  &cur->channel_out);
+		if (cur->htlc_id_out)
+			json_add_u64(response, "out_htlc_id", *cur->htlc_id_out);
+	}
 	json_add_amount_msat_compat(response,
 				    cur->msat_in,
 				    "in_msatoshi", "in_msat");
@@ -2835,7 +2840,7 @@ static void listforwardings_add_forwardings(struct json_stream *response,
 	json_array_start(response, "forwards");
 	for (size_t i=0; i<tal_count(forwardings); i++) {
 		const struct forwarding *cur = &forwardings[i];
-		json_format_forwarding_object(response, NULL, cur);
+		json_add_forwarding_object(response, NULL, cur, NULL);
 	}
 	json_array_end(response);
 

lightningd/peer_htlcs.h

@@ -77,8 +77,10 @@ void local_fail_in_htlc_needs_update(struct htlc_in *hin,
 /* This json process will be used as the serialize method for
  * forward_event_notification_gen and be used in
  * `listforwardings_add_forwardings()`. */
-void json_format_forwarding_object(struct json_stream *response, const char *fieldname,
+void json_add_forwarding_object(struct json_stream *response,
-				   const struct forwarding *cur);
+				const char *fieldname,
+				const struct forwarding *cur,
+				const struct sha256 *payment_hash);
 
 /* Helper to create (common) WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS */
 #define failmsg_incorrect_or_unknown(ctx, ld, hin) \

tests/test_misc.py

@@ -2415,6 +2415,21 @@ def test_listforwards(node_factory, bitcoind):
     all_forwards = l2.rpc.listforwards()['forwards']
     assert len(all_forwards) == 3
 
+    # Not guaranteed to be in chronological order!
+    all_forwards.sort(key=lambda f: f['in_htlc_id'])
+    assert all_forwards[0]['in_channel'] == c12
+    assert all_forwards[0]['out_channel'] == c23
+    assert all_forwards[0]['in_htlc_id'] == 0
+    assert all_forwards[0]['out_htlc_id'] == 0
+    assert all_forwards[1]['in_channel'] == c12
+    assert all_forwards[1]['out_channel'] == c24
+    assert all_forwards[1]['in_htlc_id'] == 1
+    assert all_forwards[1]['out_htlc_id'] == 0
+    assert all_forwards[2]['in_channel'] == c12
+    assert all_forwards[2]['out_channel'] == c23
+    assert all_forwards[2]['in_htlc_id'] == 2
+    assert 'out_htlc_id' not in all_forwards[2]
+
     # status=settled
     settled_forwards = l2.rpc.listforwards(status='settled')['forwards']
     assert len(settled_forwards) == 2

tests/test_plugin.py

@@ -1353,24 +1353,30 @@ def test_forward_event_notification(node_factory, bitcoind, executor):
     expect = stats[0].copy()
     # First event won't have conclusion.
     del expect['resolved_time']
+    del expect['out_htlc_id']
     expect['status'] = 'offered'
     assert plugin_stats[0] == expect
     expect = stats[0].copy()
+    del expect['out_htlc_id']
     assert plugin_stats[1] == expect
 
     expect = stats[1].copy()
     del expect['resolved_time']
+    del expect['out_htlc_id']
     expect['status'] = 'offered'
     assert plugin_stats[2] == expect
     expect = stats[1].copy()
+    del expect['out_htlc_id']
     assert plugin_stats[3] == expect
 
     expect = stats[2].copy()
     del expect['failcode']
     del expect['failreason']
+    del expect['out_htlc_id']
     expect['status'] = 'offered'
     assert plugin_stats[4] == expect
     expect = stats[2].copy()
+    del expect['out_htlc_id']
     assert plugin_stats[5] == expect
 
 

wallet/wallet.c

@@ -4552,6 +4552,8 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 		", out_msatoshi"
 		", in_channel_scid"
 		", out_channel_scid"
+		", in_htlc_id"
+		", out_htlc_id"
 		", received_time"
 		", resolved_time"
 		", failcode "
@@ -4618,9 +4620,8 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 			cur->fee =  AMOUNT_MSAT(0);
 		}
 
-		/* FIXME: This now requires complex join to determine! */
-		cur->payment_hash = NULL;
 		db_col_scid(stmt, "in_channel_scid", &cur->channel_in);
+		cur->htlc_id_in = db_col_u64(stmt, "in_htlc_id");
 
 		if (!db_col_is_null(stmt, "out_channel_scid")) {
 			db_col_scid(stmt, "out_channel_scid", &cur->channel_out);
@@ -4628,6 +4629,11 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 			assert(cur->status == FORWARD_LOCAL_FAILED);
 			cur->channel_out.u64 = 0;
 		}
+		if (!db_col_is_null(stmt, "out_htlc_id")) {
+			cur->htlc_id_out = tal(results, u64);
+			*cur->htlc_id_out = db_col_u64(stmt, "out_htlc_id");
+		} else
+			cur->htlc_id_out = NULL;
 
 		cur->received_time = db_col_timeabs(stmt, "received_time");
 

wallet/wallet.h

@@ -272,9 +272,11 @@ static inline enum htlc_state htlc_state_in_db(enum htlc_state s)
 }
 
 struct forwarding {
+	/* channel_out is all-zero if unknown. */
 	struct short_channel_id channel_in, channel_out;
+	/* htlc_id_out is NULL if unknown. */
+	u64 htlc_id_in, *htlc_id_out;
 	struct amount_msat msat_in, msat_out, fee;
-	struct sha256 *payment_hash;
 	enum forward_style forward_style;
 	enum forward_status status;
 	enum onion_wire failcode;