msggen: Map the extratlvs field of keysend

Changelog-Added: cln-rpc: `keysend` now exposes the `extratlvs` field

cln-grpc/proto/node.proto

@@ -968,6 +968,7 @@ message KeysendRequest {
 	optional uint32 maxdelay = 6;
 	optional Amount exemptfee = 7;
 	optional RoutehintList routehints = 8;
+	optional TlvStream extratlvs = 9;
 }
 
 message KeysendResponse {
@@ -986,9 +987,6 @@ message KeysendResponse {
 	KeysendStatus status = 9;
 }
 
-message KeysendExtratlvs {
-}
-
 message FundpsbtRequest {
 	AmountOrAll satoshi = 1;
 	Feerate feerate = 2;

cln-grpc/proto/primitives.proto

@@ -73,3 +73,12 @@ message Routehint {
 message RoutehintList {
 	repeated Routehint hints = 2;
 }
+
+
+message TlvEntry {
+	uint64 type = 1;
+	bytes value = 2;
+}
+message TlvStream {
+	repeated TlvEntry entries = 1;
+}

cln-grpc/src/convert.rs

@@ -1375,6 +1375,7 @@ impl From<pb::KeysendRequest> for requests::KeysendRequest {
             maxdelay: c.maxdelay, // Rule #1 for type u32?
             exemptfee: c.exemptfee.map(|a| a.into()), // Rule #1 for type msat?
             routehints: c.routehints.map(|rl| rl.into()), // Rule #1 for type RoutehintList?
+            extratlvs: c.extratlvs.map(|s| s.into()), // Rule #1 for type TlvStream?
         }
     }
 }

cln-grpc/src/pb.rs

@@ -114,6 +114,7 @@ impl From<RouteHop> for cln_rpc::primitives::Routehop {
         }
     }
 }
+
 impl From<Routehint> for cln_rpc::primitives::Routehint {
     fn from(c: Routehint) -> Self {
         Self {
@@ -121,6 +122,7 @@ impl From<Routehint> for cln_rpc::primitives::Routehint {
         }
     }
 }
+
 impl From<RoutehintList> for cln_rpc::primitives::RoutehintList {
     fn from(c: RoutehintList) -> Self {
         Self {
@@ -128,6 +130,24 @@ impl From<RoutehintList> for cln_rpc::primitives::RoutehintList {
         }
     }
 }
+
+impl From<TlvStream> for cln_rpc::primitives::TlvStream {
+    fn from(s: TlvStream) -> Self {
+        Self {
+            entries: s.entries.into_iter().map(|e| e.into()).collect(),
+        }
+    }
+}
+
+impl From<TlvEntry> for cln_rpc::primitives::TlvEntry {
+    fn from(e: TlvEntry) -> Self {
+        Self {
+            typ: e.r#type,
+            value: e.value,
+        }
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

cln-grpc/src/test.rs

@@ -278,6 +278,7 @@ fn test_keysend() {
                 }],
                 }],
             }),
+	    extratlvs: None,
         };
 
     let u: cln_rpc::model::KeysendRequest = g.into();

cln-rpc/src/model.rs

@@ -864,10 +864,6 @@ pub mod requests {
 	    type Response = super::responses::WithdrawResponse;
 	}
 
-	#[derive(Clone, Debug, Deserialize, Serialize)]
-	pub struct KeysendExtratlvs {
-	}
-
 	#[derive(Clone, Debug, Deserialize, Serialize)]
 	pub struct KeysendRequest {
 	    #[serde(alias = "destination")]
@@ -886,6 +882,8 @@ pub mod requests {
 	    pub exemptfee: Option<Amount>,
 	    #[serde(alias = "routehints", skip_serializing_if = "Option::is_none")]
 	    pub routehints: Option<RoutehintList>,
+	    #[serde(alias = "extratlvs", skip_serializing_if = "Option::is_none")]
+	    pub extratlvs: Option<TlvStream>,
 	}
 
 	impl From<KeysendRequest> for Request {

cln-rpc/src/primitives.rs

@@ -548,6 +548,19 @@ mod test {
         let serialized: String = serde_json::to_string(&od).unwrap();
         assert_eq!(a, serialized);
     }
+
+    #[test]
+    fn tlvstream() {
+	let stream = TlvStream {
+	    entries: vec![
+		TlvEntry { typ: 31337, value: vec![1,2,3,4,5]},
+			   TlvEntry { typ: 42, value: vec![]},
+	    ],
+	};
+
+	let res = serde_json::to_string(&stream).unwrap();
+        assert_eq!(res, "{\"31337\":\"0102030405\",\"42\":\"\"}");
+    }
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -624,3 +637,49 @@ impl Display for RpcError {
 }
 
 impl std::error::Error for RpcError {}
+
+#[derive(Clone, Debug)]
+pub struct TlvEntry {
+    pub typ: u64,
+    pub value: Vec<u8>,
+}
+
+#[derive(Clone, Debug)]
+pub struct TlvStream {
+    pub entries: Vec<TlvEntry>,
+}
+
+impl<'de> Deserialize<'de> for TlvStream {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let map: std::collections::HashMap<u64, String> = Deserialize::deserialize(deserializer)?;
+
+        let entries = map
+            .iter()
+            .map(|(k, v)| TlvEntry {
+                typ: *k,
+                value: hex::decode(v).unwrap(),
+            })
+            .collect();
+
+        Ok(TlvStream { entries })
+    }
+}
+
+impl Serialize for TlvStream {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        use serde::ser::SerializeMap;
+
+        let mut map = serializer.serialize_map(Some(self.entries.len()))?;
+        for e in &self.entries {
+            map.serialize_key(&e.typ)?;
+            map.serialize_value(&hex::encode(&e.value))?;
+        }
+        map.end()
+    }
+}

contrib/msggen/msggen/gen/grpc.py

@@ -440,6 +440,7 @@ class GrpcUnconverterGenerator(GrpcConverterGenerator):
                     'hash': f'Sha256::from_slice(&c.{name}).unwrap()',
                     'hash?': f'c.{name}.map(|v| Sha256::from_slice(&v).unwrap())',
                     'txid': f'hex::encode(&c.{name})',
+                    'TlvStream?': f'c.{name}.map(|s| s.into())',
                 }.get(
                     typ,
                     f'c.{name}'  # default to just assignment

contrib/msggen/msggen/model.py

@@ -343,7 +343,21 @@ InvoiceLabelField = PrimitiveField("string", None, None)
 DatastoreKeyField = ArrayField(itemtype=PrimitiveField("string", None, None), dims=1, path=None, description=None)
 InvoiceExposeprivatechannelsField = PrimitiveField("boolean", None, None)
 PayExclude = ArrayField(itemtype=PrimitiveField("string", None, None), dims=1, path=None, description=None)
-RoutehintListField = PrimitiveField("RoutehintList", None, None)
+RoutehintListField = PrimitiveField(
+    "RoutehintList",
+    None,
+    None
+)
+
+# TlvStreams are special, they don't have preset dict-keys, rather
+# they can specify `u64` keys pointing to hex payloads. So the schema
+# has to rely on additionalProperties to make it work.
+TlvStreamField = PrimitiveField(
+    "TlvStream",
+    None,
+    None
+)
+
 # Override fields with manually managed types, fieldpath -> field mapping
 overrides = {
     'Invoice.label': InvoiceLabelField,
@@ -355,6 +369,7 @@ overrides = {
     'Invoice.exposeprivatechannels': InvoiceExposeprivatechannelsField,
     'Pay.exclude': PayExclude,
     'KeySend.routehints': RoutehintListField,
+    'KeySend.extratlvs': TlvStreamField,
 }
 
 

contrib/pyln-testing/pyln/testing/primitives_pb2.py

@@ -15,7 +15,7 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10primitives.proto\x12\x03\x63ln\"\x16\n\x06\x41mount\x12\x0c\n\x04msat\x18\x01 \x01(\x04\"D\n\x0b\x41mountOrAll\x12\x1d\n\x06\x61mount\x18\x01 \x01(\x0b\x32\x0b.cln.AmountH\x00\x12\r\n\x03\x61ll\x18\x02 \x01(\x08H\x00\x42\x07\n\x05value\"D\n\x0b\x41mountOrAny\x12\x1d\n\x06\x61mount\x18\x01 \x01(\x0b\x32\x0b.cln.AmountH\x00\x12\r\n\x03\x61ny\x18\x02 \x01(\x08H\x00\x42\x07\n\x05value\"\x19\n\x17\x43hannelStateChangeCause\"(\n\x08Outpoint\x12\x0c\n\x04txid\x18\x01 \x01(\x0c\x12\x0e\n\x06outnum\x18\x02 \x01(\r\"h\n\x07\x46\x65\x65rate\x12\x0e\n\x04slow\x18\x01 \x01(\x08H\x00\x12\x10\n\x06normal\x18\x02 \x01(\x08H\x00\x12\x10\n\x06urgent\x18\x03 \x01(\x08H\x00\x12\x0f\n\x05perkb\x18\x04 \x01(\rH\x00\x12\x0f\n\x05perkw\x18\x05 \x01(\rH\x00\x42\x07\n\x05style\":\n\nOutputDesc\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\t\x12\x1b\n\x06\x61mount\x18\x02 \x01(\x0b\x32\x0b.cln.Amount\"t\n\x08RouteHop\x12\n\n\x02id\x18\x01 \x01(\x0c\x12\x18\n\x10short_channel_id\x18\x02 \x01(\t\x12\x1c\n\x07\x66\x65\x65\x62\x61se\x18\x03 \x01(\x0b\x32\x0b.cln.Amount\x12\x0f\n\x07\x66\x65\x65prop\x18\x04 \x01(\r\x12\x13\n\x0b\x65xpirydelta\x18\x05 \x01(\r\"(\n\tRoutehint\x12\x1b\n\x04hops\x18\x01 \x03(\x0b\x32\r.cln.RouteHop\".\n\rRoutehintList\x12\x1d\n\x05hints\x18\x02 \x03(\x0b\x32\x0e.cln.Routehint*\x1e\n\x0b\x43hannelSide\x12\x06\n\x02IN\x10\x00\x12\x07\n\x03OUT\x10\x01*\x84\x02\n\x0c\x43hannelState\x12\x0c\n\x08Openingd\x10\x00\x12\x1a\n\x16\x43hanneldAwaitingLockin\x10\x01\x12\x12\n\x0e\x43hanneldNormal\x10\x02\x12\x18\n\x14\x43hanneldShuttingDown\x10\x03\x12\x17\n\x13\x43losingdSigexchange\x10\x04\x12\x14\n\x10\x43losingdComplete\x10\x05\x12\x16\n\x12\x41waitingUnilateral\x10\x06\x12\x14\n\x10\x46undingSpendSeen\x10\x07\x12\x0b\n\x07Onchain\x10\x08\x12\x15\n\x11\x44ualopendOpenInit\x10\t\x12\x1b\n\x17\x44ualopendAwaitingLockin\x10\nb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x10primitives.proto\x12\x03\x63ln\"\x16\n\x06\x41mount\x12\x0c\n\x04msat\x18\x01 \x01(\x04\"D\n\x0b\x41mountOrAll\x12\x1d\n\x06\x61mount\x18\x01 \x01(\x0b\x32\x0b.cln.AmountH\x00\x12\r\n\x03\x61ll\x18\x02 \x01(\x08H\x00\x42\x07\n\x05value\"D\n\x0b\x41mountOrAny\x12\x1d\n\x06\x61mount\x18\x01 \x01(\x0b\x32\x0b.cln.AmountH\x00\x12\r\n\x03\x61ny\x18\x02 \x01(\x08H\x00\x42\x07\n\x05value\"\x19\n\x17\x43hannelStateChangeCause\"(\n\x08Outpoint\x12\x0c\n\x04txid\x18\x01 \x01(\x0c\x12\x0e\n\x06outnum\x18\x02 \x01(\r\"h\n\x07\x46\x65\x65rate\x12\x0e\n\x04slow\x18\x01 \x01(\x08H\x00\x12\x10\n\x06normal\x18\x02 \x01(\x08H\x00\x12\x10\n\x06urgent\x18\x03 \x01(\x08H\x00\x12\x0f\n\x05perkb\x18\x04 \x01(\rH\x00\x12\x0f\n\x05perkw\x18\x05 \x01(\rH\x00\x42\x07\n\x05style\":\n\nOutputDesc\x12\x0f\n\x07\x61\x64\x64ress\x18\x01 \x01(\t\x12\x1b\n\x06\x61mount\x18\x02 \x01(\x0b\x32\x0b.cln.Amount\"t\n\x08RouteHop\x12\n\n\x02id\x18\x01 \x01(\x0c\x12\x18\n\x10short_channel_id\x18\x02 \x01(\t\x12\x1c\n\x07\x66\x65\x65\x62\x61se\x18\x03 \x01(\x0b\x32\x0b.cln.Amount\x12\x0f\n\x07\x66\x65\x65prop\x18\x04 \x01(\r\x12\x13\n\x0b\x65xpirydelta\x18\x05 \x01(\r\"(\n\tRoutehint\x12\x1b\n\x04hops\x18\x01 \x03(\x0b\x32\r.cln.RouteHop\".\n\rRoutehintList\x12\x1d\n\x05hints\x18\x02 \x03(\x0b\x32\x0e.cln.Routehint\"\'\n\x08TlvEntry\x12\x0c\n\x04type\x18\x01 \x01(\x04\x12\r\n\x05value\x18\x02 \x01(\x0c\"+\n\tTlvStream\x12\x1e\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\r.cln.TlvEntry*\x1e\n\x0b\x43hannelSide\x12\x06\n\x02IN\x10\x00\x12\x07\n\x03OUT\x10\x01*\x84\x02\n\x0c\x43hannelState\x12\x0c\n\x08Openingd\x10\x00\x12\x1a\n\x16\x43hanneldAwaitingLockin\x10\x01\x12\x12\n\x0e\x43hanneldNormal\x10\x02\x12\x18\n\x14\x43hanneldShuttingDown\x10\x03\x12\x17\n\x13\x43losingdSigexchange\x10\x04\x12\x14\n\x10\x43losingdComplete\x10\x05\x12\x16\n\x12\x41waitingUnilateral\x10\x06\x12\x14\n\x10\x46undingSpendSeen\x10\x07\x12\x0b\n\x07Onchain\x10\x08\x12\x15\n\x11\x44ualopendOpenInit\x10\t\x12\x1b\n\x17\x44ualopendAwaitingLockin\x10\nb\x06proto3')
 
 _CHANNELSIDE = DESCRIPTOR.enum_types_by_name['ChannelSide']
 ChannelSide = enum_type_wrapper.EnumTypeWrapper(_CHANNELSIDE)
@@ -46,6 +46,8 @@ _OUTPUTDESC = DESCRIPTOR.message_types_by_name['OutputDesc']
 _ROUTEHOP = DESCRIPTOR.message_types_by_name['RouteHop']
 _ROUTEHINT = DESCRIPTOR.message_types_by_name['Routehint']
 _ROUTEHINTLIST = DESCRIPTOR.message_types_by_name['RoutehintList']
+_TLVENTRY = DESCRIPTOR.message_types_by_name['TlvEntry']
+_TLVSTREAM = DESCRIPTOR.message_types_by_name['TlvStream']
 Amount = _reflection.GeneratedProtocolMessageType('Amount', (_message.Message,), {
   'DESCRIPTOR' : _AMOUNT,
   '__module__' : 'primitives_pb2'
@@ -116,13 +118,27 @@ RoutehintList = _reflection.GeneratedProtocolMessageType('RoutehintList', (_mess
   })
 _sym_db.RegisterMessage(RoutehintList)
 
+TlvEntry = _reflection.GeneratedProtocolMessageType('TlvEntry', (_message.Message,), {
+  'DESCRIPTOR' : _TLVENTRY,
+  '__module__' : 'primitives_pb2'
+  # @@protoc_insertion_point(class_scope:cln.TlvEntry)
+  })
+_sym_db.RegisterMessage(TlvEntry)
+
+TlvStream = _reflection.GeneratedProtocolMessageType('TlvStream', (_message.Message,), {
+  'DESCRIPTOR' : _TLVSTREAM,
+  '__module__' : 'primitives_pb2'
+  # @@protoc_insertion_point(class_scope:cln.TlvStream)
+  })
+_sym_db.RegisterMessage(TlvStream)
+
 if _descriptor._USE_C_DESCRIPTORS == False:
 
   DESCRIPTOR._options = None
-  _CHANNELSIDE._serialized_start=632
+  _CHANNELSIDE._serialized_start=718
-  _CHANNELSIDE._serialized_end=662
+  _CHANNELSIDE._serialized_end=748
-  _CHANNELSTATE._serialized_start=665
+  _CHANNELSTATE._serialized_start=751
-  _CHANNELSTATE._serialized_end=925
+  _CHANNELSTATE._serialized_end=1011
   _AMOUNT._serialized_start=25
   _AMOUNT._serialized_end=47
   _AMOUNTORALL._serialized_start=49
@@ -143,4 +159,8 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _ROUTEHINT._serialized_end=582
   _ROUTEHINTLIST._serialized_start=584
   _ROUTEHINTLIST._serialized_end=630
+  _TLVENTRY._serialized_start=632
+  _TLVENTRY._serialized_end=671
+  _TLVSTREAM._serialized_start=673
+  _TLVSTREAM._serialized_end=716
 # @@protoc_insertion_point(module_scope)