cln-grpc: Add generation of grpc protobuf file from schema

Makefile

@@ -356,6 +356,7 @@ ifneq ($(FUZZING),0)
 endif
 ifneq ($(RUST),0)
 	include cln-rpc/Makefile
+	include cln-grpc/Makefile
 endif
 
 # We make pretty much everything depend on these.

cln-grpc/Makefile

@@ -0,0 +1,11 @@
+cln-grpc-wrongdir:
+	$(MAKE) -C .. cln-grpc-all
+
+CLN_GRPC_EXAMPLES :=
+CLN_GRPC_GENALL = cln-grpc/proto/node.proto
+DEFAULT_TARGETS += $(CLN_GRPC_EXAMPLES) $(CLN_GRPC_GENALL)
+
+$(CLN_GRPC_GENALL): $(JSON_SCHEMA)
+	PYTHONPATH=contrib/msggen python3 contrib/msggen/msggen/__main__.py
+
+cln-grpc-all: ${CLN_GRPC_GENALL} ${CLN_GRPC_EXAMPLES}

cln-grpc/proto/node.proto

@@ -0,0 +1,184 @@
+syntax = "proto3";
+package cln;
+
+// This file was automatically derived from the JSON-RPC schemas in
+// `doc/schemas`. Do not edit this file manually as it would get
+// overwritten.
+
+import "primitives.proto";
+
+service Node {
+	rpc Getinfo(GetinfoRequest) returns (GetinfoResponse) {}
+	rpc ListFunds(ListfundsRequest) returns (ListfundsResponse) {}
+	rpc ListChannels(ListchannelsRequest) returns (ListchannelsResponse) {}
+	rpc AddGossip(AddgossipRequest) returns (AddgossipResponse) {}
+	rpc AutoCleanInvoice(AutocleaninvoiceRequest) returns (AutocleaninvoiceResponse) {}
+	rpc CheckMessage(CheckmessageRequest) returns (CheckmessageResponse) {}
+	rpc Close(CloseRequest) returns (CloseResponse) {}
+}
+
+message GetinfoRequest {
+}
+
+message GetinfoResponse {
+	bytes id = 1;
+	string alias = 2;
+	bytes color = 3;
+	uint32 num_peers = 4;
+	uint32 num_pending_channels = 5;
+	uint32 num_active_channels = 6;
+	uint32 num_inactive_channels = 7;
+	string version = 8;
+	string lightning_dir = 9;
+	uint32 blockheight = 10;
+	string network = 11;
+	Amount fees_collected_msat = 12;
+	repeated GetinfoAddress address = 13;
+	repeated GetinfoBinding binding = 14;
+	optional string warning_bitcoind_sync = 15;
+	optional string warning_lightningd_sync = 16;
+}
+
+message GetinfoAddress {
+	// Getinfo.address[].type
+	enum GetinfoAddressType {
+		DNS = 0;
+		IPV4 = 1;
+		IPV6 = 2;
+		TORV2 = 3;
+		TORV3 = 4;
+		WEBSOCKET = 5;
+	}
+	GetinfoAddressType item_type = 1;
+	uint32 port = 2;
+	optional string address = 3;
+}
+
+message GetinfoBinding {
+	// Getinfo.binding[].type
+	enum GetinfoBindingType {
+		LOCAL_SOCKET = 0;
+		IPV4 = 1;
+		IPV6 = 2;
+		TORV2 = 3;
+		TORV3 = 4;
+	}
+	GetinfoBindingType item_type = 1;
+	optional string address = 2;
+	optional uint32 port = 3;
+	optional string socket = 4;
+}
+
+message ListfundsRequest {
+	optional bool spent = 1;
+}
+
+message ListfundsResponse {
+	repeated ListfundsOutputs outputs = 1;
+	repeated ListfundsChannels channels = 2;
+}
+
+message ListfundsOutputs {
+	// ListFunds.outputs[].status
+	enum ListfundsOutputsStatus {
+		UNCONFIRMED = 0;
+		CONFIRMED = 1;
+		SPENT = 2;
+	}
+	bytes txid = 1;
+	uint32 output = 2;
+	Amount amount_msat = 3;
+	bytes scriptpubkey = 4;
+	optional string address = 5;
+	optional bytes redeemscript = 6;
+	ListfundsOutputsStatus status = 7;
+	optional uint32 blockheight = 8;
+}
+
+message ListfundsChannels {
+	bytes peer_id = 1;
+	Amount our_amount_msat = 2;
+	Amount amount_msat = 3;
+	bytes funding_txid = 4;
+	uint32 funding_output = 5;
+	bool connected = 6;
+	ChannelState state = 7;
+	optional string short_channel_id = 8;
+}
+
+message ListchannelsRequest {
+	optional string short_channel_id = 1;
+	optional bytes source = 2;
+	optional bytes destination = 3;
+}
+
+message ListchannelsResponse {
+	repeated ListchannelsChannels channels = 1;
+}
+
+message ListchannelsChannels {
+	bytes source = 1;
+	bytes destination = 2;
+	bool public = 3;
+	Amount amount_msat = 4;
+	uint32 message_flags = 5;
+	uint32 channel_flags = 6;
+	bool active = 7;
+	uint32 last_update = 8;
+	uint32 base_fee_millisatoshi = 9;
+	uint32 fee_per_millionth = 10;
+	uint32 delay = 11;
+	Amount htlc_minimum_msat = 12;
+	optional Amount htlc_maximum_msat = 13;
+	bytes features = 14;
+}
+
+message AddgossipRequest {
+	bytes message = 1;
+}
+
+message AddgossipResponse {
+}
+
+message AutocleaninvoiceRequest {
+	optional uint64 expired_by = 1;
+	optional uint64 cycle_seconds = 2;
+}
+
+message AutocleaninvoiceResponse {
+	bool enabled = 1;
+	optional uint64 expired_by = 2;
+	optional uint64 cycle_seconds = 3;
+}
+
+message CheckmessageRequest {
+	string message = 1;
+	string zbase = 2;
+	optional bytes pubkey = 3;
+}
+
+message CheckmessageResponse {
+	bool verified = 1;
+	optional bytes pubkey = 2;
+}
+
+message CloseRequest {
+	bytes id = 1;
+	optional uint32 unilateraltimeout = 2;
+	optional string destination = 3;
+	optional string fee_negotiation_step = 4;
+	optional bytes wrong_funding = 5;
+	optional bool force_lease_closed = 6;
+}
+
+message CloseResponse {
+	// Close.type
+	enum CloseType {
+		MUTUAL = 0;
+		UNILATERAL = 1;
+		UNOPENED = 2;
+	}
+	CloseType item_type = 1;
+	optional bytes tx = 2;
+	optional bytes txid = 3;
+}

cln-grpc/proto/primitives.proto

@@ -0,0 +1,33 @@
+syntax = "proto3";
+package cln;
+
+message Amount {
+	oneof unit {
+		uint64 millisatoshi = 1;
+		uint64 satoshi = 2;
+		uint64 bitcoin = 3;
+		bool all = 4;
+		bool any = 5;
+	}
+}
+
+enum ChannelSide {
+	IN = 0;
+	OUT = 1;
+}
+
+enum ChannelState {
+	Openingd = 0;
+	ChanneldAwaitingLockin = 1;
+	ChanneldNormal = 2;
+	ChanneldShuttingDown = 3;
+	ClosingdSigexchange = 4;
+	ClosingdComplete = 5;
+	AwaitingUnilateral = 6;
+	FundingSpendSeen = 7;
+	Onchain = 8;
+	DualopendOpenInit = 9;
+	DualopendAwaitingLockin = 10;
+}
+
+message ChannelStateChangeCause {}

contrib/msggen/msggen/__main__.py

@@ -1,4 +1,5 @@
 from msggen.model import Method, CompositeField, Service
+from msggen.grpc import GrpcGenerator
 from msggen.rust import RustGenerator
 from pathlib import Path
 import subprocess
@@ -125,6 +126,12 @@ def load_jsonrpc_service():
     return service
 
 
+def gengrpc(service):
+    """Load all mapped RPC methods, wrap them in a Service, and split them into messages.
+    """
+    fname = repo_root() / "cln-grpc" / "proto" / "node.proto"
+    dest = open(fname, "w")
+    GrpcGenerator(dest).generate(service)
 def genrustjsonrpc(service):
     fname = repo_root() / "cln-rpc" / "src" / "model.rs"
     dest = open(fname, "w")
@@ -133,6 +140,7 @@ def genrustjsonrpc(service):
 
 def run():
     service = load_jsonrpc_service()
+    gengrpc(service)
     genrustjsonrpc(service)
 
 

contrib/msggen/msggen/grpc.py

@@ -0,0 +1,153 @@
+# A grpc model
+from .model import ArrayField, Field, CompositeField, EnumField, PrimitiveField, Service
+from typing import TextIO, List
+from textwrap import indent, dedent
+import re
+import logging
+
+
+typemap = {
+    'boolean': 'bool',
+    'hex': 'bytes',
+    'msat': 'Amount',
+    'number': 'i64',
+    'pubkey': 'bytes',
+    'short_channel_id': 'string',
+    'signature': 'bytes',
+    'string': 'string',
+    'txid': 'bytes',
+    'u8': 'uint32',  # Yep, this is the smallest integer type in grpc...
+    'u32': 'uint32',
+    'u64': 'uint64',
+    'u16': 'uint32',  # Yeah, I know...
+}
+
+
+# Manual overrides for some of the auto-generated types for paths
+overrides = {
+    'ListPeers.peers[].channels[].state_changes[].old_state': "ChannelState",
+    'ListPeers.peers[].channels[].state_changes[].new_state': "ChannelState",
+    'ListPeers.peers[].channels[].state_changes[].cause': "ChannelStateChangeCause",
+    'ListPeers.peers[].channels[].opener': "ChannelSide",
+    'ListPeers.peers[].channels[].closer': "ChannelSide",
+    'ListPeers.peers[].channels[].features[]': "string",
+    'ListFunds.channels[].state': 'ChannelState',
+}
+
+
+class GrpcGenerator:
+    """A generator that generates protobuf files.
+    """
+
+    def __init__(self, dest: TextIO):
+        self.dest = dest
+        self.logger = logging.getLogger("msggen.grpc.GrpcGenerator")
+
+    def write(self, text: str, cleanup: bool = True) -> None:
+        if cleanup:
+            self.dest.write(dedent(text))
+        else:
+            self.dest.write(text)
+
+    def gather_types(self, service):
+        """Gather all types that might need to be defined.
+        """
+
+        def gather_subfields(field: Field) -> List[Field]:
+            fields = [field]
+
+            if isinstance(field, CompositeField):
+                for f in field.fields:
+                    fields.extend(gather_subfields(f))
+            elif isinstance(field, ArrayField):
+                fields = []
+                fields.extend(gather_subfields(field.itemtype))
+
+            return fields
+
+        types = []
+        for method in service.methods:
+            types.extend([method.request, method.response])
+            for field in method.request.fields:
+                types.extend(gather_subfields(field))
+            for field in method.response.fields:
+                types.extend(gather_subfields(field))
+        return types
+
+    def generate_service(self, service: Service) -> None:
+        self.write(f"""
+        service {service.name} {{
+        """)
+
+        for method in service.methods:
+            self.write(
+                f"	rpc {method.name}({method.request.typename}) returns ({method.response.typename}) {{}}\n",
+                cleanup=False,
+            )
+
+        self.write(f"""}}
+        """)
+
+    def generate_enum(self, e: EnumField, indent=0):
+        self.logger.debug(f"Generating enum {e}")
+        prefix = "\t" * indent
+        self.write(f"{prefix}// {e.path}\n", False)
+        self.write(f"{prefix}enum {e.typename} {{\n", False)
+
+        for i, v in enumerate(e.variants):
+            self.logger.debug(f"Generating enum variant {v}")
+            self.write(f"{prefix}\t{v.normalized()} = {i};\n", False)
+
+        self.write(f"""{prefix}}}\n""", False)
+
+    def generate_message(self, message: CompositeField):
+        self.write(f"""
+        message {message.typename} {{
+        """)
+
+        # Declare enums inline so they are scoped correctly in C++
+        for i, f in enumerate(message.fields):
+            if isinstance(f, EnumField) and f.path not in overrides.keys():
+                self.generate_enum(f, indent=1)
+
+        for i, f in enumerate(message.fields):
+            opt = "optional " if not f.required else ""
+            if isinstance(f, ArrayField):
+                typename = typemap.get(f.itemtype.typename, f.itemtype.typename)
+                if f.path in overrides:
+                    typename = overrides[f.path]
+                self.write(f"\trepeated {typename} {f.normalized()} = {i+1};\n", False)
+            elif isinstance(f, PrimitiveField):
+                typename = typemap.get(f.typename, f.typename)
+                if f.path in overrides:
+                    typename = overrides[f.path]
+                self.write(f"\t{opt}{typename} {f.normalized()} = {i+1};\n", False)
+            elif isinstance(f, EnumField):
+                typename = f.typename
+                if f.path in overrides:
+                    typename = overrides[f.path]
+                self.write(f"\t{opt}{typename} {f.normalized()} = {i+1};\n", False)
+
+        self.write(f"""}}
+        """)
+
+    def generate(self, service: Service) -> None:
+        """Generate the GRPC protobuf file and write to `dest`
+        """
+        self.write(f"""syntax = "proto3";\npackage cln;\n""")
+        self.write("""
+        // This file was automatically derived from the JSON-RPC schemas in
+        // `doc/schemas`. Do not edit this file manually as it would get
+        // overwritten.
+
+        """)
+
+        for i in service.includes:
+            self.write(f"import \"{i}\";\n")
+
+        self.generate_service(service)
+
+        fields = self.gather_types(service)
+
+        for message in [f for f in fields if isinstance(f, CompositeField)]:
+            self.generate_message(message)

requirements.lock

@@ -1,8 +1,8 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
-#    pip-compile --output-file=requirements.lock requirements.txt
+#    pip-compile --output-file=requirements.lock requirements.in
 #
 alabaster==0.7.12
     # via sphinx
@@ -15,9 +15,9 @@ attrs==21.2.0
 babel==2.9.1
     # via sphinx
 base58==2.0.1
-    # via pyln.proto
+    # via -r requirements.in
 bitstring==3.1.9
-    # via pyln.proto
+    # via -r requirements.in
 certifi==2021.5.30
     # via requests
 cffi==1.14.6
@@ -27,17 +27,17 @@ cffi==1.14.6
 charset-normalizer==2.0.6
     # via requests
 cheroot==8.5.2
-    # via pyln-testing
+    # via -r requirements.in
 click==7.1.2
     # via flask
 coincurve==13.0.0
-    # via pyln.proto
+    # via -r requirements.in
 commonmark==0.9.1
     # via recommonmark
 crc32c==2.2.post0
-    # via -r requirements.txt
+    # via -r requirements.in
 cryptography==3.4.8
-    # via pyln.proto
+    # via -r requirements.in
 docutils==0.17.1
     # via
     #   recommonmark
@@ -45,15 +45,21 @@ docutils==0.17.1
 entrypoints==0.3
     # via flake8
 ephemeral-port-reserve==1.1.1
-    # via pyln-testing
+    # via -r requirements.in
 execnet==1.9.0
     # via pytest-xdist
 flake8==3.7.9
-    # via -r requirements.txt
+    # via -r requirements.in
 flaky==3.7.0
-    # via pyln-testing
+    # via -r requirements.in
 flask==1.1.4
-    # via pyln-testing
+    # via -r requirements.in
+grpcio==1.34.0
+    # via
+    #   -r requirements.in
+    #   grpcio-tools
+grpcio-tools==1.34.0
+    # via -r requirements.in
 idna==3.2
     # via requests
 imagesize==1.2.0
@@ -70,9 +76,9 @@ jinja2==2.11.3
     #   mrkd
     #   sphinx
 jsonschema==3.2.0
-    # via pyln-testing
+    # via -r requirements.in
 mako==1.1.5
-    # via -r requirements.txt
+    # via -r requirements.in
 markupsafe==2.0.1
     # via
     #   jinja2
@@ -88,9 +94,9 @@ more-itertools==8.10.0
     #   cheroot
     #   jaraco.functools
 mrkd==0.1.6
-    # via -r requirements.txt
+    # via -r requirements.in
 mypy==0.910
-    # via pyln.proto
+    # via -r requirements.in
 mypy-extensions==0.4.3
     # via mypy
 packaging==21.0
@@ -101,10 +107,12 @@ plac==1.3.3
     # via mrkd
 pluggy==0.13.1
     # via pytest
+protobuf==3.19.3
+    # via grpcio-tools
 psutil==5.7.3
-    # via pyln-testing
+    # via -r requirements.in
 psycopg2-binary==2.8.6
-    # via pyln-testing
+    # via -r requirements.in
 py==1.10.0
     # via
     #   pytest
@@ -112,7 +120,9 @@ py==1.10.0
 pycodestyle==2.5.0
     # via flake8
 pycparser==2.20
-    # via cffi
+    # via
+    #   -r requirements.in
+    #   cffi
 pyflakes==2.1.1
     # via flake8
 pygments==2.10.0
@@ -124,10 +134,10 @@ pyparsing==2.4.7
 pyrsistent==0.18.0
     # via jsonschema
 pysocks==1.7.1
-    # via pyln.proto
+    # via -r requirements.in
 pytest==6.1.2
     # via
-    #   pyln-testing
+    #   -r requirements.in
     #   pytest-forked
     #   pytest-rerunfailures
     #   pytest-timeout
@@ -135,22 +145,23 @@ pytest==6.1.2
 pytest-forked==1.3.0
     # via pytest-xdist
 pytest-rerunfailures==9.1.1
-    # via pyln-testing
+    # via -r requirements.in
 pytest-timeout==1.4.2
-    # via pyln-testing
+    # via -r requirements.in
 pytest-xdist==2.2.1
-    # via pyln-testing
+    # via -r requirements.in
 python-bitcoinlib==0.11.0
-    # via pyln-testing
+    # via -r requirements.in
 pytz==2021.1
     # via babel
 recommonmark==0.7.1
-    # via pyln-client
+    # via -r requirements.in
 requests==2.26.0
     # via sphinx
 six==1.16.0
     # via
     #   cheroot
+    #   grpcio
     #   jsonschema
 snowballstemmer==2.1.0
     # via sphinx
@@ -169,15 +180,15 @@ sphinxcontrib-qthelp==1.0.3
 sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
 toml==0.10.2
-    # via pytest
-typed-ast==1.4.3
-    # via mypy
+    # via
+    #   mypy
+    #   pytest
 typing-extensions==3.10.0.2
     # via mypy
 urllib3==1.26.7
     # via requests
 websocket-client==1.2.1
-    # via -r requirements.txt
+    # via -r requirements.in
 werkzeug==1.0.1
     # via flask