diff --git a/plugins/clnrest/.github/screenshots/Postman-bkpr-plugin.png b/plugins/clnrest/.github/screenshots/Postman-bkpr-plugin.png new file mode 100644 index 000000000..561a2c3e8 Binary files /dev/null and b/plugins/clnrest/.github/screenshots/Postman-bkpr-plugin.png differ diff --git a/plugins/clnrest/.github/screenshots/Postman-with-body.png b/plugins/clnrest/.github/screenshots/Postman-with-body.png new file mode 100644 index 000000000..f07604dbc Binary files /dev/null and b/plugins/clnrest/.github/screenshots/Postman-with-body.png differ diff --git a/plugins/clnrest/.github/screenshots/Postman.png b/plugins/clnrest/.github/screenshots/Postman.png new file mode 100644 index 000000000..df6ea7cb8 Binary files /dev/null and b/plugins/clnrest/.github/screenshots/Postman.png differ diff --git a/plugins/clnrest/.github/screenshots/Swagger-auth.png b/plugins/clnrest/.github/screenshots/Swagger-auth.png new file mode 100644 index 000000000..46c170a54 Binary files /dev/null and b/plugins/clnrest/.github/screenshots/Swagger-auth.png differ diff --git a/plugins/clnrest/.github/screenshots/Swagger-list-methods.png b/plugins/clnrest/.github/screenshots/Swagger-list-methods.png new file mode 100644 index 000000000..e5e6eeb44 Binary files /dev/null and b/plugins/clnrest/.github/screenshots/Swagger-list-methods.png differ diff --git a/plugins/clnrest/.github/screenshots/Swagger-rpc-method.png b/plugins/clnrest/.github/screenshots/Swagger-rpc-method.png new file mode 100644 index 000000000..7d0384f04 Binary files /dev/null and b/plugins/clnrest/.github/screenshots/Swagger-rpc-method.png differ diff --git a/plugins/clnrest/.github/screenshots/Swagger.png b/plugins/clnrest/.github/screenshots/Swagger.png new file mode 100644 index 000000000..6f2f4ac76 Binary files /dev/null and b/plugins/clnrest/.github/screenshots/Swagger.png differ diff --git a/plugins/clnrest/README.md b/plugins/clnrest/README.md new file mode 100644 index 000000000..52c132c39 --- /dev/null +++ b/plugins/clnrest/README.md @@ -0,0 +1,47 @@ +# CLNRest + +CLNRest is a lightweight Python-based core lightning plugin that transforms RPC calls into a REST service. By generating REST API endpoints, it enables the execution of Core Lightning's RPC methods behind the scenes and provides responses in JSON format. + +## Installation + +Install required packages with `pip install json5 flask flask_restx gunicorn pyln-client` or `pip install -r requirements.txt`. + +## Configuration + +- --rest-protocol: Specifies the REST server protocol. Default is HTTPS. +- --rest-host: Defines the REST server host. Default is 127.0.0.1. +- --rest-port: Sets the REST server port to listen to. Default is 3010. +- --rest-certs: Defines the path for HTTPS cert & key. Default path is same as RPC file path to utilize gRPC's client certificate. If it is missing at the configured location, new identity (`client.pem` and `client-key.pem`) will be generated. + +## Plugin + +- It can be configured by adding `plugin=//clnrest/clnrest.py` to the Core Lightning's config file. + +## Server + +With the default configurations, the Swagger user interface will be available at https://127.0.0.1:3010/. The POST method requires `rune` and `nodeid` headers for authorization. + +### cURL +Example curl command for GET: + `curl -k https://127.0.0.1:3010/v1/notifications` + +Example curl command for POST will also require `rune` and `nodeid` headers like below: + `curl -k -X POST 'https://127.0.0.1:3010/v1/getinfo' -H 'Rune: ' -H 'Nodeid: '` + +With `-k` or `--insecure` option curl proceeds with the connection even if the SSL certificate cannot be verified. +This option should be used only when testing with self signed certificate. + +### Swagger +

+ Swagger Dashboard + Swagger Authorize + Swagger GET List Methods + Swagger POST RPC Method +

+ +### Postman +

+ Postman Headers + Postman with JSON body + Postman bkpr plugin RPC +

diff --git a/plugins/clnrest/clnrest.py b/plugins/clnrest/clnrest.py new file mode 100755 index 000000000..3dd06d415 --- /dev/null +++ b/plugins/clnrest/clnrest.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# For --hidden-import gunicorn.glogging gunicorn.workers.sync +from gunicorn import glogging # noqa: F401 +from gunicorn.workers import sync # noqa: F401 + +from pathlib import Path +from flask import Flask +from flask_restx import Api +from gunicorn.app.base import BaseApplication +from multiprocessing import Process, cpu_count +from utilities.generate_certs import generate_certs +from utilities.shared import set_config +from utilities.rpc_routes import rpcns +from utilities.rpc_plugin import plugin + +jobs = {} + + +def create_app(): + app = Flask(__name__) + authorizations = { + "rune": {"type": "apiKey", "in": "header", "name": "Rune"}, + "nodeid": {"type": "apiKey", "in": "header", "name": "Nodeid"} + } + api = Api(app, version="1.0", title="Core Lightning Rest", description="Core Lightning REST API Swagger", authorizations=authorizations, security=["rune", "nodeid"]) + api.add_namespace(rpcns, path="/v1") + return app + + +def set_application_options(plugin): + from utilities.shared import CERTS_PATH, REST_PROTOCOL, REST_HOST, REST_PORT + plugin.log(f"REST Server is starting at {REST_PROTOCOL}://{REST_HOST}:{REST_PORT}", "debug") + if REST_PROTOCOL == "http": + options = { + "bind": f"{REST_HOST}:{REST_PORT}", + "workers": cpu_count(), + "timeout": 60, + "loglevel": "warning", + } + else: + cert_file = Path(f"{CERTS_PATH}/client.pem") + key_file = Path(f"{CERTS_PATH}/client-key.pem") + if not cert_file.is_file() or not key_file.is_file(): + plugin.log(f"Certificate not found at {CERTS_PATH}. Generating a new certificate!", "debug") + generate_certs(plugin, CERTS_PATH) + try: + plugin.log(f"Certs Path: {CERTS_PATH}", "debug") + except Exception as err: + raise Exception(f"{err}: Certificates do not exist at {CERTS_PATH}") + options = { + "bind": f"{REST_HOST}:{REST_PORT}", + "workers": cpu_count(), + "timeout": 60, + "loglevel": "warning", + "certfile": f"{CERTS_PATH}/client.pem", + "keyfile": f"{CERTS_PATH}/client-key.pem", + } + return options + + +class CLNRestApplication(BaseApplication): + def __init__(self, app, options=None): + from utilities.shared import REST_PROTOCOL, REST_HOST, REST_PORT + self.application = app + self.options = options or {} + plugin.log(f"REST server running at {REST_PROTOCOL}://{REST_HOST}:{REST_PORT}", "info") + super().__init__() + + def load_config(self): + config = {key: value for key, value in self.options.items() + if key in self.cfg.settings and value is not None} + for key, value in config.items(): + self.cfg.set(key.lower(), value) + + def load(self): + return self.application + + +def worker(): + options = set_application_options(plugin) + app = create_app() + CLNRestApplication(app, options).run() + + +def start_server(): + from utilities.shared import REST_PORT + if REST_PORT in jobs: + return False, "server already running" + + p = Process( + target=worker, + args=[], + name="server on port {}".format(REST_PORT), + ) + p.daemon = True + jobs[REST_PORT] = p + p.start() + return True + + +@plugin.init() +def init(options, configuration, plugin): + set_config(options) + start_server() + + +plugin.run() diff --git a/plugins/clnrest/requirements.txt b/plugins/clnrest/requirements.txt new file mode 100644 index 000000000..f308c9778 --- /dev/null +++ b/plugins/clnrest/requirements.txt @@ -0,0 +1,34 @@ +aniso8601==9.0.1 +asn1crypto==1.5.1 +attrs==23.1.0 +base58==2.1.1 +bitstring==3.1.9 +blinker==1.6.2 +cachelib==0.10.2 +cffi==1.15.1 +click==8.1.3 +coincurve==17.0.0 +cryptography==36.0.2 +Flask==2.3.2 +Flask-Cors==4.0.0 +flask-restx==1.1.0 +Flask-WTF==1.1.1 +gevent==22.10.2 +greenlet==2.0.2 +gunicorn==20.1.0 +itsdangerous==2.1.2 +Jinja2==3.1.2 +json5==0.9.14 +jsonschema==4.17.3 +MarkupSafe==2.1.3 +pycparser==2.21 +pyln-bolt7==1.0.246 +pyln-client==23.5 +pyln-proto==23.5 +pyrsistent==0.19.3 +PySocks==1.7.1 +pytz==2023.3 +Werkzeug==2.3.6 +WTForms==3.0.1 +zope.event==5.0 +zope.interface==6.0 diff --git a/plugins/clnrest/utilities/__init__.py b/plugins/clnrest/utilities/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/plugins/clnrest/utilities/generate_certs.py b/plugins/clnrest/utilities/generate_certs.py new file mode 100644 index 000000000..7c086bc17 --- /dev/null +++ b/plugins/clnrest/utilities/generate_certs.py @@ -0,0 +1,36 @@ +import os +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import Encoding +import datetime + + +def generate_certs(plugin, certs_path): + # Generate key + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + # Create the certs directory if it does not exist + os.makedirs(certs_path, exist_ok=True) + # Write key + with open(os.path.join(certs_path, "client-key.pem"), "wb") as f: + f.write(key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + )) + subject = issuer = x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"Core Lightning")]) + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.utcnow()) + .not_valid_after(datetime.datetime.utcnow() + datetime.timedelta(days=10 * 365)) # Ten years validity + .sign(key, hashes.SHA256()) + ) + with open(os.path.join(certs_path, "client.pem"), "wb") as f: + f.write(cert.public_bytes(Encoding.PEM)) + plugin.log(f"Certificate Generated!", "debug") diff --git a/plugins/clnrest/utilities/rpc_plugin.py b/plugins/clnrest/utilities/rpc_plugin.py new file mode 100644 index 000000000..14433cb89 --- /dev/null +++ b/plugins/clnrest/utilities/rpc_plugin.py @@ -0,0 +1,23 @@ +import os +import sys +from multiprocessing import Manager +from pyln.client import Plugin + +plugin = Plugin(autopatch=False) +manager = Manager() +queue = manager.Queue() + +plugin.add_option(name="rest-certs", default=os.getcwd(), description="Path for certificates (for https)", opt_type="string", deprecated=False) +plugin.add_option(name="rest-protocol", default="https", description="REST server protocol", opt_type="string", deprecated=False) +plugin.add_option(name="rest-host", default="127.0.0.1", description="REST server host", opt_type="string", deprecated=False) +plugin.add_option(name="rest-port", default=3010, description="REST server port to listen", opt_type="int", deprecated=False) + + +@plugin.subscribe("*") +def on_any_notification(request, **kwargs): + plugin.log("Notification: {}".format(kwargs), "debug") + if request.method == 'shutdown': + # A plugin which subscribes to shutdown is expected to exit itself. + sys.exit(0) + else: + queue.put(str(kwargs) + "\n") diff --git a/plugins/clnrest/utilities/rpc_routes.py b/plugins/clnrest/utilities/rpc_routes.py new file mode 100644 index 000000000..11a5da3e3 --- /dev/null +++ b/plugins/clnrest/utilities/rpc_routes.py @@ -0,0 +1,72 @@ +import json5 +from flask import request, make_response, Response, stream_with_context +from flask_restx import Namespace, Resource +from .shared import call_rpc_method, verify_rune, process_help_response +from .rpc_plugin import plugin + +methods_list = [] +rpcns = Namespace("RPCs") +payload_model = rpcns.model("Payload", {}, None, False) + + +@rpcns.route("/list-methods") +class ListMethodsResource(Resource): + @rpcns.response(200, "Success") + @rpcns.response(500, "Server error") + def get(self): + """Get the list of all valid rpc methods, useful for Swagger to get human readable list without calling lightning-cli help""" + try: + help_response = call_rpc_method(plugin, "help", []) + html_content = process_help_response(help_response) + response = make_response(html_content) + response.headers["Content-Type"] = "text/html" + return response + + except Exception as err: + plugin.log(f"Error: {err}", "error") + return json5.loads(str(err)), 500 + + +@rpcns.route("/") +class RpcMethodResource(Resource): + @rpcns.doc(security=[{"rune": [], "nodeid": []}]) + @rpcns.doc(params={"rpc_method": (f"Name of the RPC method to be called")}) + @rpcns.expect(payload_model, validate=False) + @rpcns.response(201, "Success") + @rpcns.response(500, "Server error") + def post(self, rpc_method): + """Call any valid core lightning method (check list-methods response)""" + try: + is_valid_rune = verify_rune(plugin, request) + + if "error" in is_valid_rune: + plugin.log(f"Error: {is_valid_rune}", "error") + raise Exception(is_valid_rune) + + except Exception as err: + return json5.loads(str(err)), 403 + + try: + if request.is_json: + payload = request.get_json() + else: + payload = request.form.to_dict() + return call_rpc_method(plugin, rpc_method, payload), 201 + + except Exception as err: + plugin.log(f"Error: {err}", "error") + return json5.loads(str(err)), 500 + + +@rpcns.route("/notifications") +class NotificationsResource(Resource): + def get(self): + try: + def notifications_stream(): + while True: + from .rpc_plugin import queue + yield queue.get() + return Response(stream_with_context(notifications_stream()), mimetype="text/event-stream") + + except Exception as err: + return json5.loads(str(err)), 500 diff --git a/plugins/clnrest/utilities/shared.py b/plugins/clnrest/utilities/shared.py new file mode 100644 index 000000000..90d3fa6a3 --- /dev/null +++ b/plugins/clnrest/utilities/shared.py @@ -0,0 +1,71 @@ +import json5 +import re +import json + +CERTS_PATH, REST_PROTOCOL, REST_HOST, REST_PORT = "", "", "", "" + + +def set_config(options): + global CERTS_PATH, REST_PROTOCOL, REST_HOST, REST_PORT + CERTS_PATH = str(options["rest-certs"]) + REST_PROTOCOL = str(options["rest-protocol"]) + REST_HOST = str(options["rest-host"]) + REST_PORT = int(options["rest-port"]) + + +def call_rpc_method(plugin, rpc_method, payload): + try: + response = plugin.rpc.call(rpc_method, payload) + if '"error":' in str(response).lower(): + raise Exception(response) + else: + plugin.log(f"{response}", "debug") + if '"result":' in str(response).lower(): + # Use json5.loads ONLY when necessary, as it increases processing time significantly + return json.loads(response)["result"] + else: + return response + + except Exception as err: + plugin.log(f"Error: {err}", "error") + if "error" in str(err).lower(): + match_err_obj = re.search(r'"error":\{.*?\}', str(err)) + if match_err_obj is not None: + err = "{" + match_err_obj.group() + "}" + else: + match_err_str = re.search(r"error: \{.*?\}", str(err)) + if match_err_str is not None: + err = "{" + match_err_str.group() + "}" + raise Exception(err) + + +def verify_rune(plugin, request): + rune = request.headers.get("rune", None) + nodeid = request.headers.get("nodeid", None) + + if nodeid is None: + raise Exception('{ "error": {"code": 403, "message": "Not authorized: Missing nodeid"} }') + + if rune is None: + raise Exception('{ "error": {"code": 403, "message": "Not authorized: Missing rune"} }') + + if request.is_json: + rpc_params = request.get_json() + else: + rpc_params = request.form.to_dict() + + return call_rpc_method(plugin, "checkrune", [rune, nodeid, request.view_args["rpc_method"], rpc_params]) + + +def process_help_response(help_response): + # Use json5.loads due to single quotes in response + processed_res = json5.loads(str(help_response))["help"] + line = "\n---------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n\n" + processed_html_res = "" + for row in processed_res: + processed_html_res += f"Command: {row['command']}\n" + processed_html_res += f"Category: {row['category']}\n" + processed_html_res += f"Description: {row['description']}\n" + processed_html_res += f"Verbose: {row['verbose']}\n" + processed_html_res += line + return processed_html_res