From 1a426fd9409ec13d94372a1081da1789ede7a306 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 28 Aug 2020 16:51:39 +0200 Subject: [PATCH] autoreload: Remove the autoreload plugin It has been superseded by the dynamic start / stop functionality added into c-lightning itself. Closes #108 --- README.md | 3 - autoreload/README.md | 85 --------- autoreload/autoreload.py | 266 ---------------------------- autoreload/requirements.txt | 2 - autoreload/tests/dummy.py | 7 - autoreload/tests/dummy2.py | 11 -- autoreload/tests/test_autoreload.py | 59 ------ 7 files changed, 433 deletions(-) delete mode 100644 autoreload/README.md delete mode 100755 autoreload/autoreload.py delete mode 100644 autoreload/requirements.txt delete mode 100755 autoreload/tests/dummy.py delete mode 100755 autoreload/tests/dummy2.py delete mode 100644 autoreload/tests/test_autoreload.py diff --git a/README.md b/README.md index b17c0de..b4a02b0 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,12 @@ Community curated plugins for c-lightning. [![Build Status](https://travis-ci.org/lightningd/plugins.svg?branch=master)](https://travis-ci.org/lightningd/plugins) -[![Coverage Status](https://codecov.io/gh/lightningd/plugins/branch/master/graph/badge.svg)](https://codecov.io/gh/lightningd/plugins) ## Available plugins | Name | Short description | |------------------------------------|-------------------------------------------------------------------------------------------| | [autopilot][autopilot] | An autopilot that suggests channels that should be established | -| [autoreload][autoreload] | A developer plugin that reloads a plugin under development when it changes | | [boltz-channel-creation][boltz] | A c-lightning plugin for Boltz Channel Creation Swaps | | [csvexportpays][csvexportpays] | A plugin that exports all payments to a CSV file | | [donations][donations] | A simple donations page to accept donations from the web | @@ -156,7 +154,6 @@ your environment. [sendinvoiceless]: https://github.com/lightningd/plugins/tree/master/sendinvoiceless [graphql]: https://github.com/nettijoe96/c-lightning-graphql [graphql-spec]: https://graphql.org/ -[autoreload]: https://github.com/lightningd/plugins/tree/master/autoreload [lightning-qt]: https://github.com/darosior/pylightning-qt [cpp-api]: https://github.com/darosior/lightningcpp [js-api]: https://github.com/darosior/clightningjs diff --git a/autoreload/README.md b/autoreload/README.md deleted file mode 100644 index 531b25f..0000000 --- a/autoreload/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Autoreload Plugin - -> You write the code, we do the rest! - -The autoreload plugin aims to help plugin developers to reload the plugin -inside of `lightningd` without having to restart the entire daemon. It watches -the executable file and will restart the plugin when a change is detected. - -The plugin adds the following command line options: - - - `--autoreload-plugin=path/to/plugin.py`: specifies which plugin you'd like - `autoreload` to manage. The syntax was chosen so you can just prefix any - existing `--plugin` with `autoreload` and start hacking. - -The plugin adds the following RPC methods: - - - `autoreload-restart`: triggers a manual restart of the plugin under - development, useful if you want to reload because of a dependency change or - you have a script that knows when a reload is needed. We will still - autoreload whenever we detect a change in the executable. - - -So in order to have a plugin `path/to/plugin.py` autoreload on changes you'll -need to call `lightningd` with the following arguments: - -```bash -lightningd --plugin=path/to/autoreload.py --autoreload-plugin=path/to/plugin.py -``` - -The first argument loads the `autoreload` plugin, and the second argument -tells `autoreload` to load, watch and restart the plugin under development. - -## Install - -This plugin relies on the `pylightning` and the `psutil` libraries. You should -be able to install these dependencies with `pip`: - -```bash -pip3 install -r requirements.txt -``` - -You might need to also specify the `--user` command line flag depending on -your environment. - -## How does it work? - -In order to hide the restarts from `lightningd` the autoreload plugin will -insert itself between `lightningd` and the plugin that is to be reloaded, -acting as an almost transparent proxy between the two. The only exception to -this are the two calls to register and initialize the plugin: - - - `getmanifest` is captured and the autoreload RPC methods and options are - injected, so `lightningd` knows about them and tells us when we get - initialized. - - `init` is captured and the options we returned above are stripped again, - otherwise we might upset the plugin under development. We also cache it, so - we can tell the plugin under development again when we restart it. - -Upon restarting we call `getmanifest` on the plugin under development just to -be safe (though you wouldn't do any initialization before being told so with -`init` would you? :wink:) and we then call `init` with the parameters we -cached when we were initialized ourselves. After this initialization dance is -complete, we will simply forward any calls directly to the plugin under -development and forward any output it produces to `lightningd`. - -We watch the modification time of the executable file and automatically -restart the plugin should it change. You can trigger this by changing the file -or even just `touch`ing it on the filesystem. You can also trigger a manual -restart using the `autoreload-restart` RPC method: - -```bash -lightning-cli autoreload-restart -``` - -## Caveats :construction: - - - Only one plugin can currently be managed by the autoreload plugin. - - Log lines will be prefixed with the autoreload plugin's name, not the - plugin under development. - - Since the registration of options, subscriptions, methods and hooks happens - during startup in `lightningd` you cannot currently add or remove any of - these without restarting `lightningd` itself. If you change them and have - autoreload restart the plugin under development you might experience - strange results. - diff --git a/autoreload/autoreload.py b/autoreload/autoreload.py deleted file mode 100755 index 9a12aec..0000000 --- a/autoreload/autoreload.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python3 -from pyln.client import Plugin -import json -import os -import psutil -import subprocess -import sys -import threading -import time - -try: - # C-lightning v0.7.2 - plugin = Plugin(dynamic=False) -except: - plugin = Plugin() - - -class ChildPlugin(object): - - def __init__(self, path, plugin): - self.path = path - self.plugin = plugin - self.status = 'stopped' - self.proc = None - self.iolock = threading.Lock() - self.decoder = json.JSONDecoder() - self.manifest = None - self.init = None - self.reader = None - - def watch(self): - last = os.path.getmtime(self.path) - while True: - time.sleep(1) - now = os.path.getmtime(self.path) - if last != now: - print("Detected a change in the child plugin, restarting...") - last = now - try: - self.restart() - except Exception as e: - self.plugin.log( - "Failed to start plugin, will wait for next change and try again:", - level='error' - ) - - def handle_init(self, request): - """Lightningd has sent us its first init message, clean and forward. - """ - params = request.params.copy() - - # These may have been added by the plugin framework and we won't be - # able to serialize them when forwarding, so delete them. - for key in ['plugin', 'request']: - if key in params: - del params[key] - - self.init = { - 'jsonrpc': '2.0', - 'method': request.method, - 'params': params, - 'id': request.id, - } - print("Forwarding", self.init) - - # Now remove any options that we registered on behalf of the child - # plugin. It'd not understand them if we forward them. - opts = self.init['params']['options'] - self.init['params']['options'] = {k: v for k, v in opts.items() if not k.startswith('autoreload')} - plugin.child.send(self.init) - print("Sent init to child plugin") - plugin.child.passthru() - - def _readobj(self, sock): - buff=b'' - while True: - try: - b = sock.readline() - buff += b - - if len(b) == 0: - return None - if b'}\n' not in buff: - continue - # Convert late to UTF-8 so glyphs split across recvs do not - # impact us - buff = buff.decode("UTF-8") - objs, len_used = self.decoder.raw_decode(buff) - buff = buff[len_used:].lstrip().encode("UTF-8") - return objs - except ValueError: - # Probably didn't read enough - buff = buff.lstrip().encode("UTF-8") - - def start(self): - assert(self.status == 'stopped') - try: - self.proc = subprocess.Popen([self.path], stdout=subprocess.PIPE, stdin=subprocess.PIPE) - self.status = 'started' - manifest = self.getmanifest() - return True - except Exception as e: - self.plugin.log(e, level='warn') - return False - - def stop(self): - assert(self.status == 'started') - self.proc.kill() - self.proc.wait() - reader = self.reader - if reader: - reader.join() - self.status = 'stopped' - - def restart(self): - print('Restarting child plugin') - self.stop() - self.start() - plugin.child.send(self.init) - print("Sent init to child plugin") - plugin.child.passthru() - - def getmanifest(self): - assert(self.status == 'started') - self.send({'jsonrpc': '2.0', 'id': 0, 'method': 'getmanifest', 'params': []}) - - while True: - msg = self._readobj(self.proc.stdout) - - if msg is None: - print("Child plugin does not seem to be sending valid JSON: {}".format(buff.strip())) - self.stop() - raise ValueError() - - if 'id' in msg and msg['id'] == 0: - - if self.manifest is not None and msg['result'] != self.manifest: - plugin.log( - "Plugin manifest changed between restarts: {new} != {old}\n\n" - "==> You need to restart c-lightning for these changes to be picked up! <==".format( - new=json.dumps(msg['result'], indent=True).replace("\"", "'"), - old=json.dumps(self.manifest, indent=True).replace("\"", "'") - ), level='warn') - raise ValueError() - self.manifest = msg['result'] - break - self.plugin._write_locked(msg) - return self.manifest - - def passthru(self): - # First read the init reply, and then we can switch to passthru - while True: - msg = self._readobj(self.proc.stdout) - if 'id' in msg and msg['id'] == self.init['id']: - break - self.plugin._write_locked(msg) - - def read_loop(): - while True: - line = self.proc.stdout.readline() - if line == b'': - break - self.plugin.stdout.buffer.write(line) - self.plugin.stdout.flush() - self.reader = None - print("Child plugin exited") - self.reader = threading.Thread(target=read_loop) - self.reader.daemon = True - self.reader.start() - - def send(self, msg): - self.proc.stdin.write(json.dumps(msg).encode('UTF-8')) - self.proc.stdin.write(b'\n\n') - self.proc.stdin.flush() - - def proxy_method(self, request, *args, **kwargs): - raw = { - 'jsonrpc': '2.0', - 'method': request.method, - 'params': request.params, - 'id': request.id, - } - self.send(raw) - - def proxy_subscription(self, request, *args, **kwargs): - raw = { - 'jsonrpc': '2.0', - 'method': request.method, - 'params': request.params, - } - self.send(raw) - - -@plugin.init() -def init(options, configuration, plugin, request): - #import remote_pdb; remote_pdb.set_trace() - if options['autoreload-plugin'] in ['null', None]: - print("Cannot run the autoreload plugin on its own, please specify --autoreload-plugin") - plugin.rpc.stop() - return - - watch_thread = threading.Thread(target=plugin.child.watch) - watch_thread.daemon = True - watch_thread.start() - plugin.child.handle_init(request) - - -def inject_manifest(plugin, manifest): - """Once we have the manifest from the child plugin, inject it into our own. - """ - for opt in manifest.get("options", []): - plugin.add_option(opt['name'], opt['default'], opt['description']) - - for m in manifest.get("rpcmethods", []): - plugin.add_method(m['name'], plugin.child.proxy_method, background=True) - - for s in manifest.get("subscriptions", []): - plugin.add_subscription(s, plugin.child.proxy_subscription) - - for h in manifest.get("hooks", []): - plugin.add_hook(h, plugin.child.proxy_method, background=True) - - -@plugin.method('autoreload-restart') -def restart(plugin): - """Manually triggers a restart of the plugin controlled by autoreload. - """ - child = plugin.child - child.restart() - - -# We can't rely on @plugin.init to tell us the plugin we need to watch and -# reload since we need to start it to pass through its manifest before we get -# any cli options. So we're doomed to get our parent cmdline and parse out the -# argument by hand. -parent = psutil.Process().parent() -while parent.name() != 'lightningd': - parent = parent.parent() -cmdline = parent.cmdline() -plugin.path = None - -prefix = '--autoreload-plugin=' - -for c in cmdline: - if c.startswith(prefix): - plugin.path = c[len(prefix):] - break - -if plugin.path: - plugin.child = ChildPlugin(plugin.path, plugin) - - # If we can't start on the first attempt we can't inject into the - # manifest, no point in continuing. - if not plugin.child.start(): - raise Exception("Could not start the plugin under development, can't continue") - - inject_manifest(plugin, plugin.child.manifest) - -else: - plugin.log("Could not locate the plugin to control: {cmdline}".format(cmdline=cmdline)) - sys.exit(1) - - -# Now we can run the actual plugin -plugin.add_option("autoreload-plugin", None, "Path to the plugin that we should be watching and reloading.") -plugin.run() diff --git a/autoreload/requirements.txt b/autoreload/requirements.txt deleted file mode 100644 index dd5ab51..0000000 --- a/autoreload/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pyln-client>=0.7.3 -psutil==5.6.6 diff --git a/autoreload/tests/dummy.py b/autoreload/tests/dummy.py deleted file mode 100755 index 93331b5..0000000 --- a/autoreload/tests/dummy.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 - -from pyln.client import Plugin - -plugin = Plugin() - -plugin.run() diff --git a/autoreload/tests/dummy2.py b/autoreload/tests/dummy2.py deleted file mode 100755 index f87205a..0000000 --- a/autoreload/tests/dummy2.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env python3 - -from pyln.client import Plugin - -plugin = Plugin() - -@plugin.method('dummy') -def on_dummy(): - pass - -plugin.run() diff --git a/autoreload/tests/test_autoreload.py b/autoreload/tests/test_autoreload.py deleted file mode 100644 index de09711..0000000 --- a/autoreload/tests/test_autoreload.py +++ /dev/null @@ -1,59 +0,0 @@ -from pyln.testing.fixtures import * -import shutil -import subprocess -import time -import os -import unittest - -def copy_plugin(src, directory, filename=None): - base = os.path.dirname(__file__) - src = os.path.join(base, src) - dst = os.path.join(directory, filename if filename is not None else os.path.basename(src)) - shutil.copy(src, dst) - shutil.copystat(src, dst) - return dst - - -def test_restart_on_change(node_factory, directory): - - # Copy the dummy plugin over - plugin = copy_plugin("dummy.py", directory, "plugin.py") - - opts = { - 'plugin': os.path.join(os.path.dirname(__file__), "..", 'autoreload.py'), - 'autoreload-plugin': plugin, - } - - l1 = node_factory.get_node(options=opts) - - subprocess.check_call(['touch', plugin]) - l1.daemon.wait_for_log(r'Detected a change in the child plugin, restarting') - - -@unittest.skipIf(True, "Doesn't work on travis yet") -def test_changing_manifest(node_factory, directory): - """Change the manifest in-between restarts. - - Adding an RPC method like the switch from dummy.py to dummy2.py does, - should result in an error while reloading the plugin. - - """ - - # Copy the dummy plugin over - plugin = copy_plugin("dummy.py", directory, "plugin.py") - plugin_path = os.path.join(os.path.dirname(__file__), "..", 'autoreload.py') - opts = { - 'plugin': os.path.join(os.path.dirname(__file__), "..", 'autoreload.py'), - 'autoreload-plugin': plugin, - } - - l1 = node_factory.get_node(options=opts, allow_broken_log=True) - - - plugin = copy_plugin("dummy2.py", directory, "plugin.py") - time.sleep(10) - subprocess.check_call(['touch', plugin_path]) - time.sleep(10) - subprocess.check_call(['touch', plugin_path]) - l1.daemon.wait_for_log(r'Detected a change in the child plugin, restarting') - l1.daemon.wait_for_log(r'You need to restart c-lightning')