pytest: Add an RPC proxy inbetween bitcoind and bitcoin-cli

This is a simple reverse proxy that `bitcoin-cli` can talk to when invoked by
`lightningd`. It allows us to trace `bitcoin-cli` calls, and intercept calls to
mock the replies, better than the current bash-script based method.

tests/btcproxy.py

@@ -0,0 +1,81 @@
+""" A bitcoind proxy that allows instrumentation and canned responses
+"""
+from flask import Flask, request
+from bitcoin.rpc import JSONRPCError
+from bitcoin.rpc import RawProxy as BitcoinProxy
+from utils import BitcoinD
+from cheroot.wsgi import Server
+from cheroot.wsgi import PathInfoDispatcher
+
+import decimal
+import json
+import logging
+import os
+import threading
+
+
+class DecimalEncoder(json.JSONEncoder):
+    """By default json.dumps does not handle Decimals correctly, so we override it's handling
+    """
+    def default(self, o):
+        if isinstance(o, decimal.Decimal):
+            return str(o)
+        return super(DecimalEncoder, self).default(o)
+
+
+class ProxiedBitcoinD(BitcoinD):
+    def __init__(self, bitcoin_dir, proxyport=0):
+        BitcoinD.__init__(self, bitcoin_dir, rpcport=None)
+        self.app = Flask("BitcoindProxy")
+        self.app.add_url_rule("/", "API entrypoint", self.proxy, methods=['POST'])
+        self.proxyport = proxyport
+
+    def proxy(self):
+        r = json.loads(request.data.decode('ASCII'))
+        conf_file = os.path.join(self.bitcoin_dir, 'bitcoin.conf')
+        brpc = BitcoinProxy(btc_conf_file=conf_file)
+
+        try:
+            reply = {
+                "result": brpc._call(r['method'], *r['params']),
+                "error": None,
+                "id": r['id']
+            }
+        except JSONRPCError as e:
+            reply = {
+                "error": e.error,
+                "id": r['id']
+            }
+        return json.dumps(reply, cls=DecimalEncoder)
+
+    def start(self):
+        d = PathInfoDispatcher({'/': self.app})
+        self.server = Server(('0.0.0.0', self.proxyport), d)
+        self.proxy_thread = threading.Thread(target=self.server.start)
+        self.proxy_thread.daemon = True
+        self.proxy_thread.start()
+        BitcoinD.start(self)
+
+        # Now that bitcoind is running on the real rpcport, let's tell all
+        # future callers to talk to the proxyport. We use the bind_addr as a
+        # signal that the port is bound and accepting connections.
+        while self.server.bind_addr[1] == 0:
+            pass
+        self.proxiedport = self.rpcport
+        self.rpcport = self.server.bind_addr[1]
+        logging.debug("bitcoind reverse proxy listening on {}, forwarding to {}".format(
+            self.rpcport, self.proxiedport
+        ))
+
+    def stop(self):
+        BitcoinD.stop(self)
+        self.server.stop()
+        self.proxy_thread.join()
+
+
+# The main entrypoint is mainly used to test the proxy. It is not used during
+# lightningd testing.
+if __name__ == "__main__":
+    p = ProxiedBitcoinD(bitcoin_dir='/tmp/bitcoind-test/', proxyport=5000)
+    p.start()
+    p.proxy_thread.join()

tests/fixtures.py

@@ -1,4 +1,5 @@
 from concurrent import futures
+from btcproxy import ProxiedBitcoinD
 from utils import NodeFactory
 
 import logging
@@ -8,7 +9,6 @@ import re
 import shutil
 import sys
 import tempfile
-import utils
 
 
 with open('config.vars') as configfile:
@@ -69,7 +69,7 @@ def test_name(request):
 
 @pytest.fixture
 def bitcoind(directory):
-    bitcoind = utils.BitcoinD(bitcoin_dir=directory, rpcport=None)
+    bitcoind = ProxiedBitcoinD(bitcoin_dir=directory)
     try:
         bitcoind.start()
     except Exception:

tests/requirements.txt

@@ -3,3 +3,5 @@ ephemeral-port-reserve==1.1.0
 pytest-forked==0.2
 pytest-xdist==1.22.2
 flaky==3.4.0
+CherryPy==17.3.0
+Flask==1.0.2

tests/test_misc.py

@@ -570,8 +570,6 @@ def test_listconfigs(node_factory, bitcoind):
 
     configs = l1.rpc.listconfigs()
     # See utils.py
-    assert configs['bitcoin-datadir'] == bitcoind.bitcoin_dir
-    assert configs['lightning-dir'] == l1.daemon.lightning_dir
     assert configs['allow-deprecated-apis'] is False
     assert configs['network'] == 'regtest'
     assert configs['ignore-fee-limits'] is False

tests/utils.py

@@ -287,7 +287,7 @@ class BitcoinD(TailableProc):
 
 
 class LightningD(TailableProc):
-    def __init__(self, lightning_dir, bitcoin_dir, port=9735, random_hsm=False, node_id=0):
+    def __init__(self, lightning_dir, bitcoin_dir, port=9735, random_hsm=False, node_id=0, bitcoin_rpcport=18332):
         TailableProc.__init__(self, lightning_dir)
         self.lightning_dir = lightning_dir
         self.port = port
@@ -296,12 +296,14 @@ class LightningD(TailableProc):
 
         self.opts = LIGHTNINGD_CONFIG.copy()
         opts = {
-            'bitcoin-datadir': bitcoin_dir,
             'lightning-dir': lightning_dir,
             'addr': '127.0.0.1:{}'.format(port),
             'allow-deprecated-apis': 'false',
             'network': 'regtest',
             'ignore-fee-limits': 'false',
+            'bitcoin-rpcport': bitcoin_rpcport,
+            'bitcoin-rpcuser': BITCOIND_CONFIG['rpcuser'],
+            'bitcoin-rpcpassword': BITCOIND_CONFIG['rpcpassword'],
         }
 
         for k, v in opts.items():
@@ -689,7 +691,8 @@ class NodeFactory(object):
         socket_path = os.path.join(lightning_dir, "lightning-rpc").format(node_id)
         daemon = LightningD(
             lightning_dir, self.bitcoind.bitcoin_dir,
-            port=port, random_hsm=random_hsm, node_id=node_id
+            port=port, random_hsm=random_hsm, node_id=node_id,
+            bitcoin_rpcport=self.bitcoind.rpcport
         )
         # If we have a disconnect string, dump it to a file for daemon.
         if disconnect: