pytest: Remove auto-proxying in favor of a per-node btc proxy

tests/btcproxy.py

@@ -3,7 +3,6 @@
 from flask import Flask, request
 from bitcoin.rpc import JSONRPCError
 from bitcoin.rpc import RawProxy as BitcoinProxy
-from utils import BitcoinD
 from cheroot.wsgi import Server
 from cheroot.wsgi import PathInfoDispatcher
 
@@ -24,16 +23,17 @@ class DecimalEncoder(json.JSONEncoder):
         return super(DecimalEncoder, self).default(o)
 
 
-class ProxiedBitcoinD(BitcoinD):
+class BitcoinRpcProxy(object):
-    def __init__(self, bitcoin_dir, proxyport=0):
+    def __init__(self, bitcoind, rpcport=0):
-        BitcoinD.__init__(self, bitcoin_dir, rpcport=None)
         self.app = Flask("BitcoindProxy")
         self.app.add_url_rule("/", "API entrypoint", self.proxy, methods=['POST'])
-        self.proxyport = proxyport
+        self.rpcport = rpcport
         self.mocks = {}
+        self.bitcoind = bitcoind
+        self.request_count = 0
 
     def _handle_request(self, r):
-        conf_file = os.path.join(self.bitcoin_dir, 'bitcoin.conf')
+        conf_file = os.path.join(self.bitcoind.bitcoin_dir, 'bitcoin.conf')
         brpc = BitcoinProxy(btc_conf_file=conf_file)
         method = r['method']
 
@@ -55,6 +55,7 @@ class ProxiedBitcoinD(BitcoinD):
                 "error": e.error,
                 "id": r['id']
             }
+        self.request_count += 1
         return reply
 
     def proxy(self):
@@ -71,27 +72,23 @@ class ProxiedBitcoinD(BitcoinD):
 
     def start(self):
         d = PathInfoDispatcher({'/': self.app})
-        self.server = Server(('0.0.0.0', self.proxyport), d)
+        self.server = Server(('0.0.0.0', self.rpcport), d)
         self.proxy_thread = threading.Thread(target=self.server.start)
         self.proxy_thread.daemon = True
         self.proxy_thread.start()
-        BitcoinD.start(self)
 
         # Now that bitcoind is running on the real rpcport, let's tell all
         # future callers to talk to the proxyport. We use the bind_addr as a
         # signal that the port is bound and accepting connections.
         while self.server.bind_addr[1] == 0:
             pass
-        self.proxiedport = self.rpcport
         self.rpcport = self.server.bind_addr[1]
-        logging.debug("bitcoind reverse proxy listening on {}, forwarding to {}".format(
+        logging.debug("BitcoinRpcProxy proxying incoming port {} to {}".format(self.rpcport, self.bitcoind.rpcport))
-            self.rpcport, self.proxiedport
-        ))
 
     def stop(self):
-        BitcoinD.stop(self)
         self.server.stop()
         self.proxy_thread.join()
+        logging.debug("BitcoinRpcProxy shut down after processing {} requests".format(self.request_count))
 
     def mock_rpc(self, method, response=None):
         """Mock the response to a future RPC call of @method
@@ -105,11 +102,3 @@ class ProxiedBitcoinD(BitcoinD):
             self.mocks[method] = response
         elif method in self.mocks:
             del self.mocks[method]
-
-
-# The main entrypoint is mainly used to test the proxy. It is not used during
-# lightningd testing.
-if __name__ == "__main__":
-    p = ProxiedBitcoinD(bitcoin_dir='/tmp/bitcoind-test/', proxyport=5000)
-    p.start()
-    p.proxy_thread.join()

tests/fixtures.py

@@ -1,6 +1,5 @@
 from concurrent import futures
-from btcproxy import ProxiedBitcoinD
+from utils import NodeFactory, BitcoinD
-from utils import NodeFactory
 
 import logging
 import os
@@ -69,7 +68,7 @@ def test_name(request):
 
 @pytest.fixture
 def bitcoind(directory):
-    bitcoind = ProxiedBitcoinD(bitcoin_dir=directory)
+    bitcoind = BitcoinD(bitcoin_dir=directory)
     try:
         bitcoind.start()
     except Exception:

tests/utils.py

@@ -10,12 +10,12 @@ import subprocess
 import threading
 import time
 
+from btcproxy import BitcoinRpcProxy
 from bitcoin.rpc import RawProxy as BitcoinProxy
 from decimal import Decimal
 from ephemeral_port_reserve import reserve
 from lightning import LightningRpc
 
-
 BITCOIND_CONFIG = {
     "regtest": 1,
     "rpcuser": "rpcuser",
@@ -287,13 +287,15 @@ class BitcoinD(TailableProc):
 
 
 class LightningD(TailableProc):
-    def __init__(self, lightning_dir, bitcoin_dir, port=9735, random_hsm=False, node_id=0, bitcoin_rpcport=18332):
+    def __init__(self, lightning_dir, bitcoind, port=9735, random_hsm=False, node_id=0):
         TailableProc.__init__(self, lightning_dir)
         self.lightning_dir = lightning_dir
         self.port = port
         self.cmd_prefix = []
         self.disconnect_file = None
 
+        self.rpcproxy = BitcoinRpcProxy(bitcoind)
+
         self.opts = LIGHTNINGD_CONFIG.copy()
         opts = {
             'lightning-dir': lightning_dir,
@@ -301,7 +303,6 @@ class LightningD(TailableProc):
             'allow-deprecated-apis': 'false',
             'network': 'regtest',
             'ignore-fee-limits': 'false',
-            'bitcoin-rpcport': bitcoin_rpcport,
             'bitcoin-rpcuser': BITCOIND_CONFIG['rpcuser'],
             'bitcoin-rpcpassword': BITCOIND_CONFIG['rpcpassword'],
         }
@@ -344,6 +345,9 @@ class LightningD(TailableProc):
         return self.cmd_prefix + ['lightningd/lightningd'] + opts
 
     def start(self):
+        self.rpcproxy.start()
+
+        self.opts['bitcoin-rpcport'] = self.rpcproxy.rpcport
         TailableProc.start(self)
         self.wait_for_log("Server started with public key")
         logging.info("LightningD started")
@@ -355,6 +359,7 @@ class LightningD(TailableProc):
         not return before the timeout triggers.
         """
         self.proc.wait(timeout)
+        self.rpcproxy.stop()
         return self.proc.returncode
 
 
@@ -690,9 +695,8 @@ class NodeFactory(object):
 
         socket_path = os.path.join(lightning_dir, "lightning-rpc").format(node_id)
         daemon = LightningD(
-            lightning_dir, self.bitcoind.bitcoin_dir,
+            lightning_dir, self.bitcoind,
-            port=port, random_hsm=random_hsm, node_id=node_id,
+            port=port, random_hsm=random_hsm, node_id=node_id
-            bitcoin_rpcport=self.bitcoind.rpcport
         )
         # If we have a disconnect string, dump it to a file for daemon.
         if disconnect: