Updates teos to work with the new conf file format and redefines how objects are built

Configuration parameters are now load from a .conf file (see template.conf as an example).

The code has 3 config levels: default (teos/__init__.py), config file (<data_dir>/teos.conf) and command line.
Most of the config parameters are only modificable trough the config file for now.
The priority order is: command line, config file, default.

Objects that need config parameters are now built in teosd instead of inside other classes. Therefore teosd acts like a factory and config parameters don't have to be passed around between objects. This also implies that a lot of the object creation logic has been changed.

This should simplify unit testing.

common/tools.py

@@ -54,42 +54,9 @@ def setup_data_folder(data_folder):
         os.makedirs(data_folder, exist_ok=True)
 
 
-def check_conf_fields(conf_fields):
-    """
-    Checks that the provided configuration field have the right type.
-
-    Args:
-        conf_fields (:obj:`dict`): a dictionary populated with the configuration file params and the expected types.
-            The format is as follows:
-
-            {"field0": {"value": value_from_conf_file, "type": expected_type, ...}}
-
-    Returns:
-        :obj:`dict`: A dictionary with the same keys as the provided one, but containing only the "value" field as value
-        if the provided ``conf_fields`` where correct.
-
-    Raises:
-        ValueError: If any of the dictionary elements does not have the expected type
-    """
-
-    conf_dict = {}
-
-    for field in conf_fields:
-        value = conf_fields[field]["value"]
-        correct_type = conf_fields[field]["type"]
-
-        if (value is not None) and isinstance(value, correct_type):
-            conf_dict[field] = value
-        else:
-            err_msg = "{} variable in config is of the wrong type".format(field)
-            raise ValueError(err_msg)
-
-    return conf_dict
-
-
 def extend_paths(base_path, config_fields):
     """
-    Extends the relative paths of a given ``config_fields`` dictionary with a diven ``base_path``.
+    Extends the relative paths of a given ``config_fields`` dictionary with a given ``base_path``.
 
     Paths in the config file are based on DATA_PATH, this method extends them so they are all absolute.
 
@@ -98,16 +65,12 @@ def extend_paths(base_path, config_fields):
         config_fields (:obj:`dict`): a dictionary of configuration fields containing a ``path`` flag, as follows:
             {"field0": {"value": value_from_conf_file, "path": True, ...}}
 
-    Returns:
-        :obj:`dict`: A ``config_fields`` with the flagged paths updated.
     """
 
     for key, field in config_fields.items():
         if field.get("path") is True:
             config_fields[key]["value"] = base_path + config_fields[key]["value"]
 
-    return config_fields
-
 
 def setup_logging(log_file_path, log_name_prefix):
     """

teos/__init__.py

@@ -1,38 +1,25 @@
 import os
-import teos.conf as conf
-from common.tools import check_conf_fields, setup_logging, extend_paths, setup_data_folder
 from teos.utils.auth_proxy import AuthServiceProxy
 
 HOST = "localhost"
 PORT = 9814
+DATA_DIR = os.path.expanduser("~/.teos/")
 LOG_PREFIX = "teos"
 
-# Load config fields
+# Default conf fields
-conf_fields = {
+DEFAULT_CONF = {
-    "BTC_RPC_USER": {"value": conf.BTC_RPC_USER, "type": str},
+    "BTC_RPC_USER": {"value": "user", "type": str},
-    "BTC_RPC_PASSWD": {"value": conf.BTC_RPC_PASSWD, "type": str},
+    "BTC_RPC_PASSWD": {"value": "passwd", "type": str},
-    "BTC_RPC_HOST": {"value": conf.BTC_RPC_HOST, "type": str},
+    "BTC_RPC_CONNECT": {"value": "127.0.0.1", "type": str},
-    "BTC_RPC_PORT": {"value": conf.BTC_RPC_PORT, "type": int},
+    "BTC_RPC_PORT": {"value": 8332, "type": int},
-    "BTC_NETWORK": {"value": conf.BTC_NETWORK, "type": str},
+    "BTC_NETWORK": {"value": "mainnet", "type": str},
-    "FEED_PROTOCOL": {"value": conf.FEED_PROTOCOL, "type": str},
+    "FEED_PROTOCOL": {"value": "tcp", "type": str},
-    "FEED_ADDR": {"value": conf.FEED_ADDR, "type": str},
+    "FEED_CONNECT": {"value": "127.0.0.1", "type": str},
-    "FEED_PORT": {"value": conf.FEED_PORT, "type": int},
+    "FEED_PORT": {"value": 28332, "type": int},
-    "DATA_FOLDER": {"value": conf.DATA_FOLDER, "type": str},
+    "MAX_APPOINTMENTS": {"value": 100, "type": int},
-    "MAX_APPOINTMENTS": {"value": conf.MAX_APPOINTMENTS, "type": int},
+    "EXPIRY_DELTA": {"value": 6, "type": int},
-    "EXPIRY_DELTA": {"value": conf.EXPIRY_DELTA, "type": int},
+    "MIN_TO_SELF_DELAY": {"value": 20, "type": int},
-    "MIN_TO_SELF_DELAY": {"value": conf.MIN_TO_SELF_DELAY, "type": int},
+    "LOG_FILE": {"value": "teos.log", "type": str, "path": True},
-    "SERVER_LOG_FILE": {"value": conf.SERVER_LOG_FILE, "type": str, "path": True},
+    "TEOS_SECRET_KEY": {"value": "teos_sk.der", "type": str, "path": True},
-    "TEOS_SECRET_KEY": {"value": conf.TEOS_SECRET_KEY, "type": str, "path": True},
+    "DB_PATH": {"value": "appointments", "type": str, "path": True},
-    "DB_PATH": {"value": conf.DB_PATH, "type": str, "path": True},
 }
-
-# Expand user (~) if found and check fields are correct
-conf_fields["DATA_FOLDER"]["value"] = os.path.expanduser(conf_fields["DATA_FOLDER"]["value"])
-# Extend relative paths
-conf_fields = extend_paths(conf_fields["DATA_FOLDER"]["value"], conf_fields)
-
-# Sanity check fields and build config dictionary
-config = check_conf_fields(conf_fields)
-
-setup_data_folder(config.get("DATA_FOLDER"))
-setup_logging(config.get("SERVER_LOG_FILE"), LOG_PREFIX)

teos/api.py

@@ -5,7 +5,6 @@ from flask import Flask, request, abort, jsonify
 
 from teos import HOST, PORT, LOG_PREFIX
 from common.logger import Logger
-from teos.inspector import Inspector
 from common.appointment import Appointment
 
 from common.constants import HTTP_OK, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, LOCATOR_LEN_HEX
@@ -17,9 +16,18 @@ logger = Logger(actor="API", log_name_prefix=LOG_PREFIX)
 
 
 class API:
-    def __init__(self, watcher, config):
+    """
+    The :class:`API` is in charge of the interface between the user and the tower. It handles and server user requests.
+
+    Args:
+        inspector (:obj:`Inspector <teos.inspector.Inspector>`): an ``Inspector`` instance to check the correctness of
+            the received data.
+        watcher (:obj:`Watcher <teos.watcher.Watcher>`): a ``Watcher`` instance to pass the requests to.
+    """
+
+    def __init__(self, inspector, watcher):
+        self.inspector = inspector
         self.watcher = watcher
-        self.config = config
 
     def add_appointment(self):
         """
@@ -48,8 +56,7 @@ class API:
         if request.is_json:
             # Check content type once if properly defined
             request_data = json.loads(request.get_json())
-            inspector = Inspector(self.config)
+            appointment = self.inspector.inspect(
-            appointment = inspector.inspect(
                 request_data.get("appointment"), request_data.get("signature"), request_data.get("public_key")
             )
 

teos/block_processor.py

@@ -11,10 +11,16 @@ class BlockProcessor:
     """
     The :class:`BlockProcessor` contains methods related to the blockchain. Most of its methods require communication
     with ``bitcoind``.
+
+    Args:
+        btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind
+            (rpc user, rpc passwd, host and port)
     """
 
-    @staticmethod
+    def __init__(self, btc_connect_params):
-    def get_block(block_hash):
+        self.btc_connect_params = btc_connect_params
+
+    def get_block(self, block_hash):
         """
         Gives a block given a block hash by querying ``bitcoind``.
 
@@ -28,7 +34,7 @@ class BlockProcessor:
         """
 
         try:
-            block = bitcoin_cli().getblock(block_hash)
+            block = bitcoin_cli(self.btc_connect_params).getblock(block_hash)
 
         except JSONRPCException as e:
             block = None
@@ -36,8 +42,7 @@ class BlockProcessor:
 
         return block
 
-    @staticmethod
+    def get_best_block_hash(self):
-    def get_best_block_hash():
         """
         Returns the hash of the current best chain tip.
 
@@ -48,7 +53,7 @@ class BlockProcessor:
         """
 
         try:
-            block_hash = bitcoin_cli().getbestblockhash()
+            block_hash = bitcoin_cli(self.btc_connect_params).getbestblockhash()
 
         except JSONRPCException as e:
             block_hash = None
@@ -56,8 +61,7 @@ class BlockProcessor:
 
         return block_hash
 
-    @staticmethod
+    def get_block_count(self):
-    def get_block_count():
         """
         Returns the block height of the best chain.
 
@@ -68,7 +72,7 @@ class BlockProcessor:
         """
 
         try:
-            block_count = bitcoin_cli().getblockcount()
+            block_count = bitcoin_cli(self.btc_connect_params).getblockcount()
 
         except JSONRPCException as e:
             block_count = None
@@ -76,8 +80,7 @@ class BlockProcessor:
 
         return block_count
 
-    @staticmethod
+    def decode_raw_transaction(self, raw_tx):
-    def decode_raw_transaction(raw_tx):
         """
         Deserializes a given raw transaction (hex encoded) and builds a dictionary representing it with all the
         associated metadata given by ``bitcoind`` (e.g. confirmation count).
@@ -92,7 +95,7 @@ class BlockProcessor:
         """
 
         try:
-            tx = bitcoin_cli().decoderawtransaction(raw_tx)
+            tx = bitcoin_cli(self.btc_connect_params).decoderawtransaction(raw_tx)
 
         except JSONRPCException as e:
             tx = None
@@ -100,8 +103,7 @@ class BlockProcessor:
 
         return tx
 
-    @staticmethod
+    def get_distance_to_tip(self, target_block_hash):
-    def get_distance_to_tip(target_block_hash):
         """
         Compute the distance between a given block hash and the best chain tip.
 
@@ -117,10 +119,10 @@ class BlockProcessor:
 
         distance = None
 
-        chain_tip = BlockProcessor.get_best_block_hash()
+        chain_tip = self.get_best_block_hash()
-        chain_tip_height = BlockProcessor.get_block(chain_tip).get("height")
+        chain_tip_height = self.get_block(chain_tip).get("height")
 
-        target_block = BlockProcessor.get_block(target_block_hash)
+        target_block = self.get_block(target_block_hash)
 
         if target_block is not None:
             target_block_height = target_block.get("height")
@@ -129,8 +131,7 @@ class BlockProcessor:
 
         return distance
 
-    @staticmethod
+    def get_missed_blocks(self, last_know_block_hash):
-    def get_missed_blocks(last_know_block_hash):
         """
         Compute the blocks between the current best chain tip and a given block hash (``last_know_block_hash``).
 
@@ -144,19 +145,18 @@ class BlockProcessor:
             child of ``last_know_block_hash``.
         """
 
-        current_block_hash = BlockProcessor.get_best_block_hash()
+        current_block_hash = self.get_best_block_hash()
         missed_blocks = []
 
         while current_block_hash != last_know_block_hash and current_block_hash is not None:
             missed_blocks.append(current_block_hash)
 
-            current_block = BlockProcessor.get_block(current_block_hash)
+            current_block = self.get_block(current_block_hash)
             current_block_hash = current_block.get("previousblockhash")
 
         return missed_blocks[::-1]
 
-    @staticmethod
+    def is_block_in_best_chain(self, block_hash):
-    def is_block_in_best_chain(block_hash):
         """
         Checks whether or not a given block is on the best chain. Blocks are identified by block_hash.
 
@@ -173,7 +173,7 @@ class BlockProcessor:
             KeyError: If the block cannot be found in the blockchain.
         """
 
-        block = BlockProcessor.get_block(block_hash)
+        block = self.get_block(block_hash)
 
         if block is None:
             # This should never happen as long as we are using the same node, since bitcoind never drops orphan blocks
@@ -185,8 +185,7 @@ class BlockProcessor:
         else:
             return False
 
-    @staticmethod
+    def find_last_common_ancestor(self, last_known_block_hash):
-    def find_last_common_ancestor(last_known_block_hash):
         """
         Finds the last common ancestor between the current best chain tip and the last block known by us (older block).
 
@@ -204,8 +203,8 @@ class BlockProcessor:
         target_block_hash = last_known_block_hash
         dropped_txs = []
 
-        while not BlockProcessor.is_block_in_best_chain(target_block_hash):
+        while not self.is_block_in_best_chain(target_block_hash):
-            block = BlockProcessor.get_block(target_block_hash)
+            block = self.get_block(target_block_hash)
             dropped_txs.extend(block.get("tx"))
             target_block_hash = block.get("previousblockhash")
 

teos/builder.py

@@ -1,6 +1,6 @@
 class Builder:
     """
-    The :class:`Builder` class is in charge or reconstructing data loaded from the database and build the data
+    The :class:`Builder` class is in charge of reconstructing data loaded from the database and build the data
     structures of the :obj:`Watcher <teos.watcher.Watcher>` and the :obj:`Responder <teos.responder.Responder>`.
     """
 

teos/carrier.py

@@ -39,13 +39,18 @@ class Carrier:
     The :class:`Carrier` is the class in charge of interacting with ``bitcoind`` to send/get transactions. It uses
     :obj:`Receipt` objects to report about the sending outcome.
 
+    Args:
+        btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind
+            (rpc user, rpc passwd, host and port)
+
     Attributes:
         issued_receipts (:obj:`dict`): a dictionary of issued receipts to prevent resending the same transaction over
             and over. It should periodically be reset to prevent it from growing unbounded.
 
     """
 
-    def __init__(self):
+    def __init__(self, btc_connect_params):
+        self.btc_connect_params = btc_connect_params
         self.issued_receipts = {}
 
     # NOTCOVERED
@@ -69,7 +74,7 @@ class Carrier:
 
         try:
             logger.info("Pushing transaction to the network", txid=txid, rawtx=rawtx)
-            bitcoin_cli().sendrawtransaction(rawtx)
+            bitcoin_cli(self.btc_connect_params).sendrawtransaction(rawtx)
 
             receipt = Receipt(delivered=True)
 
@@ -119,8 +124,7 @@ class Carrier:
 
         return receipt
 
-    @staticmethod
+    def get_transaction(self, txid):
-    def get_transaction(txid):
         """
         Queries transaction data to ``bitcoind`` given a transaction id.
 
@@ -134,7 +138,7 @@ class Carrier:
         """
 
         try:
-            tx_info = bitcoin_cli().getrawtransaction(txid, 1)
+            tx_info = bitcoin_cli(self.btc_connect_params).getrawtransaction(txid, 1)
 
         except JSONRPCException as e:
             tx_info = None

teos/chain_monitor.py

@@ -4,8 +4,6 @@ from threading import Thread, Event, Condition
 
 from teos import LOG_PREFIX
 from common.logger import Logger
-from teos.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT, POLLING_DELTA, BLOCK_WINDOW_SIZE
-from teos.block_processor import BlockProcessor
 
 logger = Logger(actor="ChainMonitor", log_name_prefix=LOG_PREFIX)
 
@@ -22,6 +20,8 @@ class ChainMonitor:
     Args:
         watcher_queue (:obj:`Queue`): the queue to be used to send blocks hashes to the ``Watcher``.
         responder_queue (:obj:`Queue`): the queue to be used to send blocks hashes to the ``Responder``.
+        block_processor (:obj:`BlockProcessor <teos.block_processor.BlockProcessor>`): a blockProcessor instance.
+        bitcoind_feed_params (:obj:`dict`): a dict with the feed (ZMQ) connection parameters.
 
     Attributes:
         best_tip (:obj:`str`): a block hash representing the current best tip.
@@ -34,9 +34,13 @@ class ChainMonitor:
         watcher_queue (:obj:`Queue`): a queue to send new best tips to the :obj:`Watcher <teos.watcher.Watcher>`.
         responder_queue (:obj:`Queue`): a queue to send new best tips to the
             :obj:`Responder <teos.responder.Responder>`.
+
+        polling_delta (:obj:`int`): time between polls (in seconds).
+        max_block_window_size (:obj:`int`): max size of last_tips.
+        block_processor (:obj:`BlockProcessor <teos.block_processor.BlockProcessor>`): a blockProcessor instance.
     """
 
-    def __init__(self, watcher_queue, responder_queue):
+    def __init__(self, watcher_queue, responder_queue, block_processor, bitcoind_feed_params):
         self.best_tip = None
         self.last_tips = []
         self.terminate = False
@@ -48,11 +52,22 @@ class ChainMonitor:
         self.zmqSubSocket = self.zmqContext.socket(zmq.SUB)
         self.zmqSubSocket.setsockopt(zmq.RCVHWM, 0)
         self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock")
-        self.zmqSubSocket.connect("%s://%s:%s" % (FEED_PROTOCOL, FEED_ADDR, FEED_PORT))
+        self.zmqSubSocket.connect(
+            "%s://%s:%s"
+            % (
+                bitcoind_feed_params.get("FEED_PROTOCOL"),
+                bitcoind_feed_params.get("FEED_CONNECT"),
+                bitcoind_feed_params.get("FEED_PORT"),
+            )
+        )
 
         self.watcher_queue = watcher_queue
         self.responder_queue = responder_queue
 
+        self.polling_delta = 60
+        self.max_block_window_size = 10
+        self.block_processor = block_processor
+
     def notify_subscribers(self, block_hash):
         """
         Notifies the subscribers (``Watcher`` and ``Responder``) about a new block. It does so by putting the hash in
@@ -66,14 +81,13 @@ class ChainMonitor:
         self.watcher_queue.put(block_hash)
         self.responder_queue.put(block_hash)
 
-    def update_state(self, block_hash, max_block_window_size=BLOCK_WINDOW_SIZE):
+    def update_state(self, block_hash):
         """
         Updates the state of the ``ChainMonitor``. The state is represented as the ``best_tip`` and the list of
         ``last_tips``. ``last_tips`` is bounded to ``max_block_window_size``.
 
         Args:
             block_hash (:obj:`block_hash`): the new best tip.
-            max_block_window_size (:obj:`int`): the maximum length of the ``last_tips`` list.
 
         Returns:
             (:obj:`bool`): ``True`` is the state was successfully updated, ``False`` otherwise.
@@ -83,7 +97,7 @@ class ChainMonitor:
             self.last_tips.append(self.best_tip)
             self.best_tip = block_hash
 
-            if len(self.last_tips) > max_block_window_size:
+            if len(self.last_tips) > self.max_block_window_size:
                 self.last_tips.pop(0)
 
             return True
@@ -91,22 +105,19 @@ class ChainMonitor:
         else:
             return False
 
-    def monitor_chain_polling(self, polling_delta=POLLING_DELTA):
+    def monitor_chain_polling(self):
         """
         Monitors ``bitcoind`` via polling. Once the method is fired, it keeps monitoring as long as ``terminate`` is not
         set. Polling is performed once every ``polling_delta`` seconds. If a new best tip if found, the shared lock is
         acquired, the state is updated and the subscribers are notified, and finally the lock is released.
-
-        Args:
-             polling_delta (:obj:`int`): the time delta between polls.
         """
 
         while not self.terminate:
-            self.check_tip.wait(timeout=polling_delta)
+            self.check_tip.wait(timeout=self.polling_delta)
 
             # Terminate could have been set while the thread was blocked in wait
             if not self.terminate:
-                current_tip = BlockProcessor.get_best_block_hash()
+                current_tip = self.block_processor.get_best_block_hash()
 
                 self.lock.acquire()
                 if self.update_state(current_tip):
@@ -138,16 +149,13 @@ class ChainMonitor:
                         logger.info("New block received via zmq", block_hash=block_hash)
                     self.lock.release()
 
-    def monitor_chain(self, polling_delta=POLLING_DELTA):
+    def monitor_chain(self):
         """
         Main :class:`ChainMonitor` method. It initializes the ``best_tip`` to the current one (by querying the
         :obj:`BlockProcessor <teos.block_processor.BlockProcessor>`) and creates two threads, one per each monitoring
         approach (``zmq`` and ``polling``).
-
-        Args:
-             polling_delta (:obj:`int`): the time delta between polls by the ``monitor_chain_polling`` thread.
         """
 
-        self.best_tip = BlockProcessor.get_best_block_hash()
+        self.best_tip = self.block_processor.get_best_block_hash()
-        Thread(target=self.monitor_chain_polling, daemon=True, kwargs={"polling_delta": polling_delta}).start()
+        Thread(target=self.monitor_chain_polling, daemon=True).start()
         Thread(target=self.monitor_chain_zmq, daemon=True).start()

teos/cleaner.py

@@ -81,7 +81,7 @@ class Cleaner:
                 logger.error("Some UUIDs not found in the db", locator=locator, all_uuids=uuids)
 
         else:
-            logger.error("Locator map not found in the db", uuid=locator)
+            logger.error("Locator map not found in the db", locator=locator)
 
     @staticmethod
     def delete_expired_appointments(expired_appointments, appointments, locator_uuid_map, db_manager):

teos/conf.py

@@ -1,26 +0,0 @@
-# bitcoind
-BTC_RPC_USER = "user"
-BTC_RPC_PASSWD = "passwd"
-BTC_RPC_HOST = "localhost"
-BTC_RPC_PORT = 18443
-BTC_NETWORK = "regtest"
-
-# ZMQ
-FEED_PROTOCOL = "tcp"
-FEED_ADDR = "127.0.0.1"
-FEED_PORT = 28332
-
-# TEOS
-DATA_FOLDER = "~/.teos/"
-MAX_APPOINTMENTS = 100
-EXPIRY_DELTA = 6
-MIN_TO_SELF_DELAY = 20
-SERVER_LOG_FILE = "teos.log"
-TEOS_SECRET_KEY = "teos_sk.der"
-
-# CHAIN MONITOR
-POLLING_DELTA = 60
-BLOCK_WINDOW_SIZE = 10
-
-# LEVELDB
-DB_PATH = "appointments"

teos/inspector.py

@@ -8,7 +8,6 @@ from common.cryptographer import Cryptographer, PublicKey
 from teos import errors, LOG_PREFIX
 from common.logger import Logger
 from common.appointment import Appointment
-from teos.block_processor import BlockProcessor
 
 logger = Logger(actor="Inspector", log_name_prefix=LOG_PREFIX)
 common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX)
@@ -26,10 +25,16 @@ ENCRYPTED_BLOB_MAX_SIZE_HEX = 2 * 2048
 class Inspector:
     """
     The :class:`Inspector` class is in charge of verifying that the appointment data provided by the user is correct.
+
+    Args:
+        block_processor (:obj:`BlockProcessor <teos.block_processor.BlockProcessor>`): a ``BlockProcessor`` instance.
+        min_to_self_delay (:obj:`int`): the minimum to_self_delay accepted in appointments.
+
     """
 
-    def __init__(self, config):
+    def __init__(self, block_processor, min_to_self_delay):
-        self.config = config
+        self.block_processor = block_processor
+        self.min_to_self_delay = min_to_self_delay
 
     def inspect(self, appointment_data, signature, public_key):
         """
@@ -49,7 +54,7 @@ class Inspector:
             Errors are defined in :mod:`Errors <teos.errors>`.
         """
 
-        block_height = BlockProcessor.get_block_count()
+        block_height = self.block_processor.get_block_count()
 
         if block_height is not None:
             rcode, message = self.check_locator(appointment_data.get("locator"))
@@ -279,10 +284,10 @@ class Inspector:
                 to_self_delay, pow(2, 32)
             )
 
-        elif to_self_delay < self.config.get("MIN_TO_SELF_DELAY"):
+        elif to_self_delay < self.min_to_self_delay:
             rcode = errors.APPOINTMENT_FIELD_TOO_SMALL
             message = "to_self_delay too small. The to_self_delay should be at least {} (current: {})".format(
-                self.config.get("MIN_TO_SELF_DELAY"), to_self_delay
+                self.min_to_self_delay, to_self_delay
             )
 
         if message is not None:

teos/responder.py

@@ -5,8 +5,6 @@ from threading import Thread
 from teos import LOG_PREFIX
 from common.logger import Logger
 from teos.cleaner import Cleaner
-from teos.carrier import Carrier
-from teos.block_processor import BlockProcessor
 
 CONFIRMATIONS_BEFORE_RETRY = 6
 MIN_CONFIRMATIONS = 6
@@ -125,17 +123,22 @@ class Responder:
         is populated by the :obj:`ChainMonitor <teos.chain_monitor.ChainMonitor>`.
         db_manager (:obj:`DBManager <teos.db_manager.DBManager>`): A ``DBManager`` instance to interact with the
             database.
+        carrier (:obj:`Carrier <teos.carrier.Carrier>`): a ``Carrier`` instance to send transactions to bitcoind.
+        block_processor (:obj:`DBManager <teos.block_processor.BlockProcessor>`): a ``BlockProcessor`` instance to get
+            data from bitcoind.
+        last_known_block (:obj:`str`): the last block known by the ``Responder``.
 
     """
 
-    def __init__(self, db_manager):
+    def __init__(self, db_manager, carrier, block_processor):
         self.trackers = dict()
         self.tx_tracker_map = dict()
         self.unconfirmed_txs = []
         self.missed_confirmations = dict()
         self.block_queue = Queue()
         self.db_manager = db_manager
-        self.carrier = Carrier()
+        self.carrier = carrier
+        self.block_processor = block_processor
         self.last_known_block = db_manager.load_last_block_hash_responder()
 
     def awake(self):
@@ -144,8 +147,7 @@ class Responder:
 
         return responder_thread
 
-    @staticmethod
+    def on_sync(self, block_hash):
-    def on_sync(block_hash):
         """
         Whether the :obj:`Responder` is on sync with ``bitcoind`` or not. Used when recovering from a crash.
 
@@ -165,8 +167,7 @@ class Responder:
             :obj:`bool`: Whether or not the :obj:`Responder` and ``bitcoind`` are on sync.
         """
 
-        block_processor = BlockProcessor()
+        distance_from_tip = self.block_processor.get_distance_to_tip(block_hash)
-        distance_from_tip = block_processor.get_distance_to_tip(block_hash)
 
         if distance_from_tip is not None and distance_from_tip > 1:
             synchronized = False
@@ -266,11 +267,11 @@ class Responder:
 
         # Distinguish fresh bootstraps from bootstraps from db
         if self.last_known_block is None:
-            self.last_known_block = BlockProcessor.get_best_block_hash()
+            self.last_known_block = self.block_processor.get_best_block_hash()
 
         while True:
             block_hash = self.block_queue.get()
-            block = BlockProcessor.get_block(block_hash)
+            block = self.block_processor.get_block(block_hash)
             logger.info("New block received", block_hash=block_hash, prev_block_hash=block.get("previousblockhash"))
 
             if len(self.trackers) > 0 and block is not None:
@@ -377,7 +378,7 @@ class Responder:
             if appointment_end <= height and penalty_txid not in self.unconfirmed_txs:
 
                 if penalty_txid not in checked_txs:
-                    tx = Carrier.get_transaction(penalty_txid)
+                    tx = self.carrier.get_transaction(penalty_txid)
                 else:
                     tx = checked_txs.get(penalty_txid)
 

teos/teosd.py

@@ -1,15 +1,20 @@
-from getopt import getopt
+import os
 from sys import argv, exit
+from getopt import getopt, GetoptError
 from signal import signal, SIGINT, SIGQUIT, SIGTERM
 
 import common.cryptographer
 from common.logger import Logger
+from common.config_loader import ConfigLoader
 from common.cryptographer import Cryptographer
+from common.tools import setup_logging, setup_data_folder
 
-from teos import config, LOG_PREFIX
+from teos import LOG_PREFIX, DATA_DIR, DEFAULT_CONF
 from teos.api import API
 from teos.watcher import Watcher
 from teos.builder import Builder
+from teos.carrier import Carrier
+from teos.inspector import Inspector
 from teos.responder import Responder
 from teos.db_manager import DBManager
 from teos.chain_monitor import ChainMonitor
@@ -36,13 +41,22 @@ def main():
     signal(SIGTERM, handle_signals)
     signal(SIGQUIT, handle_signals)
 
+    # Loads config and sets up the data folder and log file
+    config_loader = ConfigLoader(DATA_DIR, DEFAULT_CONF, command_line_conf)
+    config = config_loader.build_config()
+    setup_data_folder(DATA_DIR)
+    setup_logging(config.get("LOG_FILE"), LOG_PREFIX)
+
     logger.info("Starting TEOS")
     db_manager = DBManager(config.get("DB_PATH"))
 
-    if not can_connect_to_bitcoind():
+    bitcoind_connect_params = {k: v for k, v in config.items() if k.startswith("BTC")}
+    bitcoind_feed_params = {k: v for k, v in config.items() if k.startswith("FEED")}
+
+    if not can_connect_to_bitcoind(bitcoind_connect_params):
         logger.error("Can't connect to bitcoind. Shutting down")
 
-    elif not in_correct_network(config.get("BTC_NETWORK")):
+    elif not in_correct_network(bitcoind_connect_params, config.get("BTC_NETWORK")):
         logger.error("bitcoind is running on a different network, check conf.py and bitcoin.conf. Shutting down")
 
     else:
@@ -51,10 +65,23 @@ def main():
             if not secret_key_der:
                 raise IOError("TEOS private key can't be loaded")
 
-            watcher = Watcher(db_manager, Responder(db_manager), secret_key_der, config)
+            block_processor = BlockProcessor(bitcoind_connect_params)
+            carrier = Carrier(bitcoind_connect_params)
+
+            responder = Responder(db_manager, carrier, block_processor)
+            watcher = Watcher(
+                db_manager,
+                block_processor,
+                responder,
+                secret_key_der,
+                config.get("MAX_APPOINTMENTS"),
+                config.get("EXPIRY_DELTA"),
+            )
 
             # Create the chain monitor and start monitoring the chain
-            chain_monitor = ChainMonitor(watcher.block_queue, watcher.responder.block_queue)
+            chain_monitor = ChainMonitor(
+                watcher.block_queue, watcher.responder.block_queue, block_processor, bitcoind_feed_params
+            )
 
             watcher_appointments_data = db_manager.load_watcher_appointments()
             responder_trackers_data = db_manager.load_responder_trackers()
@@ -89,7 +116,6 @@ def main():
 
                 # Populate the block queues with data if they've missed some while offline. If the blocks of both match
                 # we don't perform the search twice.
-                block_processor = BlockProcessor()
 
                 # FIXME: 32-reorgs-offline dropped txs are not used at this point.
                 last_common_ancestor_watcher, dropped_txs_watcher = block_processor.find_last_common_ancestor(
@@ -123,16 +149,37 @@ def main():
             # Fire the API and the ChainMonitor
             # FIXME: 92-block-data-during-bootstrap-db
             chain_monitor.monitor_chain()
-            API(watcher, config=config).start()
+            API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher).start()
         except Exception as e:
             logger.error("An error occurred: {}. Shutting down".format(e))
             exit(1)
 
 
 if __name__ == "__main__":
-    opts, _ = getopt(argv[1:], "", [""])
+    command_line_conf = {}
+
+    try:
+        opts, _ = getopt(
+            argv[1:], "", ["btcnetwork=", "btcrpcuser=", "btcrpcpassword=", "btcrpcconnect=", "btcrpcport=", "datadir="]
+        )
         for opt, arg in opts:
-        # FIXME: Leaving this here for future option/arguments
+            if opt in ["--btcnetwork"]:
-        pass
+                command_line_conf["BTC_NETWORK"] = arg
+            if opt in ["--btcrpcuser"]:
+                command_line_conf["BTC_RPC_USER"] = arg
+            if opt in ["--btcrpcpassword"]:
+                command_line_conf["BTC_RPC_PASSWD"] = arg
+            if opt in ["--btcrpcconnect"]:
+                command_line_conf["BTC_RPC_CONNECT"] = arg
+            if opt in ["--btcrpcport"]:
+                try:
+                    command_line_conf["BTC_RPC_PORT"] = int(arg)
+                except ValueError:
+                    exit("btcrpcport must be an integer")
+            if opt in ["--datadir"]:
+                DATA_DIR = os.path.expanduser(arg)
+
+    except GetoptError as e:
+        exit(e)
 
     main()

teos/tools.py

@@ -1,7 +1,6 @@
-from http.client import HTTPException
 from socket import timeout
+from http.client import HTTPException
 
-import teos.conf as conf
 from teos.utils.auth_proxy import AuthServiceProxy, JSONRPCException
 
 """
@@ -10,25 +9,38 @@ Tools is a module with general methods that can used by different entities in th
 
 
 # NOTCOVERED
-def bitcoin_cli():
+def bitcoin_cli(btc_connect_params):
     """
     An ``http`` connection with ``bitcoind`` using the ``json-rpc`` interface.
 
+    Args:
+        btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind
+            (rpc user, rpc passwd, host and port)
+
     Returns:
         :obj:`AuthServiceProxy <teos.utils.auth_proxy.AuthServiceProxy>`: An authenticated service proxy to ``bitcoind``
         that can be used to send ``json-rpc`` commands.
     """
 
     return AuthServiceProxy(
-        "http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST, conf.BTC_RPC_PORT)
+        "http://%s:%s@%s:%d"
+        % (
+            btc_connect_params.get("BTC_RPC_USER"),
+            btc_connect_params.get("BTC_RPC_PASSWD"),
+            btc_connect_params.get("BTC_RPC_CONNECT"),
+            btc_connect_params.get("BTC_RPC_PORT"),
+        )
     )
 
 
 # NOTCOVERED
-def can_connect_to_bitcoind():
+def can_connect_to_bitcoind(btc_connect_params):
     """
     Checks if the tower has connection to ``bitcoind``.
 
+    Args:
+        btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind
+            (rpc user, rpc passwd, host and port)
     Returns:
         :obj:`bool`: ``True`` if the connection can be established. ``False`` otherwise.
     """
@@ -36,18 +48,23 @@ def can_connect_to_bitcoind():
     can_connect = True
 
     try:
-        bitcoin_cli().help()
+        bitcoin_cli(btc_connect_params).help()
     except (timeout, ConnectionRefusedError, JSONRPCException, HTTPException, OSError):
         can_connect = False
 
     return can_connect
 
 
-def in_correct_network(network):
+def in_correct_network(btc_connect_params, network):
     """
     Checks if ``bitcoind`` and the tower are configured to run in the same network (``mainnet``, ``testnet`` or
     ``regtest``)
 
+    Args:
+        btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind
+            (rpc user, rpc passwd, host and port)
+        network (:obj:`str`): the network the tower is connected to.
+
     Returns:
         :obj:`bool`: ``True`` if the network configuration matches. ``False`` otherwise.
     """
@@ -56,7 +73,7 @@ def in_correct_network(network):
     testnet3_genesis_block_hash = "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"
     correct_network = False
 
-    genesis_block_hash = bitcoin_cli().getblockhash(0)
+    genesis_block_hash = bitcoin_cli(btc_connect_params).getblockhash(0)
 
     if network == "mainnet" and genesis_block_hash == mainnet_genesis_block_hash:
         correct_network = True

teos/watcher.py

@@ -3,15 +3,13 @@ from queue import Queue
 from threading import Thread
 
 import common.cryptographer
-from common.cryptographer import Cryptographer
-from common.appointment import Appointment
-from common.tools import compute_locator
-
 from common.logger import Logger
+from common.tools import compute_locator
+from common.appointment import Appointment
+from common.cryptographer import Cryptographer
 
 from teos import LOG_PREFIX
 from teos.cleaner import Cleaner
-from teos.block_processor import BlockProcessor
 
 logger = Logger(actor="Watcher", log_name_prefix=LOG_PREFIX)
 common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX)
@@ -34,11 +32,12 @@ class Watcher:
 
     Args:
         db_manager (:obj:`DBManager <teos.db_manager>`): a ``DBManager`` instance to interact with the database.
-        sk_der (:obj:`bytes`): a DER encoded private key used to sign appointment receipts (signaling acceptance).
+        block_processor (:obj:`BlockProcessor <teos.block_processor.BlockProcessor>`): a ``BlockProcessor`` instance to
-        config (:obj:`dict`): a dictionary containing all the configuration parameters. Used locally to retrieve
+            get block from bitcoind.
-            ``MAX_APPOINTMENTS``  and ``EXPIRY_DELTA``.
         responder (:obj:`Responder <teos.responder.Responder>`): a ``Responder`` instance.
-
+        sk_der (:obj:`bytes`): a DER encoded private key used to sign appointment receipts (signaling acceptance).
+        max_appointments (:obj:`int`): the maximum ammount of appointments accepted by the ``Watcher`` at the same time.
+        expiry_delta (:obj:`int`): the additional time the ``Watcher`` will keep an expired appointment around.
 
     Attributes:
         appointments (:obj:`dict`): a dictionary containing a simplification of the appointments (:obj:`Appointment
@@ -48,23 +47,28 @@ class Watcher:
             appointments with the same ``locator``.
         block_queue (:obj:`Queue`): A queue used by the :obj:`Watcher` to receive block hashes from ``bitcoind``. It is
         populated by the :obj:`ChainMonitor <teos.chain_monitor.ChainMonitor>`.
-        config (:obj:`dict`): a dictionary containing all the configuration parameters. Used locally to retrieve
-            ``MAX_APPOINTMENTS``  and ``EXPIRY_DELTA``.
         db_manager (:obj:`DBManager <teos.db_manager>`): A db manager instance to interact with the database.
+        block_processor (:obj:`BlockProcessor <teos.block_processor.BlockProcessor>`): a ``BlockProcessor`` instance to
+            get block from bitcoind.
+        responder (:obj:`Responder <teos.responder.Responder>`): a ``Responder`` instance.
         signing_key (:mod:`PrivateKey`): a private key used to sign accepted appointments.
+        max_appointments (:obj:`int`): the maximum ammount of appointments accepted by the ``Watcher`` at the same time.
+        expiry_delta (:obj:`int`): the additional time the ``Watcher`` will keep an expired appointment around.
 
     Raises:
         ValueError: if `teos_sk_file` is not found.
 
     """
 
-    def __init__(self, db_manager, responder, sk_der, config):
+    def __init__(self, db_manager, block_processor, responder, sk_der, max_appointments, expiry_delta):
         self.appointments = dict()
         self.locator_uuid_map = dict()
         self.block_queue = Queue()
-        self.config = config
         self.db_manager = db_manager
+        self.block_processor = block_processor
         self.responder = responder
+        self.max_appointments = max_appointments
+        self.expiry_delta = expiry_delta
         self.signing_key = Cryptographer.load_private_key_der(sk_der)
 
     def awake(self):
@@ -102,7 +106,7 @@ class Watcher:
 
         """
 
-        if len(self.appointments) < self.config.get("MAX_APPOINTMENTS"):
+        if len(self.appointments) < self.max_appointments:
 
             uuid = uuid4().hex
             self.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time}
@@ -139,7 +143,7 @@ class Watcher:
 
         while True:
             block_hash = self.block_queue.get()
-            block = BlockProcessor.get_block(block_hash)
+            block = self.block_processor.get_block(block_hash)
             logger.info("New block received", block_hash=block_hash, prev_block_hash=block.get("previousblockhash"))
 
             if len(self.appointments) > 0 and block is not None:
@@ -148,7 +152,7 @@ class Watcher:
                 expired_appointments = [
                     uuid
                     for uuid, appointment_data in self.appointments.items()
-                    if block["height"] > appointment_data.get("end_time") + self.config.get("EXPIRY_DELTA")
+                    if block["height"] > appointment_data.get("end_time") + self.expiry_delta
                 ]
 
                 Cleaner.delete_expired_appointments(
@@ -265,7 +269,7 @@ class Watcher:
                     except ValueError:
                         penalty_rawtx = None
 
-                    penalty_tx = BlockProcessor.decode_raw_transaction(penalty_rawtx)
+                    penalty_tx = self.block_processor.decode_raw_transaction(penalty_rawtx)
                     decrypted_blobs[appointment.encrypted_blob.data] = (penalty_tx, penalty_rawtx)
 
                 if penalty_tx is not None: