diff --git a/common/tools.py b/common/tools.py index 55dfe09..2c4969e 100644 --- a/common/tools.py +++ b/common/tools.py @@ -54,42 +54,9 @@ def setup_data_folder(data_folder): os.makedirs(data_folder, exist_ok=True) -def check_conf_fields(conf_fields): - """ - Checks that the provided configuration field have the right type. - - Args: - conf_fields (:obj:`dict`): a dictionary populated with the configuration file params and the expected types. - The format is as follows: - - {"field0": {"value": value_from_conf_file, "type": expected_type, ...}} - - Returns: - :obj:`dict`: A dictionary with the same keys as the provided one, but containing only the "value" field as value - if the provided ``conf_fields`` where correct. - - Raises: - ValueError: If any of the dictionary elements does not have the expected type - """ - - conf_dict = {} - - for field in conf_fields: - value = conf_fields[field]["value"] - correct_type = conf_fields[field]["type"] - - if (value is not None) and isinstance(value, correct_type): - conf_dict[field] = value - else: - err_msg = "{} variable in config is of the wrong type".format(field) - raise ValueError(err_msg) - - return conf_dict - - def extend_paths(base_path, config_fields): """ - Extends the relative paths of a given ``config_fields`` dictionary with a diven ``base_path``. + Extends the relative paths of a given ``config_fields`` dictionary with a given ``base_path``. Paths in the config file are based on DATA_PATH, this method extends them so they are all absolute. @@ -98,16 +65,12 @@ def extend_paths(base_path, config_fields): config_fields (:obj:`dict`): a dictionary of configuration fields containing a ``path`` flag, as follows: {"field0": {"value": value_from_conf_file, "path": True, ...}} - Returns: - :obj:`dict`: A ``config_fields`` with the flagged paths updated. """ for key, field in config_fields.items(): if field.get("path") is True: config_fields[key]["value"] = base_path + config_fields[key]["value"] - return config_fields - def setup_logging(log_file_path, log_name_prefix): """ diff --git a/teos/__init__.py b/teos/__init__.py index a56eabf..9a67e97 100644 --- a/teos/__init__.py +++ b/teos/__init__.py @@ -1,38 +1,25 @@ import os -import teos.conf as conf -from common.tools import check_conf_fields, setup_logging, extend_paths, setup_data_folder from teos.utils.auth_proxy import AuthServiceProxy HOST = "localhost" PORT = 9814 +DATA_DIR = os.path.expanduser("~/.teos/") LOG_PREFIX = "teos" -# Load config fields -conf_fields = { - "BTC_RPC_USER": {"value": conf.BTC_RPC_USER, "type": str}, - "BTC_RPC_PASSWD": {"value": conf.BTC_RPC_PASSWD, "type": str}, - "BTC_RPC_HOST": {"value": conf.BTC_RPC_HOST, "type": str}, - "BTC_RPC_PORT": {"value": conf.BTC_RPC_PORT, "type": int}, - "BTC_NETWORK": {"value": conf.BTC_NETWORK, "type": str}, - "FEED_PROTOCOL": {"value": conf.FEED_PROTOCOL, "type": str}, - "FEED_ADDR": {"value": conf.FEED_ADDR, "type": str}, - "FEED_PORT": {"value": conf.FEED_PORT, "type": int}, - "DATA_FOLDER": {"value": conf.DATA_FOLDER, "type": str}, - "MAX_APPOINTMENTS": {"value": conf.MAX_APPOINTMENTS, "type": int}, - "EXPIRY_DELTA": {"value": conf.EXPIRY_DELTA, "type": int}, - "MIN_TO_SELF_DELAY": {"value": conf.MIN_TO_SELF_DELAY, "type": int}, - "SERVER_LOG_FILE": {"value": conf.SERVER_LOG_FILE, "type": str, "path": True}, - "TEOS_SECRET_KEY": {"value": conf.TEOS_SECRET_KEY, "type": str, "path": True}, - "DB_PATH": {"value": conf.DB_PATH, "type": str, "path": True}, +# Default conf fields +DEFAULT_CONF = { + "BTC_RPC_USER": {"value": "user", "type": str}, + "BTC_RPC_PASSWD": {"value": "passwd", "type": str}, + "BTC_RPC_CONNECT": {"value": "127.0.0.1", "type": str}, + "BTC_RPC_PORT": {"value": 8332, "type": int}, + "BTC_NETWORK": {"value": "mainnet", "type": str}, + "FEED_PROTOCOL": {"value": "tcp", "type": str}, + "FEED_CONNECT": {"value": "127.0.0.1", "type": str}, + "FEED_PORT": {"value": 28332, "type": int}, + "MAX_APPOINTMENTS": {"value": 100, "type": int}, + "EXPIRY_DELTA": {"value": 6, "type": int}, + "MIN_TO_SELF_DELAY": {"value": 20, "type": int}, + "LOG_FILE": {"value": "teos.log", "type": str, "path": True}, + "TEOS_SECRET_KEY": {"value": "teos_sk.der", "type": str, "path": True}, + "DB_PATH": {"value": "appointments", "type": str, "path": True}, } - -# Expand user (~) if found and check fields are correct -conf_fields["DATA_FOLDER"]["value"] = os.path.expanduser(conf_fields["DATA_FOLDER"]["value"]) -# Extend relative paths -conf_fields = extend_paths(conf_fields["DATA_FOLDER"]["value"], conf_fields) - -# Sanity check fields and build config dictionary -config = check_conf_fields(conf_fields) - -setup_data_folder(config.get("DATA_FOLDER")) -setup_logging(config.get("SERVER_LOG_FILE"), LOG_PREFIX) diff --git a/teos/api.py b/teos/api.py index fa04387..97fcf6d 100644 --- a/teos/api.py +++ b/teos/api.py @@ -5,7 +5,6 @@ from flask import Flask, request, abort, jsonify from teos import HOST, PORT, LOG_PREFIX from common.logger import Logger -from teos.inspector import Inspector from common.appointment import Appointment from common.constants import HTTP_OK, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, LOCATOR_LEN_HEX @@ -17,9 +16,18 @@ logger = Logger(actor="API", log_name_prefix=LOG_PREFIX) class API: - def __init__(self, watcher, config): + """ + The :class:`API` is in charge of the interface between the user and the tower. It handles and server user requests. + + Args: + inspector (:obj:`Inspector `): an ``Inspector`` instance to check the correctness of + the received data. + watcher (:obj:`Watcher `): a ``Watcher`` instance to pass the requests to. + """ + + def __init__(self, inspector, watcher): + self.inspector = inspector self.watcher = watcher - self.config = config def add_appointment(self): """ @@ -48,8 +56,7 @@ class API: if request.is_json: # Check content type once if properly defined request_data = json.loads(request.get_json()) - inspector = Inspector(self.config) - appointment = inspector.inspect( + appointment = self.inspector.inspect( request_data.get("appointment"), request_data.get("signature"), request_data.get("public_key") ) diff --git a/teos/block_processor.py b/teos/block_processor.py index bef9c6e..7b9ea24 100644 --- a/teos/block_processor.py +++ b/teos/block_processor.py @@ -11,10 +11,16 @@ class BlockProcessor: """ The :class:`BlockProcessor` contains methods related to the blockchain. Most of its methods require communication with ``bitcoind``. + + Args: + btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind + (rpc user, rpc passwd, host and port) """ - @staticmethod - def get_block(block_hash): + def __init__(self, btc_connect_params): + self.btc_connect_params = btc_connect_params + + def get_block(self, block_hash): """ Gives a block given a block hash by querying ``bitcoind``. @@ -28,7 +34,7 @@ class BlockProcessor: """ try: - block = bitcoin_cli().getblock(block_hash) + block = bitcoin_cli(self.btc_connect_params).getblock(block_hash) except JSONRPCException as e: block = None @@ -36,8 +42,7 @@ class BlockProcessor: return block - @staticmethod - def get_best_block_hash(): + def get_best_block_hash(self): """ Returns the hash of the current best chain tip. @@ -48,7 +53,7 @@ class BlockProcessor: """ try: - block_hash = bitcoin_cli().getbestblockhash() + block_hash = bitcoin_cli(self.btc_connect_params).getbestblockhash() except JSONRPCException as e: block_hash = None @@ -56,8 +61,7 @@ class BlockProcessor: return block_hash - @staticmethod - def get_block_count(): + def get_block_count(self): """ Returns the block height of the best chain. @@ -68,7 +72,7 @@ class BlockProcessor: """ try: - block_count = bitcoin_cli().getblockcount() + block_count = bitcoin_cli(self.btc_connect_params).getblockcount() except JSONRPCException as e: block_count = None @@ -76,8 +80,7 @@ class BlockProcessor: return block_count - @staticmethod - def decode_raw_transaction(raw_tx): + def decode_raw_transaction(self, raw_tx): """ Deserializes a given raw transaction (hex encoded) and builds a dictionary representing it with all the associated metadata given by ``bitcoind`` (e.g. confirmation count). @@ -92,7 +95,7 @@ class BlockProcessor: """ try: - tx = bitcoin_cli().decoderawtransaction(raw_tx) + tx = bitcoin_cli(self.btc_connect_params).decoderawtransaction(raw_tx) except JSONRPCException as e: tx = None @@ -100,8 +103,7 @@ class BlockProcessor: return tx - @staticmethod - def get_distance_to_tip(target_block_hash): + def get_distance_to_tip(self, target_block_hash): """ Compute the distance between a given block hash and the best chain tip. @@ -117,10 +119,10 @@ class BlockProcessor: distance = None - chain_tip = BlockProcessor.get_best_block_hash() - chain_tip_height = BlockProcessor.get_block(chain_tip).get("height") + chain_tip = self.get_best_block_hash() + chain_tip_height = self.get_block(chain_tip).get("height") - target_block = BlockProcessor.get_block(target_block_hash) + target_block = self.get_block(target_block_hash) if target_block is not None: target_block_height = target_block.get("height") @@ -129,8 +131,7 @@ class BlockProcessor: return distance - @staticmethod - def get_missed_blocks(last_know_block_hash): + def get_missed_blocks(self, last_know_block_hash): """ Compute the blocks between the current best chain tip and a given block hash (``last_know_block_hash``). @@ -144,19 +145,18 @@ class BlockProcessor: child of ``last_know_block_hash``. """ - current_block_hash = BlockProcessor.get_best_block_hash() + current_block_hash = self.get_best_block_hash() missed_blocks = [] while current_block_hash != last_know_block_hash and current_block_hash is not None: missed_blocks.append(current_block_hash) - current_block = BlockProcessor.get_block(current_block_hash) + current_block = self.get_block(current_block_hash) current_block_hash = current_block.get("previousblockhash") return missed_blocks[::-1] - @staticmethod - def is_block_in_best_chain(block_hash): + def is_block_in_best_chain(self, block_hash): """ Checks whether or not a given block is on the best chain. Blocks are identified by block_hash. @@ -173,7 +173,7 @@ class BlockProcessor: KeyError: If the block cannot be found in the blockchain. """ - block = BlockProcessor.get_block(block_hash) + block = self.get_block(block_hash) if block is None: # This should never happen as long as we are using the same node, since bitcoind never drops orphan blocks @@ -185,8 +185,7 @@ class BlockProcessor: else: return False - @staticmethod - def find_last_common_ancestor(last_known_block_hash): + def find_last_common_ancestor(self, last_known_block_hash): """ Finds the last common ancestor between the current best chain tip and the last block known by us (older block). @@ -204,8 +203,8 @@ class BlockProcessor: target_block_hash = last_known_block_hash dropped_txs = [] - while not BlockProcessor.is_block_in_best_chain(target_block_hash): - block = BlockProcessor.get_block(target_block_hash) + while not self.is_block_in_best_chain(target_block_hash): + block = self.get_block(target_block_hash) dropped_txs.extend(block.get("tx")) target_block_hash = block.get("previousblockhash") diff --git a/teos/builder.py b/teos/builder.py index dc140d0..5a4dc42 100644 --- a/teos/builder.py +++ b/teos/builder.py @@ -1,6 +1,6 @@ class Builder: """ - The :class:`Builder` class is in charge or reconstructing data loaded from the database and build the data + The :class:`Builder` class is in charge of reconstructing data loaded from the database and build the data structures of the :obj:`Watcher ` and the :obj:`Responder `. """ diff --git a/teos/carrier.py b/teos/carrier.py index 1e92329..0537c63 100644 --- a/teos/carrier.py +++ b/teos/carrier.py @@ -39,13 +39,18 @@ class Carrier: The :class:`Carrier` is the class in charge of interacting with ``bitcoind`` to send/get transactions. It uses :obj:`Receipt` objects to report about the sending outcome. + Args: + btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind + (rpc user, rpc passwd, host and port) + Attributes: issued_receipts (:obj:`dict`): a dictionary of issued receipts to prevent resending the same transaction over and over. It should periodically be reset to prevent it from growing unbounded. """ - def __init__(self): + def __init__(self, btc_connect_params): + self.btc_connect_params = btc_connect_params self.issued_receipts = {} # NOTCOVERED @@ -69,7 +74,7 @@ class Carrier: try: logger.info("Pushing transaction to the network", txid=txid, rawtx=rawtx) - bitcoin_cli().sendrawtransaction(rawtx) + bitcoin_cli(self.btc_connect_params).sendrawtransaction(rawtx) receipt = Receipt(delivered=True) @@ -119,8 +124,7 @@ class Carrier: return receipt - @staticmethod - def get_transaction(txid): + def get_transaction(self, txid): """ Queries transaction data to ``bitcoind`` given a transaction id. @@ -134,7 +138,7 @@ class Carrier: """ try: - tx_info = bitcoin_cli().getrawtransaction(txid, 1) + tx_info = bitcoin_cli(self.btc_connect_params).getrawtransaction(txid, 1) except JSONRPCException as e: tx_info = None diff --git a/teos/chain_monitor.py b/teos/chain_monitor.py index 86a2126..186a31d 100644 --- a/teos/chain_monitor.py +++ b/teos/chain_monitor.py @@ -4,8 +4,6 @@ from threading import Thread, Event, Condition from teos import LOG_PREFIX from common.logger import Logger -from teos.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT, POLLING_DELTA, BLOCK_WINDOW_SIZE -from teos.block_processor import BlockProcessor logger = Logger(actor="ChainMonitor", log_name_prefix=LOG_PREFIX) @@ -22,6 +20,8 @@ class ChainMonitor: Args: watcher_queue (:obj:`Queue`): the queue to be used to send blocks hashes to the ``Watcher``. responder_queue (:obj:`Queue`): the queue to be used to send blocks hashes to the ``Responder``. + block_processor (:obj:`BlockProcessor `): a blockProcessor instance. + bitcoind_feed_params (:obj:`dict`): a dict with the feed (ZMQ) connection parameters. Attributes: best_tip (:obj:`str`): a block hash representing the current best tip. @@ -34,9 +34,13 @@ class ChainMonitor: watcher_queue (:obj:`Queue`): a queue to send new best tips to the :obj:`Watcher `. responder_queue (:obj:`Queue`): a queue to send new best tips to the :obj:`Responder `. + + polling_delta (:obj:`int`): time between polls (in seconds). + max_block_window_size (:obj:`int`): max size of last_tips. + block_processor (:obj:`BlockProcessor `): a blockProcessor instance. """ - def __init__(self, watcher_queue, responder_queue): + def __init__(self, watcher_queue, responder_queue, block_processor, bitcoind_feed_params): self.best_tip = None self.last_tips = [] self.terminate = False @@ -48,11 +52,22 @@ class ChainMonitor: self.zmqSubSocket = self.zmqContext.socket(zmq.SUB) self.zmqSubSocket.setsockopt(zmq.RCVHWM, 0) self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock") - self.zmqSubSocket.connect("%s://%s:%s" % (FEED_PROTOCOL, FEED_ADDR, FEED_PORT)) + self.zmqSubSocket.connect( + "%s://%s:%s" + % ( + bitcoind_feed_params.get("FEED_PROTOCOL"), + bitcoind_feed_params.get("FEED_CONNECT"), + bitcoind_feed_params.get("FEED_PORT"), + ) + ) self.watcher_queue = watcher_queue self.responder_queue = responder_queue + self.polling_delta = 60 + self.max_block_window_size = 10 + self.block_processor = block_processor + def notify_subscribers(self, block_hash): """ Notifies the subscribers (``Watcher`` and ``Responder``) about a new block. It does so by putting the hash in @@ -66,14 +81,13 @@ class ChainMonitor: self.watcher_queue.put(block_hash) self.responder_queue.put(block_hash) - def update_state(self, block_hash, max_block_window_size=BLOCK_WINDOW_SIZE): + def update_state(self, block_hash): """ Updates the state of the ``ChainMonitor``. The state is represented as the ``best_tip`` and the list of ``last_tips``. ``last_tips`` is bounded to ``max_block_window_size``. Args: block_hash (:obj:`block_hash`): the new best tip. - max_block_window_size (:obj:`int`): the maximum length of the ``last_tips`` list. Returns: (:obj:`bool`): ``True`` is the state was successfully updated, ``False`` otherwise. @@ -83,7 +97,7 @@ class ChainMonitor: self.last_tips.append(self.best_tip) self.best_tip = block_hash - if len(self.last_tips) > max_block_window_size: + if len(self.last_tips) > self.max_block_window_size: self.last_tips.pop(0) return True @@ -91,22 +105,19 @@ class ChainMonitor: else: return False - def monitor_chain_polling(self, polling_delta=POLLING_DELTA): + def monitor_chain_polling(self): """ Monitors ``bitcoind`` via polling. Once the method is fired, it keeps monitoring as long as ``terminate`` is not set. Polling is performed once every ``polling_delta`` seconds. If a new best tip if found, the shared lock is acquired, the state is updated and the subscribers are notified, and finally the lock is released. - - Args: - polling_delta (:obj:`int`): the time delta between polls. """ while not self.terminate: - self.check_tip.wait(timeout=polling_delta) + self.check_tip.wait(timeout=self.polling_delta) # Terminate could have been set while the thread was blocked in wait if not self.terminate: - current_tip = BlockProcessor.get_best_block_hash() + current_tip = self.block_processor.get_best_block_hash() self.lock.acquire() if self.update_state(current_tip): @@ -138,16 +149,13 @@ class ChainMonitor: logger.info("New block received via zmq", block_hash=block_hash) self.lock.release() - def monitor_chain(self, polling_delta=POLLING_DELTA): + def monitor_chain(self): """ Main :class:`ChainMonitor` method. It initializes the ``best_tip`` to the current one (by querying the :obj:`BlockProcessor `) and creates two threads, one per each monitoring approach (``zmq`` and ``polling``). - - Args: - polling_delta (:obj:`int`): the time delta between polls by the ``monitor_chain_polling`` thread. """ - self.best_tip = BlockProcessor.get_best_block_hash() - Thread(target=self.monitor_chain_polling, daemon=True, kwargs={"polling_delta": polling_delta}).start() + self.best_tip = self.block_processor.get_best_block_hash() + Thread(target=self.monitor_chain_polling, daemon=True).start() Thread(target=self.monitor_chain_zmq, daemon=True).start() diff --git a/teos/cleaner.py b/teos/cleaner.py index 93924ba..539b603 100644 --- a/teos/cleaner.py +++ b/teos/cleaner.py @@ -81,7 +81,7 @@ class Cleaner: logger.error("Some UUIDs not found in the db", locator=locator, all_uuids=uuids) else: - logger.error("Locator map not found in the db", uuid=locator) + logger.error("Locator map not found in the db", locator=locator) @staticmethod def delete_expired_appointments(expired_appointments, appointments, locator_uuid_map, db_manager): diff --git a/teos/conf.py b/teos/conf.py deleted file mode 100644 index 450d5ed..0000000 --- a/teos/conf.py +++ /dev/null @@ -1,26 +0,0 @@ -# bitcoind -BTC_RPC_USER = "user" -BTC_RPC_PASSWD = "passwd" -BTC_RPC_HOST = "localhost" -BTC_RPC_PORT = 18443 -BTC_NETWORK = "regtest" - -# ZMQ -FEED_PROTOCOL = "tcp" -FEED_ADDR = "127.0.0.1" -FEED_PORT = 28332 - -# TEOS -DATA_FOLDER = "~/.teos/" -MAX_APPOINTMENTS = 100 -EXPIRY_DELTA = 6 -MIN_TO_SELF_DELAY = 20 -SERVER_LOG_FILE = "teos.log" -TEOS_SECRET_KEY = "teos_sk.der" - -# CHAIN MONITOR -POLLING_DELTA = 60 -BLOCK_WINDOW_SIZE = 10 - -# LEVELDB -DB_PATH = "appointments" diff --git a/teos/inspector.py b/teos/inspector.py index 1c1a5d1..ed288fc 100644 --- a/teos/inspector.py +++ b/teos/inspector.py @@ -8,7 +8,6 @@ from common.cryptographer import Cryptographer, PublicKey from teos import errors, LOG_PREFIX from common.logger import Logger from common.appointment import Appointment -from teos.block_processor import BlockProcessor logger = Logger(actor="Inspector", log_name_prefix=LOG_PREFIX) common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX) @@ -26,10 +25,16 @@ ENCRYPTED_BLOB_MAX_SIZE_HEX = 2 * 2048 class Inspector: """ The :class:`Inspector` class is in charge of verifying that the appointment data provided by the user is correct. + + Args: + block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance. + min_to_self_delay (:obj:`int`): the minimum to_self_delay accepted in appointments. + """ - def __init__(self, config): - self.config = config + def __init__(self, block_processor, min_to_self_delay): + self.block_processor = block_processor + self.min_to_self_delay = min_to_self_delay def inspect(self, appointment_data, signature, public_key): """ @@ -49,7 +54,7 @@ class Inspector: Errors are defined in :mod:`Errors `. """ - block_height = BlockProcessor.get_block_count() + block_height = self.block_processor.get_block_count() if block_height is not None: rcode, message = self.check_locator(appointment_data.get("locator")) @@ -279,10 +284,10 @@ class Inspector: to_self_delay, pow(2, 32) ) - elif to_self_delay < self.config.get("MIN_TO_SELF_DELAY"): + elif to_self_delay < self.min_to_self_delay: rcode = errors.APPOINTMENT_FIELD_TOO_SMALL message = "to_self_delay too small. The to_self_delay should be at least {} (current: {})".format( - self.config.get("MIN_TO_SELF_DELAY"), to_self_delay + self.min_to_self_delay, to_self_delay ) if message is not None: diff --git a/teos/responder.py b/teos/responder.py index d2c995d..526850f 100644 --- a/teos/responder.py +++ b/teos/responder.py @@ -5,8 +5,6 @@ from threading import Thread from teos import LOG_PREFIX from common.logger import Logger from teos.cleaner import Cleaner -from teos.carrier import Carrier -from teos.block_processor import BlockProcessor CONFIRMATIONS_BEFORE_RETRY = 6 MIN_CONFIRMATIONS = 6 @@ -125,17 +123,22 @@ class Responder: is populated by the :obj:`ChainMonitor `. db_manager (:obj:`DBManager `): A ``DBManager`` instance to interact with the database. + carrier (:obj:`Carrier `): a ``Carrier`` instance to send transactions to bitcoind. + block_processor (:obj:`DBManager `): a ``BlockProcessor`` instance to get + data from bitcoind. + last_known_block (:obj:`str`): the last block known by the ``Responder``. """ - def __init__(self, db_manager): + def __init__(self, db_manager, carrier, block_processor): self.trackers = dict() self.tx_tracker_map = dict() self.unconfirmed_txs = [] self.missed_confirmations = dict() self.block_queue = Queue() self.db_manager = db_manager - self.carrier = Carrier() + self.carrier = carrier + self.block_processor = block_processor self.last_known_block = db_manager.load_last_block_hash_responder() def awake(self): @@ -144,8 +147,7 @@ class Responder: return responder_thread - @staticmethod - def on_sync(block_hash): + def on_sync(self, block_hash): """ Whether the :obj:`Responder` is on sync with ``bitcoind`` or not. Used when recovering from a crash. @@ -165,8 +167,7 @@ class Responder: :obj:`bool`: Whether or not the :obj:`Responder` and ``bitcoind`` are on sync. """ - block_processor = BlockProcessor() - distance_from_tip = block_processor.get_distance_to_tip(block_hash) + distance_from_tip = self.block_processor.get_distance_to_tip(block_hash) if distance_from_tip is not None and distance_from_tip > 1: synchronized = False @@ -266,11 +267,11 @@ class Responder: # Distinguish fresh bootstraps from bootstraps from db if self.last_known_block is None: - self.last_known_block = BlockProcessor.get_best_block_hash() + self.last_known_block = self.block_processor.get_best_block_hash() while True: block_hash = self.block_queue.get() - block = BlockProcessor.get_block(block_hash) + block = self.block_processor.get_block(block_hash) logger.info("New block received", block_hash=block_hash, prev_block_hash=block.get("previousblockhash")) if len(self.trackers) > 0 and block is not None: @@ -377,7 +378,7 @@ class Responder: if appointment_end <= height and penalty_txid not in self.unconfirmed_txs: if penalty_txid not in checked_txs: - tx = Carrier.get_transaction(penalty_txid) + tx = self.carrier.get_transaction(penalty_txid) else: tx = checked_txs.get(penalty_txid) diff --git a/teos/teosd.py b/teos/teosd.py index a82f874..2509d02 100644 --- a/teos/teosd.py +++ b/teos/teosd.py @@ -1,15 +1,20 @@ -from getopt import getopt +import os from sys import argv, exit +from getopt import getopt, GetoptError from signal import signal, SIGINT, SIGQUIT, SIGTERM import common.cryptographer from common.logger import Logger +from common.config_loader import ConfigLoader from common.cryptographer import Cryptographer +from common.tools import setup_logging, setup_data_folder -from teos import config, LOG_PREFIX +from teos import LOG_PREFIX, DATA_DIR, DEFAULT_CONF from teos.api import API from teos.watcher import Watcher from teos.builder import Builder +from teos.carrier import Carrier +from teos.inspector import Inspector from teos.responder import Responder from teos.db_manager import DBManager from teos.chain_monitor import ChainMonitor @@ -36,13 +41,22 @@ def main(): signal(SIGTERM, handle_signals) signal(SIGQUIT, handle_signals) + # Loads config and sets up the data folder and log file + config_loader = ConfigLoader(DATA_DIR, DEFAULT_CONF, command_line_conf) + config = config_loader.build_config() + setup_data_folder(DATA_DIR) + setup_logging(config.get("LOG_FILE"), LOG_PREFIX) + logger.info("Starting TEOS") db_manager = DBManager(config.get("DB_PATH")) - if not can_connect_to_bitcoind(): + bitcoind_connect_params = {k: v for k, v in config.items() if k.startswith("BTC")} + bitcoind_feed_params = {k: v for k, v in config.items() if k.startswith("FEED")} + + if not can_connect_to_bitcoind(bitcoind_connect_params): logger.error("Can't connect to bitcoind. Shutting down") - elif not in_correct_network(config.get("BTC_NETWORK")): + elif not in_correct_network(bitcoind_connect_params, config.get("BTC_NETWORK")): logger.error("bitcoind is running on a different network, check conf.py and bitcoin.conf. Shutting down") else: @@ -51,10 +65,23 @@ def main(): if not secret_key_der: raise IOError("TEOS private key can't be loaded") - watcher = Watcher(db_manager, Responder(db_manager), secret_key_der, config) + block_processor = BlockProcessor(bitcoind_connect_params) + carrier = Carrier(bitcoind_connect_params) + + responder = Responder(db_manager, carrier, block_processor) + watcher = Watcher( + db_manager, + block_processor, + responder, + secret_key_der, + config.get("MAX_APPOINTMENTS"), + config.get("EXPIRY_DELTA"), + ) # Create the chain monitor and start monitoring the chain - chain_monitor = ChainMonitor(watcher.block_queue, watcher.responder.block_queue) + chain_monitor = ChainMonitor( + watcher.block_queue, watcher.responder.block_queue, block_processor, bitcoind_feed_params + ) watcher_appointments_data = db_manager.load_watcher_appointments() responder_trackers_data = db_manager.load_responder_trackers() @@ -89,7 +116,6 @@ def main(): # Populate the block queues with data if they've missed some while offline. If the blocks of both match # we don't perform the search twice. - block_processor = BlockProcessor() # FIXME: 32-reorgs-offline dropped txs are not used at this point. last_common_ancestor_watcher, dropped_txs_watcher = block_processor.find_last_common_ancestor( @@ -123,16 +149,37 @@ def main(): # Fire the API and the ChainMonitor # FIXME: 92-block-data-during-bootstrap-db chain_monitor.monitor_chain() - API(watcher, config=config).start() + API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher).start() except Exception as e: logger.error("An error occurred: {}. Shutting down".format(e)) exit(1) if __name__ == "__main__": - opts, _ = getopt(argv[1:], "", [""]) - for opt, arg in opts: - # FIXME: Leaving this here for future option/arguments - pass + command_line_conf = {} + + try: + opts, _ = getopt( + argv[1:], "", ["btcnetwork=", "btcrpcuser=", "btcrpcpassword=", "btcrpcconnect=", "btcrpcport=", "datadir="] + ) + for opt, arg in opts: + if opt in ["--btcnetwork"]: + command_line_conf["BTC_NETWORK"] = arg + if opt in ["--btcrpcuser"]: + command_line_conf["BTC_RPC_USER"] = arg + if opt in ["--btcrpcpassword"]: + command_line_conf["BTC_RPC_PASSWD"] = arg + if opt in ["--btcrpcconnect"]: + command_line_conf["BTC_RPC_CONNECT"] = arg + if opt in ["--btcrpcport"]: + try: + command_line_conf["BTC_RPC_PORT"] = int(arg) + except ValueError: + exit("btcrpcport must be an integer") + if opt in ["--datadir"]: + DATA_DIR = os.path.expanduser(arg) + + except GetoptError as e: + exit(e) main() diff --git a/teos/tools.py b/teos/tools.py index 53b85cc..25e6f20 100644 --- a/teos/tools.py +++ b/teos/tools.py @@ -1,7 +1,6 @@ -from http.client import HTTPException from socket import timeout +from http.client import HTTPException -import teos.conf as conf from teos.utils.auth_proxy import AuthServiceProxy, JSONRPCException """ @@ -10,25 +9,38 @@ Tools is a module with general methods that can used by different entities in th # NOTCOVERED -def bitcoin_cli(): +def bitcoin_cli(btc_connect_params): """ An ``http`` connection with ``bitcoind`` using the ``json-rpc`` interface. + Args: + btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind + (rpc user, rpc passwd, host and port) + Returns: :obj:`AuthServiceProxy `: An authenticated service proxy to ``bitcoind`` that can be used to send ``json-rpc`` commands. """ return AuthServiceProxy( - "http://%s:%s@%s:%d" % (conf.BTC_RPC_USER, conf.BTC_RPC_PASSWD, conf.BTC_RPC_HOST, conf.BTC_RPC_PORT) + "http://%s:%s@%s:%d" + % ( + btc_connect_params.get("BTC_RPC_USER"), + btc_connect_params.get("BTC_RPC_PASSWD"), + btc_connect_params.get("BTC_RPC_CONNECT"), + btc_connect_params.get("BTC_RPC_PORT"), + ) ) # NOTCOVERED -def can_connect_to_bitcoind(): +def can_connect_to_bitcoind(btc_connect_params): """ Checks if the tower has connection to ``bitcoind``. + Args: + btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind + (rpc user, rpc passwd, host and port) Returns: :obj:`bool`: ``True`` if the connection can be established. ``False`` otherwise. """ @@ -36,18 +48,23 @@ def can_connect_to_bitcoind(): can_connect = True try: - bitcoin_cli().help() + bitcoin_cli(btc_connect_params).help() except (timeout, ConnectionRefusedError, JSONRPCException, HTTPException, OSError): can_connect = False return can_connect -def in_correct_network(network): +def in_correct_network(btc_connect_params, network): """ Checks if ``bitcoind`` and the tower are configured to run in the same network (``mainnet``, ``testnet`` or ``regtest``) + Args: + btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind + (rpc user, rpc passwd, host and port) + network (:obj:`str`): the network the tower is connected to. + Returns: :obj:`bool`: ``True`` if the network configuration matches. ``False`` otherwise. """ @@ -56,7 +73,7 @@ def in_correct_network(network): testnet3_genesis_block_hash = "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943" correct_network = False - genesis_block_hash = bitcoin_cli().getblockhash(0) + genesis_block_hash = bitcoin_cli(btc_connect_params).getblockhash(0) if network == "mainnet" and genesis_block_hash == mainnet_genesis_block_hash: correct_network = True diff --git a/teos/watcher.py b/teos/watcher.py index b732da8..33efa3c 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -3,15 +3,13 @@ from queue import Queue from threading import Thread import common.cryptographer -from common.cryptographer import Cryptographer -from common.appointment import Appointment -from common.tools import compute_locator - from common.logger import Logger +from common.tools import compute_locator +from common.appointment import Appointment +from common.cryptographer import Cryptographer from teos import LOG_PREFIX from teos.cleaner import Cleaner -from teos.block_processor import BlockProcessor logger = Logger(actor="Watcher", log_name_prefix=LOG_PREFIX) common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX) @@ -34,11 +32,12 @@ class Watcher: Args: db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the database. - sk_der (:obj:`bytes`): a DER encoded private key used to sign appointment receipts (signaling acceptance). - config (:obj:`dict`): a dictionary containing all the configuration parameters. Used locally to retrieve - ``MAX_APPOINTMENTS`` and ``EXPIRY_DELTA``. + block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance to + get block from bitcoind. responder (:obj:`Responder `): a ``Responder`` instance. - + sk_der (:obj:`bytes`): a DER encoded private key used to sign appointment receipts (signaling acceptance). + max_appointments (:obj:`int`): the maximum ammount of appointments accepted by the ``Watcher`` at the same time. + expiry_delta (:obj:`int`): the additional time the ``Watcher`` will keep an expired appointment around. Attributes: appointments (:obj:`dict`): a dictionary containing a simplification of the appointments (:obj:`Appointment @@ -48,23 +47,28 @@ class Watcher: appointments with the same ``locator``. block_queue (:obj:`Queue`): A queue used by the :obj:`Watcher` to receive block hashes from ``bitcoind``. It is populated by the :obj:`ChainMonitor `. - config (:obj:`dict`): a dictionary containing all the configuration parameters. Used locally to retrieve - ``MAX_APPOINTMENTS`` and ``EXPIRY_DELTA``. db_manager (:obj:`DBManager `): A db manager instance to interact with the database. + block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance to + get block from bitcoind. + responder (:obj:`Responder `): a ``Responder`` instance. signing_key (:mod:`PrivateKey`): a private key used to sign accepted appointments. + max_appointments (:obj:`int`): the maximum ammount of appointments accepted by the ``Watcher`` at the same time. + expiry_delta (:obj:`int`): the additional time the ``Watcher`` will keep an expired appointment around. Raises: ValueError: if `teos_sk_file` is not found. """ - def __init__(self, db_manager, responder, sk_der, config): + def __init__(self, db_manager, block_processor, responder, sk_der, max_appointments, expiry_delta): self.appointments = dict() self.locator_uuid_map = dict() self.block_queue = Queue() - self.config = config self.db_manager = db_manager + self.block_processor = block_processor self.responder = responder + self.max_appointments = max_appointments + self.expiry_delta = expiry_delta self.signing_key = Cryptographer.load_private_key_der(sk_der) def awake(self): @@ -102,7 +106,7 @@ class Watcher: """ - if len(self.appointments) < self.config.get("MAX_APPOINTMENTS"): + if len(self.appointments) < self.max_appointments: uuid = uuid4().hex self.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} @@ -139,7 +143,7 @@ class Watcher: while True: block_hash = self.block_queue.get() - block = BlockProcessor.get_block(block_hash) + block = self.block_processor.get_block(block_hash) logger.info("New block received", block_hash=block_hash, prev_block_hash=block.get("previousblockhash")) if len(self.appointments) > 0 and block is not None: @@ -148,7 +152,7 @@ class Watcher: expired_appointments = [ uuid for uuid, appointment_data in self.appointments.items() - if block["height"] > appointment_data.get("end_time") + self.config.get("EXPIRY_DELTA") + if block["height"] > appointment_data.get("end_time") + self.expiry_delta ] Cleaner.delete_expired_appointments( @@ -265,7 +269,7 @@ class Watcher: except ValueError: penalty_rawtx = None - penalty_tx = BlockProcessor.decode_raw_transaction(penalty_rawtx) + penalty_tx = self.block_processor.decode_raw_transaction(penalty_rawtx) decrypted_blobs[appointment.encrypted_blob.data] = (penalty_tx, penalty_rawtx) if penalty_tx is not None: