From 245d5b49f32f870f44c8ef8a078e063bf6d2116e Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 16:23:07 +0100 Subject: [PATCH 01/20] Updates config files Removes unused values from pisa config file and adds cli config file --- .gitignore | 1 + apps/cli/sample_conf.py | 13 +++++++++++++ pisa/sample_conf.py | 13 ++++--------- test/pisa/e2e/pisa-conf.py | 13 ++++--------- 4 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 apps/cli/sample_conf.py diff --git a/.gitignore b/.gitignore index 978a651..8d7a2c1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ test.py .coverage htmlcov docs/ +.pisa_btc diff --git a/apps/cli/sample_conf.py b/apps/cli/sample_conf.py new file mode 100644 index 0000000..d9f2b90 --- /dev/null +++ b/apps/cli/sample_conf.py @@ -0,0 +1,13 @@ +# PISA-SERVER +DEFAULT_PISA_API_SERVER = "btc.pisa.watch" +DEFAULT_PISA_API_PORT = 9814 + +# PISA-CLI +DATA_FOLDER = "~/.pisa_btc/" + +CLIENT_LOG_FILE = "pisa-cli.log" +APPOINTMENTS_FOLDER_NAME = "appointment_receipts" + +CLI_PUBLIC_KEY = "cli_pk.der" +CLI_PRIVATE_KEY = "cli_sk.der" +PISA_PUBLIC_KEY = "pisa_pk.der" diff --git a/pisa/sample_conf.py b/pisa/sample_conf.py index 8d08590..3c219c1 100644 --- a/pisa/sample_conf.py +++ b/pisa/sample_conf.py @@ -5,27 +5,22 @@ BTC_RPC_HOST = "localhost" BTC_RPC_PORT = 18443 BTC_NETWORK = "regtest" -# CHAIN MONITOR -POLLING_DELTA = 60 -BLOCK_WINDOW_SIZE = 10 - # ZMQ FEED_PROTOCOL = "tcp" FEED_ADDR = "127.0.0.1" FEED_PORT = 28332 # PISA +DATA_FOLDER = "~/.pisa_btc/" MAX_APPOINTMENTS = 100 EXPIRY_DELTA = 6 MIN_TO_SELF_DELAY = 20 SERVER_LOG_FILE = "pisa.log" PISA_SECRET_KEY = "pisa_sk.der" -# PISA-CLI -CLIENT_LOG_FILE = "pisa.log" - -# TEST -TEST_LOG_FILE = "test.log" +# CHAIN MONITOR +POLLING_DELTA = 60 +BLOCK_WINDOW_SIZE = 10 # LEVELDB DB_PATH = "appointments" diff --git a/test/pisa/e2e/pisa-conf.py b/test/pisa/e2e/pisa-conf.py index 83fe719..f53a81b 100644 --- a/test/pisa/e2e/pisa-conf.py +++ b/test/pisa/e2e/pisa-conf.py @@ -5,27 +5,22 @@ BTC_RPC_HOST = "localhost" BTC_RPC_PORT = 18445 BTC_NETWORK = "regtest" -# CHAIN MONITOR -POLLING_DELTA = 60 -BLOCK_WINDOW_SIZE = 10 - # ZMQ FEED_PROTOCOL = "tcp" FEED_ADDR = "127.0.0.1" FEED_PORT = 28335 # PISA +DATA_FOLDER = "~/.pisa_btc/" MAX_APPOINTMENTS = 100 EXPIRY_DELTA = 6 MIN_TO_SELF_DELAY = 20 SERVER_LOG_FILE = "pisa.log" PISA_SECRET_KEY = "pisa_sk.der" -# PISA-CLI -CLIENT_LOG_FILE = "pisa.log" - -# TEST -TEST_LOG_FILE = "test.log" +# CHAIN MONITOR +POLLING_DELTA = 60 +BLOCK_WINDOW_SIZE = 10 # LEVELDB DB_PATH = "appointments" From 418b7b49ab62f1b25f0af2676474f942e3aaae4e Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 16:27:18 +0100 Subject: [PATCH 02/20] Creates main function and updates config to add data folder pisad can now be run easier from other files (simplifies e2e testing) --- pisa/pisad.py | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/pisa/pisad.py b/pisa/pisad.py index 2643a19..6be2c98 100644 --- a/pisa/pisad.py +++ b/pisa/pisad.py @@ -1,8 +1,11 @@ +import os from getopt import getopt from sys import argv, exit from signal import signal, SIGINT, SIGQUIT, SIGTERM from common.logger import Logger +from common.tools import check_conf_fields, setup_data_folder + from pisa.api import API from pisa.watcher import Watcher from pisa.builder import Builder @@ -38,6 +41,12 @@ def load_config(config): conf_dict = {} + data_folder = config.DATA_FOLDER + if isinstance(data_folder, str): + data_folder = os.path.expanduser(data_folder) + else: + raise ValueError("The provided user folder is invalid.") + conf_fields = { "BTC_RPC_USER": {"value": config.BTC_RPC_USER, "type": str}, "BTC_RPC_PASSWD": {"value": config.BTC_RPC_PASSWD, "type": str}, @@ -47,43 +56,31 @@ def load_config(config): "FEED_PROTOCOL": {"value": config.FEED_PROTOCOL, "type": str}, "FEED_ADDR": {"value": config.FEED_ADDR, "type": str}, "FEED_PORT": {"value": config.FEED_PORT, "type": int}, + "DATA_FOLDER": {"value": data_folder, "type": str}, "MAX_APPOINTMENTS": {"value": config.MAX_APPOINTMENTS, "type": int}, "EXPIRY_DELTA": {"value": config.EXPIRY_DELTA, "type": int}, "MIN_TO_SELF_DELAY": {"value": config.MIN_TO_SELF_DELAY, "type": int}, - "SERVER_LOG_FILE": {"value": config.SERVER_LOG_FILE, "type": str}, - "PISA_SECRET_KEY": {"value": config.PISA_SECRET_KEY, "type": str}, - "CLIENT_LOG_FILE": {"value": config.CLIENT_LOG_FILE, "type": str}, - "TEST_LOG_FILE": {"value": config.TEST_LOG_FILE, "type": str}, - "DB_PATH": {"value": config.DB_PATH, "type": str}, + "SERVER_LOG_FILE": {"value": data_folder, "type": str}, + "PISA_SECRET_KEY": {"value": data_folder + config.PISA_SECRET_KEY, "type": str}, + "DB_PATH": {"value": data_folder + config.DB_PATH, "type": str}, } - for field in conf_fields: - value = conf_fields[field]["value"] - correct_type = conf_fields[field]["type"] - - if (value is not None) and isinstance(value, correct_type): - conf_dict[field] = value - else: - err_msg = "{} variable in config is of the wrong type".format(field) - logger.error(err_msg) - raise ValueError(err_msg) + check_conf_fields(conf_fields, logger) return conf_dict -if __name__ == "__main__": - logger.info("Starting PISA") +def main(): + global db_manager, chain_monitor signal(SIGINT, handle_signals) signal(SIGTERM, handle_signals) signal(SIGQUIT, handle_signals) - opts, _ = getopt(argv[1:], "", [""]) - for opt, arg in opts: - # FIXME: Leaving this here for future option/arguments - pass - pisa_config = load_config(conf) + logger.info("Starting PISA") + + setup_data_folder(pisa_config.get("DATA_FOLDER"), logger) db_manager = DBManager(pisa_config.get("DB_PATH")) if not can_connect_to_bitcoind(): @@ -155,3 +152,12 @@ if __name__ == "__main__": except Exception as e: logger.error("An error occurred: {}. Shutting down".format(e)) exit(1) + + +if __name__ == "__main__": + opts, _ = getopt(argv[1:], "", [""]) + for opt, arg in opts: + # FIXME: Leaving this here for future option/arguments + pass + + main() From c1ad1a4924a262362edd0ed60d9c2a6f638e71ac Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 16:30:09 +0100 Subject: [PATCH 03/20] Updates cli to run with config file --- apps/cli/pisa_cli.py | 72 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/apps/cli/pisa_cli.py b/apps/cli/pisa_cli.py index 5e7c4cf..742f562 100644 --- a/apps/cli/pisa_cli.py +++ b/apps/cli/pisa_cli.py @@ -11,19 +11,18 @@ from uuid import uuid4 from apps.cli.help import help_add_appointment, help_get_appointment from apps.cli.blob import Blob -from apps.cli import ( - DEFAULT_PISA_API_SERVER, - DEFAULT_PISA_API_PORT, - CLI_PUBLIC_KEY, - CLI_PRIVATE_KEY, - PISA_PUBLIC_KEY, - APPOINTMENTS_FOLDER_NAME, -) +import apps.cli.conf as conf from common.logger import Logger from common.appointment import Appointment from common.cryptographer import Cryptographer -from common.tools import check_sha256_hex_format, check_locator_format, compute_locator +from common.tools import ( + check_sha256_hex_format, + check_locator_format, + compute_locator, + check_conf_fields, + setup_data_folder, +) HTTP_OK = 200 @@ -54,6 +53,42 @@ def generate_dummy_appointment(): logger.info("\nData stored in dummy_appointment_data.json") +def load_config(config): + """ + Looks through all of the config options to make sure they contain the right type of data and builds a config + dictionary. + + Args: + config (:obj:`module`): It takes in a config module object. + + Returns: + :obj:`dict` A dictionary containing the config values. + """ + + conf_dict = {} + + data_folder = config.DATA_FOLDER + if isinstance(data_folder, str): + data_folder = os.path.expanduser(data_folder) + else: + raise ValueError("The provided user folder is invalid.") + + conf_fields = { + "DEFAULT_PISA_API_SERVER": {"value": config.DEFAULT_PISA_API_SERVER, "type": str}, + "DEFAULT_PISA_API_PORT": {"value": config.DEFAULT_PISA_API_PORT, "type": int}, + "DATA_FOLDER": {"value": data_folder, "type": str}, + "CLIENT_LOG_FILE": {"value": data_folder + config.CLIENT_LOG_FILE, "type": str}, + "APPOINTMENTS_FOLDER_NAME": {"value": data_folder + config.APPOINTMENTS_FOLDER_NAME, "type": str}, + "CLI_PUBLIC_KEY": {"value": data_folder + config.CLI_PUBLIC_KEY, "type": str}, + "CLI_PRIVATE_KEY": {"value": data_folder + config.CLI_PRIVATE_KEY, "type": str}, + "PISA_PUBLIC_KEY": {"value": data_folder + config.PISA_PUBLIC_KEY, "type": str}, + } + + check_conf_fields(conf_fields, logger) + + return conf_dict + + # Loads and returns Pisa keys from disk def load_key_file_data(file_name): try: @@ -73,13 +108,13 @@ def load_key_file_data(file_name): # Makes sure that the folder APPOINTMENTS_FOLDER_NAME exists, then saves the appointment and signature in it. def save_signed_appointment(appointment, signature): # Create the appointments directory if it doesn't already exist - os.makedirs(APPOINTMENTS_FOLDER_NAME, exist_ok=True) + os.makedirs(config.get("APPOINTMENTS_FOLDER_NAME"), exist_ok=True) timestamp = int(time.time()) locator = appointment["locator"] uuid = uuid4().hex # prevent filename collisions - filename = "{}/appointment-{}-{}-{}.json".format(APPOINTMENTS_FOLDER_NAME, timestamp, locator, uuid) + filename = "{}/appointment-{}-{}-{}.json".format(config.get("APPOINTMENTS_FOLDER_NAME"), timestamp, locator, uuid) data = {"appointment": appointment, "signature": signature} with open(filename, "w") as f: @@ -233,7 +268,7 @@ def post_data_to_add_appointment_endpoint(data): # Verify that the signature returned from the watchtower is valid. def check_signature(signature, appointment): try: - pisa_pk_der = load_key_file_data(PISA_PUBLIC_KEY) + pisa_pk_der = load_key_file_data(config.get("PISA_PUBLIC_KEY")) pisa_pk = Cryptographer.load_public_key_der(pisa_pk_der) if pisa_pk is None: @@ -287,7 +322,7 @@ def get_appointment(args): def get_appointment_signature(appointment): try: - sk_der = load_key_file_data(CLI_PRIVATE_KEY) + sk_der = load_key_file_data(config.get("CLI_PRIVATE_KEY")) cli_sk = Cryptographer.load_private_key_der(sk_der) signature = Cryptographer.sign(appointment.serialize(), cli_sk) @@ -309,7 +344,7 @@ def get_appointment_signature(appointment): def get_pk(): try: - cli_pk_der = load_key_file_data(CLI_PUBLIC_KEY) + cli_pk_der = load_key_file_data(config.get("CLI_PUBLIC_KEY")) hex_pk_der = binascii.hexlify(cli_pk_der) return hex_pk_der @@ -345,11 +380,16 @@ def show_usage(): if __name__ == "__main__": - pisa_api_server = DEFAULT_PISA_API_SERVER - pisa_api_port = DEFAULT_PISA_API_PORT + config = load_config(conf) + + pisa_api_server = config.get("DEFAULT_PISA_API_SERVER") + pisa_api_port = config.get("DEFAULT_PISA_API_PORT") commands = ["add_appointment", "get_appointment", "help"] testing_commands = ["generate_dummy_appointment"] + # Create user folder if missing + setup_data_folder(config.get("DATA_FOLDER"), logger) + try: opts, args = getopt(argv[1:], "s:p:h", ["server", "port", "help"]) From a3f2d20499bbee501bae995a923416fa3292cedf Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:05:44 +0100 Subject: [PATCH 04/20] Moves/adds methods to deal with config to common --- common/tools.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/common/tools.py b/common/tools.py index d208272..6c2d0c2 100644 --- a/common/tools.py +++ b/common/tools.py @@ -1,4 +1,5 @@ import re +import os from common.constants import LOCATOR_LEN_HEX @@ -38,3 +39,33 @@ def compute_locator(tx_id): """ return tx_id[:LOCATOR_LEN_HEX] + + +def setup_data_folder(data_folder, logger): + if not os.path.isdir(data_folder): + logger.info("Data folder not found. Creating it") + os.makedirs(data_folder, exist_ok=True) + + +def check_conf_fields(conf_fields): + conf_dict = {} + + for field in conf_fields: + value = conf_fields[field]["value"] + correct_type = conf_fields[field]["type"] + + if (value is not None) and isinstance(value, correct_type): + conf_dict[field] = value + else: + err_msg = "{} variable in config is of the wrong type".format(field) + raise ValueError(err_msg) + + return conf_dict + + +def extend_paths(base_path, config_fields): + for key, field in config_fields.items(): + if field.get("path"): + config_fields[key]["value"] = base_path + config_fields[key]["value"] + + return config_fields From fddf2e6968ba849077787fdded6fba62878f3b7c Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:13:30 +0100 Subject: [PATCH 05/20] Fixes logger not properly working for cli The Logger was set to use c_logger and f_logger from pisad, so the cli file logs were never created --- common/logger.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/common/logger.py b/common/logger.py index ac683c2..b175ebf 100644 --- a/common/logger.py +++ b/common/logger.py @@ -1,8 +1,7 @@ import json +import logging from datetime import datetime -from pisa import f_logger, c_logger - class _StructuredMessage: def __init__(self, message, **kwargs): @@ -22,8 +21,10 @@ class Logger: actor (:obj:`str`): the system actor that is logging the event (e.g. ``Watcher``, ``Cryptographer``, ...). """ - def __init__(self, actor=None): + def __init__(self, log_name_prefix, actor=None): self.actor = actor + self.f_logger = logging.getLogger("{}_file_log".format(log_name_prefix)) + self.c_logger = logging.getLogger("{}_console_log".format(log_name_prefix)) def _add_prefix(self, msg): return msg if self.actor is None else "[{}]: {}".format(self.actor, msg) @@ -54,8 +55,8 @@ class Logger: kwargs: a ``key:value`` collection parameters to be added to the output. """ - f_logger.info(self._create_file_message(msg, **kwargs)) - c_logger.info(self._create_console_message(msg, **kwargs)) + self.f_logger.info(self._create_file_message(msg, **kwargs)) + self.c_logger.info(self._create_console_message(msg, **kwargs)) def debug(self, msg, **kwargs): """ @@ -66,8 +67,8 @@ class Logger: kwargs: a ``key:value`` collection parameters to be added to the output. """ - f_logger.debug(self._create_file_message(msg, **kwargs)) - c_logger.debug(self._create_console_message(msg, **kwargs)) + self.f_logger.debug(self._create_file_message(msg, **kwargs)) + self.c_logger.debug(self._create_console_message(msg, **kwargs)) def error(self, msg, **kwargs): """ @@ -78,8 +79,8 @@ class Logger: kwargs: a ``key:value`` collection parameters to be added to the output. """ - f_logger.error(self._create_file_message(msg, **kwargs)) - c_logger.error(self._create_console_message(msg, **kwargs)) + self.f_logger.error(self._create_file_message(msg, **kwargs)) + self.c_logger.error(self._create_console_message(msg, **kwargs)) def warning(self, msg, **kwargs): """ @@ -90,5 +91,5 @@ class Logger: kwargs: a ``key:value`` collection parameters to be added to the output. """ - f_logger.warning(self._create_file_message(msg, **kwargs)) - c_logger.warning(self._create_console_message(msg, **kwargs)) + self.f_logger.warning(self._create_file_message(msg, **kwargs)) + self.c_logger.warning(self._create_console_message(msg, **kwargs)) From 1f46e6eb26749cf3b633753f257ead414b3cc887 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:14:34 +0100 Subject: [PATCH 06/20] Moves logging setup to common The setup it's identical for cli and pisad --- common/tools.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/common/tools.py b/common/tools.py index 6c2d0c2..ec02451 100644 --- a/common/tools.py +++ b/common/tools.py @@ -1,5 +1,6 @@ import re import os +import logging from common.constants import LOCATOR_LEN_HEX @@ -69,3 +70,32 @@ def extend_paths(base_path, config_fields): config_fields[key]["value"] = base_path + config_fields[key]["value"] return config_fields + + +def setup_logging(log_file_path, log_name_prefix): + if not isinstance(log_file_path, str): + print(log_file_path) + raise ValueError("Wrong log file path.") + + if not isinstance(log_name_prefix, str): + raise ValueError("Wrong log file name.") + + # Create the file logger + f_logger = logging.getLogger("{}_file_log".format(log_name_prefix)) + f_logger.setLevel(logging.INFO) + + fh = logging.FileHandler(log_file_path) + fh.setLevel(logging.INFO) + fh_formatter = logging.Formatter("%(message)s") + fh.setFormatter(fh_formatter) + f_logger.addHandler(fh) + + # Create the console logger + c_logger = logging.getLogger("{}_console_log".format(log_name_prefix)) + c_logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.INFO) + ch_formatter = logging.Formatter("%(message)s.", "%Y-%m-%d %H:%M:%S") + ch.setFormatter(ch_formatter) + c_logger.addHandler(ch) From 6884db9f58abb3d7b827a8e2ea8256f09f11ba86 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:16:25 +0100 Subject: [PATCH 07/20] Updates logger codebase-wise to match the Logger updates Logger instances now specify the logger prefix so the logger can be properly loaded, fixing the issues with the cli file logger --- pisa/api.py | 5 +++-- pisa/block_processor.py | 4 +++- pisa/carrier.py | 3 ++- pisa/chain_monitor.py | 3 ++- pisa/cleaner.py | 4 +++- pisa/db_manager.py | 4 +++- pisa/inspector.py | 4 ++-- pisa/responder.py | 3 ++- pisa/watcher.py | 3 ++- 9 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pisa/api.py b/pisa/api.py index a70ca34..e31d0d2 100644 --- a/pisa/api.py +++ b/pisa/api.py @@ -1,8 +1,9 @@ import os import json +import logging from flask import Flask, request, abort, jsonify -from pisa import HOST, PORT, logging +from pisa import HOST, PORT, LOG_PREFIX from common.logger import Logger from pisa.inspector import Inspector from common.appointment import Appointment @@ -13,7 +14,7 @@ from common.constants import HTTP_OK, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE # ToDo: #5-add-async-to-api app = Flask(__name__) -logger = Logger("API") +logger = Logger(actor="API", log_name_prefix=LOG_PREFIX) class API: diff --git a/pisa/block_processor.py b/pisa/block_processor.py index 1970b42..c5a7dd1 100644 --- a/pisa/block_processor.py +++ b/pisa/block_processor.py @@ -1,8 +1,10 @@ from common.logger import Logger + +from pisa import LOG_PREFIX from pisa.tools import bitcoin_cli from pisa.utils.auth_proxy import JSONRPCException -logger = Logger("BlockProcessor") +logger = Logger(actor="BlockProcessor", log_name_prefix=LOG_PREFIX) class BlockProcessor: diff --git a/pisa/carrier.py b/pisa/carrier.py index 00602e9..dec4ba6 100644 --- a/pisa/carrier.py +++ b/pisa/carrier.py @@ -1,10 +1,11 @@ +from pisa import LOG_PREFIX from pisa.rpc_errors import * from common.logger import Logger from pisa.tools import bitcoin_cli from pisa.utils.auth_proxy import JSONRPCException from pisa.errors import UNKNOWN_JSON_RPC_EXCEPTION, RPC_TX_REORGED_AFTER_BROADCAST -logger = Logger("Carrier") +logger = Logger(actor="Carrier", log_name_prefix=LOG_PREFIX) # FIXME: This class is not fully covered by unit tests diff --git a/pisa/chain_monitor.py b/pisa/chain_monitor.py index 689a223..22ef377 100644 --- a/pisa/chain_monitor.py +++ b/pisa/chain_monitor.py @@ -2,11 +2,12 @@ import zmq import binascii from threading import Thread, Event, Condition +from pisa import LOG_PREFIX from common.logger import Logger from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT, POLLING_DELTA, BLOCK_WINDOW_SIZE from pisa.block_processor import BlockProcessor -logger = Logger("ChainMonitor") +logger = Logger(actor="ChainMonitor", log_name_prefix=LOG_PREFIX) class ChainMonitor: diff --git a/pisa/cleaner.py b/pisa/cleaner.py index d2e6925..777834c 100644 --- a/pisa/cleaner.py +++ b/pisa/cleaner.py @@ -1,6 +1,8 @@ +from pisa import LOG_PREFIX + from common.logger import Logger -logger = Logger("Cleaner") +logger = Logger(actor="Cleaner", log_name_prefix=LOG_PREFIX) class Cleaner: diff --git a/pisa/db_manager.py b/pisa/db_manager.py index 2c693b6..983a26e 100644 --- a/pisa/db_manager.py +++ b/pisa/db_manager.py @@ -1,9 +1,11 @@ import json import plyvel +from pisa import LOG_PREFIX + from common.logger import Logger -logger = Logger("DBManager") +logger = Logger(actor="DBManager", log_name_prefix=LOG_PREFIX) WATCHER_PREFIX = "w" WATCHER_LAST_BLOCK_KEY = "bw" diff --git a/pisa/inspector.py b/pisa/inspector.py index fcc570e..dfdb0a8 100644 --- a/pisa/inspector.py +++ b/pisa/inspector.py @@ -4,12 +4,12 @@ from binascii import unhexlify from common.constants import LOCATOR_LEN_HEX from common.cryptographer import Cryptographer -from pisa import errors +from pisa import errors, LOG_PREFIX from common.logger import Logger from common.appointment import Appointment from pisa.block_processor import BlockProcessor -logger = Logger("Inspector") +logger = Logger(actor="Inspector", log_name_prefix=LOG_PREFIX) # FIXME: The inspector logs the wrong messages sent form the users. A possible attack surface would be to send a really # long field that, even if not accepted by PISA, would be stored in the logs. This is a possible DoS surface diff --git a/pisa/responder.py b/pisa/responder.py index 5d4ac9d..1198553 100644 --- a/pisa/responder.py +++ b/pisa/responder.py @@ -2,6 +2,7 @@ import json from queue import Queue from threading import Thread +from pisa import LOG_PREFIX from common.logger import Logger from pisa.cleaner import Cleaner from pisa.carrier import Carrier @@ -10,7 +11,7 @@ from pisa.block_processor import BlockProcessor CONFIRMATIONS_BEFORE_RETRY = 6 MIN_CONFIRMATIONS = 6 -logger = Logger("Responder") +logger = Logger(actor="Responder", log_name_prefix=LOG_PREFIX) class TransactionTracker: diff --git a/pisa/watcher.py b/pisa/watcher.py index c0e852a..86a5c40 100644 --- a/pisa/watcher.py +++ b/pisa/watcher.py @@ -8,11 +8,12 @@ from common.tools import compute_locator from common.logger import Logger +from pisa import LOG_PREFIX from pisa.cleaner import Cleaner from pisa.responder import Responder from pisa.block_processor import BlockProcessor -logger = Logger("Watcher") +logger = Logger(actor="Watcher", log_name_prefix=LOG_PREFIX) class Watcher: From 00a989e1b23577ae9d0b13a9f96c974283ddcdc9 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:27:04 +0100 Subject: [PATCH 08/20] Updates pisad and __init__ to put together the log and config fixes The setup of the logs and the parsing of the config file are closely related. The former need info from the later to be created, and needs to be setup only once per pisa instance. In the same way, the later need to only be loaded and validated once per pisa intance and contains info to setup the logs. Intead of setting up the logs in init and loading the config file in pisad, now both are dealt with in __init__ --- pisa/__init__.py | 48 +++++++++++++++++++++--------------- pisa/pisad.py | 63 +++++++----------------------------------------- 2 files changed, 38 insertions(+), 73 deletions(-) diff --git a/pisa/__init__.py b/pisa/__init__.py index dd06913..e7d380b 100644 --- a/pisa/__init__.py +++ b/pisa/__init__.py @@ -1,27 +1,37 @@ -import logging - -from pisa.utils.auth_proxy import AuthServiceProxy +import os import pisa.conf as conf +from common.tools import check_conf_fields, setup_logging, extend_paths +from pisa.utils.auth_proxy import AuthServiceProxy HOST = "localhost" PORT = 9814 +LOG_PREFIX = "pisa" -# Create the file logger -f_logger = logging.getLogger("pisa_file_log") -f_logger.setLevel(logging.INFO) +# Load config fields +conf_fields = { + "BTC_RPC_USER": {"value": conf.BTC_RPC_USER, "type": str}, + "BTC_RPC_PASSWD": {"value": conf.BTC_RPC_PASSWD, "type": str}, + "BTC_RPC_HOST": {"value": conf.BTC_RPC_HOST, "type": str}, + "BTC_RPC_PORT": {"value": conf.BTC_RPC_PORT, "type": int}, + "BTC_NETWORK": {"value": conf.BTC_NETWORK, "type": str}, + "FEED_PROTOCOL": {"value": conf.FEED_PROTOCOL, "type": str}, + "FEED_ADDR": {"value": conf.FEED_ADDR, "type": str}, + "FEED_PORT": {"value": conf.FEED_PORT, "type": int}, + "DATA_FOLDER": {"value": conf.DATA_FOLDER, "type": str}, + "MAX_APPOINTMENTS": {"value": conf.MAX_APPOINTMENTS, "type": int}, + "EXPIRY_DELTA": {"value": conf.EXPIRY_DELTA, "type": int}, + "MIN_TO_SELF_DELAY": {"value": conf.MIN_TO_SELF_DELAY, "type": int}, + "SERVER_LOG_FILE": {"value": conf.SERVER_LOG_FILE, "type": str, "path": True}, + "PISA_SECRET_KEY": {"value": conf.PISA_SECRET_KEY, "type": str, "path": True}, + "DB_PATH": {"value": conf.DB_PATH, "type": str, "path": True}, +} -fh = logging.FileHandler(conf.SERVER_LOG_FILE) -fh.setLevel(logging.INFO) -fh_formatter = logging.Formatter("%(message)s") -fh.setFormatter(fh_formatter) -f_logger.addHandler(fh) +# Expand user (~) if found and check fields are correct +conf_fields["DATA_FOLDER"]["value"] = os.path.expanduser(conf_fields["DATA_FOLDER"]["value"]) +# Extend relative paths +conf_fields = extend_paths(conf_fields["DATA_FOLDER"]["value"], conf_fields) -# Create the console logger -c_logger = logging.getLogger("pisa_console_log") -c_logger.setLevel(logging.INFO) +# Sanity check fields and build config dictionary +config = check_conf_fields(conf_fields) -ch = logging.StreamHandler() -ch.setLevel(logging.INFO) -ch_formatter = logging.Formatter("%(message)s.", "%Y-%m-%d %H:%M:%S") -ch.setFormatter(ch_formatter) -c_logger.addHandler(ch) +setup_logging(config.get("SERVER_LOG_FILE"), LOG_PREFIX) diff --git a/pisa/pisad.py b/pisa/pisad.py index 6be2c98..1c35000 100644 --- a/pisa/pisad.py +++ b/pisa/pisad.py @@ -1,21 +1,20 @@ -import os from getopt import getopt from sys import argv, exit from signal import signal, SIGINT, SIGQUIT, SIGTERM from common.logger import Logger -from common.tools import check_conf_fields, setup_data_folder +from common.tools import setup_data_folder +from pisa import config, LOG_PREFIX from pisa.api import API from pisa.watcher import Watcher from pisa.builder import Builder -import pisa.conf as conf from pisa.db_manager import DBManager from pisa.chain_monitor import ChainMonitor from pisa.block_processor import BlockProcessor from pisa.tools import can_connect_to_bitcoind, in_correct_network -logger = Logger("Daemon") +logger = Logger(actor="Daemon", log_name_prefix=LOG_PREFIX) def handle_signals(signal_received, frame): @@ -27,49 +26,6 @@ def handle_signals(signal_received, frame): exit(0) -def load_config(config): - """ - Looks through all of the config options to make sure they contain the right type of data and builds a config - dictionary. - - Args: - config (:obj:`module`): It takes in a config module object. - - Returns: - :obj:`dict` A dictionary containing the config values. - """ - - conf_dict = {} - - data_folder = config.DATA_FOLDER - if isinstance(data_folder, str): - data_folder = os.path.expanduser(data_folder) - else: - raise ValueError("The provided user folder is invalid.") - - conf_fields = { - "BTC_RPC_USER": {"value": config.BTC_RPC_USER, "type": str}, - "BTC_RPC_PASSWD": {"value": config.BTC_RPC_PASSWD, "type": str}, - "BTC_RPC_HOST": {"value": config.BTC_RPC_HOST, "type": str}, - "BTC_RPC_PORT": {"value": config.BTC_RPC_PORT, "type": int}, - "BTC_NETWORK": {"value": config.BTC_NETWORK, "type": str}, - "FEED_PROTOCOL": {"value": config.FEED_PROTOCOL, "type": str}, - "FEED_ADDR": {"value": config.FEED_ADDR, "type": str}, - "FEED_PORT": {"value": config.FEED_PORT, "type": int}, - "DATA_FOLDER": {"value": data_folder, "type": str}, - "MAX_APPOINTMENTS": {"value": config.MAX_APPOINTMENTS, "type": int}, - "EXPIRY_DELTA": {"value": config.EXPIRY_DELTA, "type": int}, - "MIN_TO_SELF_DELAY": {"value": config.MIN_TO_SELF_DELAY, "type": int}, - "SERVER_LOG_FILE": {"value": data_folder, "type": str}, - "PISA_SECRET_KEY": {"value": data_folder + config.PISA_SECRET_KEY, "type": str}, - "DB_PATH": {"value": data_folder + config.DB_PATH, "type": str}, - } - - check_conf_fields(conf_fields, logger) - - return conf_dict - - def main(): global db_manager, chain_monitor @@ -77,16 +33,15 @@ def main(): signal(SIGTERM, handle_signals) signal(SIGQUIT, handle_signals) - pisa_config = load_config(conf) logger.info("Starting PISA") - setup_data_folder(pisa_config.get("DATA_FOLDER"), logger) - db_manager = DBManager(pisa_config.get("DB_PATH")) + setup_data_folder(config.get("DATA_FOLDER"), logger) + db_manager = DBManager(config.get("DB_PATH")) if not can_connect_to_bitcoind(): logger.error("Can't connect to bitcoind. Shutting down") - elif not in_correct_network(pisa_config.get("BTC_NETWORK")): + elif not in_correct_network(config.get("BTC_NETWORK")): logger.error("bitcoind is running on a different network, check conf.py and bitcoin.conf. Shutting down") else: @@ -98,10 +53,10 @@ def main(): watcher_appointments_data = db_manager.load_watcher_appointments() responder_trackers_data = db_manager.load_responder_trackers() - with open(pisa_config.get("PISA_SECRET_KEY"), "rb") as key_file: + with open(config.get("PISA_SECRET_KEY"), "rb") as key_file: secret_key_der = key_file.read() - watcher = Watcher(db_manager, chain_monitor, secret_key_der, pisa_config) + watcher = Watcher(db_manager, chain_monitor, secret_key_der, config) chain_monitor.attach_watcher(watcher.block_queue, watcher.asleep) chain_monitor.attach_responder(watcher.responder.block_queue, watcher.responder.asleep) @@ -147,7 +102,7 @@ def main(): watcher.block_queue = Builder.build_block_queue(missed_blocks_watcher) # Fire the API - API(watcher, config=pisa_config).start() + API(watcher, config=config).start() except Exception as e: logger.error("An error occurred: {}. Shutting down".format(e)) From 836048c54d0fe91e693f7228124a29ef8e4fa17b Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:31:47 +0100 Subject: [PATCH 09/20] Same as 00a989e1b23577ae9d0b13a9f96c974283ddcdc9 but for the cli --- apps/cli/__init__.py | 50 +++++++++++++++++++------------------------- apps/cli/pisa_cli.py | 50 +++----------------------------------------- 2 files changed, 25 insertions(+), 75 deletions(-) diff --git a/apps/cli/__init__.py b/apps/cli/__init__.py index 2c9e149..df7cf41 100644 --- a/apps/cli/__init__.py +++ b/apps/cli/__init__.py @@ -1,33 +1,27 @@ -import logging +import os +import apps.cli.conf as conf +from common.tools import extend_paths, check_conf_fields, setup_logging -# PISA-SERVER -DEFAULT_PISA_API_SERVER = "btc.pisa.watch" -DEFAULT_PISA_API_PORT = 9814 +LOG_PREFIX = "cli" -# PISA-CLI -CLIENT_LOG_FILE = "pisa-cli.log" -APPOINTMENTS_FOLDER_NAME = "appointments" +# Load config fields +conf_fields = { + "DEFAULT_PISA_API_SERVER": {"value": conf.DEFAULT_PISA_API_SERVER, "type": str}, + "DEFAULT_PISA_API_PORT": {"value": conf.DEFAULT_PISA_API_PORT, "type": int}, + "DATA_FOLDER": {"value": conf.DATA_FOLDER, "type": str}, + "CLIENT_LOG_FILE": {"value": conf.CLIENT_LOG_FILE, "type": str, "path": True}, + "APPOINTMENTS_FOLDER_NAME": {"value": conf.APPOINTMENTS_FOLDER_NAME, "type": str, "path": True}, + "CLI_PUBLIC_KEY": {"value": conf.CLI_PUBLIC_KEY, "type": str, "path": True}, + "CLI_PRIVATE_KEY": {"value": conf.CLI_PRIVATE_KEY, "type": str, "path": True}, + "PISA_PUBLIC_KEY": {"value": conf.PISA_PUBLIC_KEY, "type": str, "path": True}, +} -CLI_PUBLIC_KEY = "cli_pk.der" -CLI_PRIVATE_KEY = "cli_sk.der" -PISA_PUBLIC_KEY = "pisa_pk.der" +# Expand user (~) if found and check fields are correct +conf_fields["DATA_FOLDER"]["value"] = os.path.expanduser(conf_fields["DATA_FOLDER"]["value"]) +# Extend relative paths +conf_fields = extend_paths(conf_fields["DATA_FOLDER"]["value"], conf_fields) -# Create the file logger -f_logger = logging.getLogger("cli_file_log") -f_logger.setLevel(logging.INFO) +# Sanity check fields and build config dictionary +config = check_conf_fields(conf_fields) -fh = logging.FileHandler(CLIENT_LOG_FILE) -fh.setLevel(logging.INFO) -fh_formatter = logging.Formatter("%(message)s") -fh.setFormatter(fh_formatter) -f_logger.addHandler(fh) - -# Create the console logger -c_logger = logging.getLogger("cli_console_log") -c_logger.setLevel(logging.INFO) - -ch = logging.StreamHandler() -ch.setLevel(logging.INFO) -ch_formatter = logging.Formatter("%(asctime)s %(message)s.", "%Y-%m-%d %H:%M:%S") -ch.setFormatter(ch_formatter) -c_logger.addHandler(ch) +setup_logging(config.get("CLIENT_LOG_FILE"), LOG_PREFIX) diff --git a/apps/cli/pisa_cli.py b/apps/cli/pisa_cli.py index 742f562..6bc8346 100644 --- a/apps/cli/pisa_cli.py +++ b/apps/cli/pisa_cli.py @@ -9,24 +9,18 @@ from getopt import getopt, GetoptError from requests import ConnectTimeout, ConnectionError from uuid import uuid4 +from apps.cli import config, LOG_PREFIX from apps.cli.help import help_add_appointment, help_get_appointment from apps.cli.blob import Blob -import apps.cli.conf as conf from common.logger import Logger from common.appointment import Appointment from common.cryptographer import Cryptographer -from common.tools import ( - check_sha256_hex_format, - check_locator_format, - compute_locator, - check_conf_fields, - setup_data_folder, -) +from common.tools import check_sha256_hex_format, check_locator_format, compute_locator, setup_data_folder HTTP_OK = 200 -logger = Logger("Client") +logger = Logger(actor="Client", log_name_prefix=LOG_PREFIX) # FIXME: TESTING ENDPOINT, WON'T BE THERE IN PRODUCTION @@ -53,42 +47,6 @@ def generate_dummy_appointment(): logger.info("\nData stored in dummy_appointment_data.json") -def load_config(config): - """ - Looks through all of the config options to make sure they contain the right type of data and builds a config - dictionary. - - Args: - config (:obj:`module`): It takes in a config module object. - - Returns: - :obj:`dict` A dictionary containing the config values. - """ - - conf_dict = {} - - data_folder = config.DATA_FOLDER - if isinstance(data_folder, str): - data_folder = os.path.expanduser(data_folder) - else: - raise ValueError("The provided user folder is invalid.") - - conf_fields = { - "DEFAULT_PISA_API_SERVER": {"value": config.DEFAULT_PISA_API_SERVER, "type": str}, - "DEFAULT_PISA_API_PORT": {"value": config.DEFAULT_PISA_API_PORT, "type": int}, - "DATA_FOLDER": {"value": data_folder, "type": str}, - "CLIENT_LOG_FILE": {"value": data_folder + config.CLIENT_LOG_FILE, "type": str}, - "APPOINTMENTS_FOLDER_NAME": {"value": data_folder + config.APPOINTMENTS_FOLDER_NAME, "type": str}, - "CLI_PUBLIC_KEY": {"value": data_folder + config.CLI_PUBLIC_KEY, "type": str}, - "CLI_PRIVATE_KEY": {"value": data_folder + config.CLI_PRIVATE_KEY, "type": str}, - "PISA_PUBLIC_KEY": {"value": data_folder + config.PISA_PUBLIC_KEY, "type": str}, - } - - check_conf_fields(conf_fields, logger) - - return conf_dict - - # Loads and returns Pisa keys from disk def load_key_file_data(file_name): try: @@ -380,8 +338,6 @@ def show_usage(): if __name__ == "__main__": - config = load_config(conf) - pisa_api_server = config.get("DEFAULT_PISA_API_SERVER") pisa_api_port = config.get("DEFAULT_PISA_API_PORT") commands = ["add_appointment", "get_appointment", "help"] From 3185ae124d5e2240c220009fb35e3c8b6b06b01b Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:34:53 +0100 Subject: [PATCH 10/20] Fixes paths on cli tests --- test/apps/cli/unit/test_pisa_cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/apps/cli/unit/test_pisa_cli.py b/test/apps/cli/unit/test_pisa_cli.py index 4927c7a..0ce40c6 100644 --- a/test/apps/cli/unit/test_pisa_cli.py +++ b/test/apps/cli/unit/test_pisa_cli.py @@ -154,12 +154,13 @@ def test_load_key_file_data(): def test_save_signed_appointment(monkeypatch): - monkeypatch.setattr(pisa_cli, "APPOINTMENTS_FOLDER_NAME", "test_appointments") + appointments_folder = "test_appointments_receipts" + pisa_cli.config["APPOINTMENTS_FOLDER_NAME"] = appointments_folder pisa_cli.save_signed_appointment(dummy_appointment.to_dict(), get_dummy_signature()) # In folder "Appointments," grab all files and print them. - files = os.listdir("test_appointments") + files = os.listdir(appointments_folder) found = False for f in files: @@ -169,10 +170,10 @@ def test_save_signed_appointment(monkeypatch): assert found # If "appointments" directory doesn't exist, function should create it. - assert os.path.exists("test_appointments") + assert os.path.exists(appointments_folder) # Delete test directory once we're done. - shutil.rmtree("test_appointments") + shutil.rmtree(appointments_folder) def test_parse_add_appointment_args(): From 9be2c2475fa4c3529f3e040098b8e2ad9438c1cc Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:35:32 +0100 Subject: [PATCH 11/20] Removes logger instance from test_appointment --- test/common/unit/test_appointment.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/common/unit/test_appointment.py b/test/common/unit/test_appointment.py index 2dea9b0..8087138 100644 --- a/test/common/unit/test_appointment.py +++ b/test/common/unit/test_appointment.py @@ -3,7 +3,6 @@ import struct import binascii from pytest import fixture -from pisa import c_logger from common.appointment import Appointment from pisa.encrypted_blob import EncryptedBlob @@ -12,9 +11,6 @@ from test.pisa.unit.conftest import get_random_value_hex from common.constants import LOCATOR_LEN_BYTES -c_logger.disabled = True - - # Not much to test here, adding it for completeness @fixture def appointment_data(): From 96ab0039e7aab5d09896dad544d22af00b5af652 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 18:36:18 +0100 Subject: [PATCH 12/20] Updates config file params for testing --- test/pisa/unit/conftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/pisa/unit/conftest.py b/test/pisa/unit/conftest.py index e70d2c7..3e373e1 100644 --- a/test/pisa/unit/conftest.py +++ b/test/pisa/unit/conftest.py @@ -1,3 +1,4 @@ +import os import pytest import random import requests @@ -161,6 +162,7 @@ def generate_dummy_tracker(): def get_config(): + data_folder = os.path.expanduser("~/.pisa_btc") config = { "BTC_RPC_USER": "username", "BTC_RPC_PASSWD": "password", @@ -170,13 +172,12 @@ def get_config(): "FEED_PROTOCOL": "tcp", "FEED_ADDR": "127.0.0.1", "FEED_PORT": 28332, + "DATA_FOLDER": data_folder, "MAX_APPOINTMENTS": 100, "EXPIRY_DELTA": 6, "MIN_TO_SELF_DELAY": 20, - "SERVER_LOG_FILE": "pisa.log", - "PISA_SECRET_KEY": "pisa_sk.der", - "CLIENT_LOG_FILE": "pisa.log", - "TEST_LOG_FILE": "test.log", + "SERVER_LOG_FILE": data_folder + "pisa.log", + "PISA_SECRET_KEY": data_folder + "pisa_sk.der", "DB_PATH": "appointments", } From dce7b4d39ed0e351baad505c028e966a51e4a0de Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 19:00:55 +0100 Subject: [PATCH 13/20] Adds missing tools docs --- common/tools.py | 57 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/common/tools.py b/common/tools.py index ec02451..8e0617c 100644 --- a/common/tools.py +++ b/common/tools.py @@ -12,7 +12,7 @@ def check_sha256_hex_format(value): value(:mod:`str`): the value to be checked. Returns: - :mod:`bool`: Whether or not the value matches the format. + :obj:`bool`: Whether or not the value matches the format. """ return isinstance(value, str) and re.match(r"^[0-9A-Fa-f]{64}$", value) is not None @@ -25,7 +25,7 @@ def check_locator_format(value): value(:mod:`str`): the value to be checked. Returns: - :mod:`bool`: Whether or not the value matches the format. + :obj:`bool`: Whether or not the value matches the format. """ return isinstance(value, str) and re.match(r"^[0-9A-Fa-f]{32}$", value) is not None @@ -36,19 +36,44 @@ def compute_locator(tx_id): Args: tx_id (:obj:`str`): the transaction id used to compute the locator. Returns: - (:obj:`str`): The computed locator. + :obj:`str`: The computed locator. """ return tx_id[:LOCATOR_LEN_HEX] def setup_data_folder(data_folder, logger): + """ + Create a data folder for either the client or the server side if the folder does not exists. + + Args: + data_folder (:obj:`str`): the path of the folder + logger (:obj: `Logger `): a logger instance to notify about the folder creation. + """ + if not os.path.isdir(data_folder): logger.info("Data folder not found. Creating it") os.makedirs(data_folder, exist_ok=True) def check_conf_fields(conf_fields): + """ + Checks that the provided configuration field have the right type. + + Args: + conf_fields (:obj:`dict`): a dictionary populated with the configuration file params and the expected types. + The format is as follows: + + {"field0": {"value": value_from_conf_file, "type": expected_type, ...}} + + Returns: + :obj:`dict`: A dictionary with the same keys as the provided one, but containing only the "value" field as value + if the provided ``conf_fields`` where correct. + + Raises: + ValueError: If any of the dictionary elements does not have the expected type + """ + conf_dict = {} for field in conf_fields: @@ -65,14 +90,38 @@ def check_conf_fields(conf_fields): def extend_paths(base_path, config_fields): + """ + Extends the relative paths of a given ``config_fields`` dictionary with a diven ``base_path``. + + Paths in the config file are based on DATA_PATH, this method extends them so they are all absolute. + + Args: + base_path (:obj:`str`): the base path to prepend the other paths. + config_fields (:obj:`dict`): a dictionary of configuration fields containing a ``path`` flag, as follows: + {"field0": {"value": value_from_conf_file, "path": True, ...}} + + Returns: + :obj:`dict`: A ``config_fields`` with the flagged paths updated. + """ + for key, field in config_fields.items(): - if field.get("path"): + if field.get("path") is True: config_fields[key]["value"] = base_path + config_fields[key]["value"] return config_fields def setup_logging(log_file_path, log_name_prefix): + """ + Setups a couple of loggers (console and file) given a prefix and a file path. The log names are: + + prefix | _file_log and prefix | _console_log + + Args: + log_file_path (:obj:`str`): the path of the file to output the file log. + log_name_prefix (:obj:`str`): the prefix to identify the log. + """ + if not isinstance(log_file_path, str): print(log_file_path) raise ValueError("Wrong log file path.") From 5c75b1f40da1c935b0275e4e1b8d8e764fcd26d7 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 19:32:40 +0100 Subject: [PATCH 14/20] Removes Logger dependency from setup_data_folder Logger was only used to log when a new folder was created, and was making that the setup needed to be done on the main of pisad and cli instead of __init__, which seems a better fit --- apps/cli/__init__.py | 3 ++- apps/cli/pisa_cli.py | 5 +---- common/tools.py | 4 +--- pisa/__init__.py | 3 ++- pisa/pisad.py | 3 --- 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/apps/cli/__init__.py b/apps/cli/__init__.py index df7cf41..0861ee2 100644 --- a/apps/cli/__init__.py +++ b/apps/cli/__init__.py @@ -1,6 +1,6 @@ import os import apps.cli.conf as conf -from common.tools import extend_paths, check_conf_fields, setup_logging +from common.tools import extend_paths, check_conf_fields, setup_logging, setup_data_folder LOG_PREFIX = "cli" @@ -24,4 +24,5 @@ conf_fields = extend_paths(conf_fields["DATA_FOLDER"]["value"], conf_fields) # Sanity check fields and build config dictionary config = check_conf_fields(conf_fields) +setup_data_folder(config.get("DATA_FOLDER")) setup_logging(config.get("CLIENT_LOG_FILE"), LOG_PREFIX) diff --git a/apps/cli/pisa_cli.py b/apps/cli/pisa_cli.py index 6bc8346..41fce7f 100644 --- a/apps/cli/pisa_cli.py +++ b/apps/cli/pisa_cli.py @@ -16,7 +16,7 @@ from apps.cli.blob import Blob from common.logger import Logger from common.appointment import Appointment from common.cryptographer import Cryptographer -from common.tools import check_sha256_hex_format, check_locator_format, compute_locator, setup_data_folder +from common.tools import check_sha256_hex_format, check_locator_format, compute_locator HTTP_OK = 200 @@ -343,9 +343,6 @@ if __name__ == "__main__": commands = ["add_appointment", "get_appointment", "help"] testing_commands = ["generate_dummy_appointment"] - # Create user folder if missing - setup_data_folder(config.get("DATA_FOLDER"), logger) - try: opts, args = getopt(argv[1:], "s:p:h", ["server", "port", "help"]) diff --git a/common/tools.py b/common/tools.py index 8e0617c..0c131da 100644 --- a/common/tools.py +++ b/common/tools.py @@ -42,17 +42,15 @@ def compute_locator(tx_id): return tx_id[:LOCATOR_LEN_HEX] -def setup_data_folder(data_folder, logger): +def setup_data_folder(data_folder): """ Create a data folder for either the client or the server side if the folder does not exists. Args: data_folder (:obj:`str`): the path of the folder - logger (:obj: `Logger `): a logger instance to notify about the folder creation. """ if not os.path.isdir(data_folder): - logger.info("Data folder not found. Creating it") os.makedirs(data_folder, exist_ok=True) diff --git a/pisa/__init__.py b/pisa/__init__.py index e7d380b..2e5149f 100644 --- a/pisa/__init__.py +++ b/pisa/__init__.py @@ -1,6 +1,6 @@ import os import pisa.conf as conf -from common.tools import check_conf_fields, setup_logging, extend_paths +from common.tools import check_conf_fields, setup_logging, extend_paths, setup_data_folder from pisa.utils.auth_proxy import AuthServiceProxy HOST = "localhost" @@ -34,4 +34,5 @@ conf_fields = extend_paths(conf_fields["DATA_FOLDER"]["value"], conf_fields) # Sanity check fields and build config dictionary config = check_conf_fields(conf_fields) +setup_data_folder(config.get("DATA_FOLDER")) setup_logging(config.get("SERVER_LOG_FILE"), LOG_PREFIX) diff --git a/pisa/pisad.py b/pisa/pisad.py index 1c35000..0335832 100644 --- a/pisa/pisad.py +++ b/pisa/pisad.py @@ -3,7 +3,6 @@ from sys import argv, exit from signal import signal, SIGINT, SIGQUIT, SIGTERM from common.logger import Logger -from common.tools import setup_data_folder from pisa import config, LOG_PREFIX from pisa.api import API @@ -34,8 +33,6 @@ def main(): signal(SIGQUIT, handle_signals) logger.info("Starting PISA") - - setup_data_folder(config.get("DATA_FOLDER"), logger) db_manager = DBManager(config.get("DB_PATH")) if not can_connect_to_bitcoind(): From 14724ceda9b963738c95708121360059807bb22d Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 19:54:40 +0100 Subject: [PATCH 15/20] Adds missing common/tools unit tests and removes pisad tests `test_pisad.py` was only covering config parsing tests, that are now part of `common/tools.py`. --- test/common/unit/test_tools.py | 116 ++++++++++++++++++++++++++++++++- test/pisa/unit/test_pisad.py | 51 --------------- 2 files changed, 115 insertions(+), 52 deletions(-) delete mode 100644 test/pisa/unit/test_pisad.py diff --git a/test/common/unit/test_tools.py b/test/common/unit/test_tools.py index eebdab9..b4d2ad4 100644 --- a/test/common/unit/test_tools.py +++ b/test/common/unit/test_tools.py @@ -1,7 +1,26 @@ -from common.tools import check_sha256_hex_format +import os +import pytest +import logging +from copy import deepcopy + +from pisa import conf_fields + +from common.constants import LOCATOR_LEN_BYTES +from common.tools import ( + check_sha256_hex_format, + check_locator_format, + compute_locator, + setup_data_folder, + check_conf_fields, + extend_paths, + setup_logging, +) from test.common.unit.conftest import get_random_value_hex +conf_fields_copy = deepcopy(conf_fields) + + def test_check_sha256_hex_format(): # Only 32-byte hex encoded strings should pass the test wrong_inputs = [None, str(), 213, 46.67, dict(), "A" * 63, "C" * 65, bytes(), get_random_value_hex(31)] @@ -10,3 +29,98 @@ def test_check_sha256_hex_format(): for v in range(100): assert check_sha256_hex_format(get_random_value_hex(32)) is True + + +def test_check_locator_format(): + # Check that only LOCATOR_LEN_BYTES long string pass the test + + wrong_inputs = [ + None, + str(), + 213, + 46.67, + dict(), + "A" * (2 * LOCATOR_LEN_BYTES - 1), + "C" * (2 * LOCATOR_LEN_BYTES + 1), + bytes(), + get_random_value_hex(LOCATOR_LEN_BYTES - 1), + ] + for wtype in wrong_inputs: + assert check_sha256_hex_format(wtype) is False + + for _ in range(100): + assert check_locator_format(get_random_value_hex(LOCATOR_LEN_BYTES)) is True + + +def test_compute_locator(): + # The best way of checking that compute locator is correct is by using check_locator_format + for _ in range(100): + assert check_locator_format(compute_locator(get_random_value_hex(LOCATOR_LEN_BYTES))) is True + + # String of length smaller than LOCATOR_LEN_BYTES bytes must fail + for i in range(1, LOCATOR_LEN_BYTES): + assert check_locator_format(compute_locator(get_random_value_hex(i))) is False + + +def test_setup_data_folder(): + # This method should create a folder if it does not exist, and do nothing otherwise + test_folder = "test_folder" + assert not os.path.isdir(test_folder) + + setup_data_folder(test_folder) + + assert os.path.isdir(test_folder) + + os.rmdir(test_folder) + + +def test_check_conf_fields(): + # The test should work with a valid config_fields (obtained from a valid conf.py) + assert type(check_conf_fields(conf_fields_copy)) == dict + + +def test_bad_check_conf_fields(): + # Create a messed up version of the file that should throw an error. + conf_fields_copy["BTC_RPC_USER"] = 0000 + conf_fields_copy["BTC_RPC_PASSWD"] = "password" + conf_fields_copy["BTC_RPC_HOST"] = 000 + + # We should get a ValueError here. + with pytest.raises(Exception): + check_conf_fields(conf_fields_copy) + + +def test_extend_paths(): + # Test that only items with the path flag are extended + config_fields = { + "foo": {"value": "foofoo"}, + "var": {"value": "varvar", "path": True}, + "foovar": {"value": "foovarfoovar"}, + } + base_path = "base_path/" + extended_config_field = extend_paths(base_path, config_fields) + + for k, field in extended_config_field.items(): + if field.get("path") is True: + assert base_path in field.get("value") + else: + assert base_path not in field.get("value") + + +def test_setup_logging(): + # Check that setup_logging creates two new logs for every prefix + prefix = "foo" + log_file = "var.log" + + f_log_suffix = "_file_log" + c_log_suffix = "_console_log" + + assert len(logging.getLogger(prefix + f_log_suffix).handlers) is 0 + assert len(logging.getLogger(prefix + c_log_suffix).handlers) is 0 + + setup_logging(log_file, prefix) + + assert len(logging.getLogger(prefix + f_log_suffix).handlers) is 1 + assert len(logging.getLogger(prefix + c_log_suffix).handlers) is 1 + + os.remove(log_file) diff --git a/test/pisa/unit/test_pisad.py b/test/pisa/unit/test_pisad.py deleted file mode 100644 index 30db71e..0000000 --- a/test/pisa/unit/test_pisad.py +++ /dev/null @@ -1,51 +0,0 @@ -import importlib -import os -import pytest -from shutil import copyfile - -from pisa.pisad import load_config - -test_conf_file_path = os.getcwd() + "/test/pisa/unit/test_conf.py" - - -def test_load_config(): - # Copy the sample-conf.py file to use as a test config file. - copyfile(os.getcwd() + "/pisa/sample_conf.py", test_conf_file_path) - - import test.pisa.unit.test_conf as conf - - # If the file has all the correct fields and data, it should return a dict. - conf_dict = load_config(conf) - assert type(conf_dict) == dict - - # Delete the file. - os.remove(test_conf_file_path) - - -def test_bad_load_config(): - # Create a messed up version of the file that should throw an error. - with open(test_conf_file_path, "w") as f: - f.write('# bitcoind\nBTC_RPC_USER = 0000\nBTC_RPC_PASSWD = "password"\nBTC_RPC_HOST = 000') - - import test.pisa.unit.test_conf as conf - - importlib.reload(conf) - - with pytest.raises(Exception): - conf_dict = load_config(conf) - - os.remove(test_conf_file_path) - - -def test_empty_load_config(): - # Create an empty version of the file that should throw an error. - open(test_conf_file_path, "a") - - import test.pisa.unit.test_conf as conf - - importlib.reload(conf) - - with pytest.raises(Exception): - conf_dict = load_config(conf) - - os.remove(test_conf_file_path) From 3fd84a8d7f1b8d461c15f1e19bf2dbfce19e76d4 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 19:57:21 +0100 Subject: [PATCH 16/20] Modifies e2e tests so pisad can be run and stop from there --- test/pisa/e2e/conftest.py | 9 +++++++++ test/pisa/e2e/test_basic_e2e.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/test/pisa/e2e/conftest.py b/test/pisa/e2e/conftest.py index cef3237..fbf00c9 100644 --- a/test/pisa/e2e/conftest.py +++ b/test/pisa/e2e/conftest.py @@ -1,8 +1,10 @@ import pytest import random +from multiprocessing import Process from decimal import Decimal, getcontext import pisa.conf as conf +from pisa.pisad import main from pisa.utils.auth_proxy import AuthServiceProxy getcontext().prec = 10 @@ -48,6 +50,13 @@ def create_txs(bitcoin_cli): return signed_commitment_tx, signed_penalty_tx +def run_pisad(): + pisad_process = Process(target=main, daemon=True) + pisad_process.start() + + return pisad_process + + def get_random_value_hex(nbytes): pseudo_random_value = random.getrandbits(8 * nbytes) prv_hex = "{:x}".format(pseudo_random_value) diff --git a/test/pisa/e2e/test_basic_e2e.py b/test/pisa/e2e/test_basic_e2e.py index 8c26867..a1b0cec 100644 --- a/test/pisa/e2e/test_basic_e2e.py +++ b/test/pisa/e2e/test_basic_e2e.py @@ -9,13 +9,22 @@ from common.tools import compute_locator from common.appointment import Appointment from common.cryptographer import Cryptographer from pisa.utils.auth_proxy import JSONRPCException -from test.pisa.e2e.conftest import END_TIME_DELTA, build_appointment_data, get_random_value_hex, create_penalty_tx +from test.pisa.e2e.conftest import ( + END_TIME_DELTA, + build_appointment_data, + get_random_value_hex, + create_penalty_tx, + run_pisad, +) # We'll use pisa_cli to add appointments. The expected input format is a list of arguments with a json-encoded # appointment pisa_cli.pisa_api_server = HOST pisa_cli.pisa_api_port = PORT +# Run pisad +pisad_process = run_pisad() + def broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, addr): # Broadcast the commitment transaction and mine a block From 3d13cfbe392bffee1b9f0eaed883315d2152bec9 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 19:58:07 +0100 Subject: [PATCH 17/20] Removes pisad from the commands to be run by circle-ci before e2e tests --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index da0ba21..98cb5e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,7 +97,6 @@ jobs: name: Run e2e tests command: | . venv/bin/activate - python3 -m pisa.pisad & pytest test/pisa/e2e/ # - store_artifacts: From ebea93c103ee4ee64e85e302041d8e78c37d012a Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 20:02:05 +0100 Subject: [PATCH 18/20] Updates circle-ci to create cli conf for unit test --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 98cb5e0..12f9be5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,8 +60,10 @@ jobs: # Run unit tests - run: - name: Create pisa config - command: cp pisa/sample_conf.py pisa/conf.py + name: Creates config files + command: | + cp pisa/sample_conf.py pisa/conf.py + cp apps/cli/sample_conf.py apps/cli/conf.py - run: name: Run pisa unit tests From a84d753c4a299c10c1d75af881a622012556abc8 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 20:45:45 +0100 Subject: [PATCH 19/20] Updates generate_key to accept output directory. It stores the keys in the current dir by default. --- apps/generate_key.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/generate_key.py b/apps/generate_key.py index 74ba84c..30c1b26 100644 --- a/apps/generate_key.py +++ b/apps/generate_key.py @@ -30,14 +30,21 @@ def save_pk(pk, filename): if __name__ == "__main__": name = "pisa" + output_dir = "." - opts, _ = getopt(argv[1:], "n:", ["name"]) + opts, _ = getopt(argv[1:], "n:d:", ["name", "dir"]) for opt, arg in opts: if opt in ["-n", "--name"]: name = arg - SK_FILE_NAME = "../{}_sk.der".format(name) - PK_FILE_NAME = "../{}_pk.der".format(name) + if opt in ["-d", "--dir"]: + output_dir = arg + + if output_dir.endswith("/"): + output_dir = output_dir[:-1] + + SK_FILE_NAME = "{}/{}_sk.der".format(output_dir, name) + PK_FILE_NAME = "{}/{}_pk.der".format(output_dir, name) if os.path.exists(SK_FILE_NAME): print('A key with name "{}" already exists. Aborting.'.format(SK_FILE_NAME)) From deb182edda28023d5096b03c530c38e0e73ca7ee Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 23 Jan 2020 20:58:37 +0100 Subject: [PATCH 20/20] Updates circle-ci to store keys in data folder --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12f9be5..2a67e6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,9 +89,8 @@ jobs: command: | . venv/bin/activate cp test/pisa/e2e/pisa-conf.py pisa/conf.py - cd apps/ - python3 -m generate_key - python3 -m generate_key -n cli + python3 -m apps.generate_key -d ~/.pisa_btc/ + python3 -m apps.generate_key -n cli -d ~/.pisa_btc/ # Run E2E tests