aljaz/python-teos
1
0
Fork 0
mirror of https://github.com/aljazceru/python-teos.git synced 2025-12-17 22:24:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #79 from orbitalturtle/isolate-config

Browse Source 
Isolate pisa config options
This commit is contained in:
Sergi Delgado Segura
2020-01-15 10:25:51 +01:00
committed by GitHub
parent 36eaa93dc4 6bf2cb5379
commit 0519ea9ca8
12 changed files with 176 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pisa/api.py
View File

@@ -17,8 +17,9 @@ logger = Logger("API")
class API:
def __init__(self, watcher):
def __init__(self, watcher, config):
self.watcher = watcher
self.config = config
def add_appointment(self):
"""
@@ -41,7 +42,7 @@ class API:
# Check content type once if properly defined
request_data = json.loads(request.get_json())
inspector = Inspector()
inspector = Inspector(self.config)
appointment = inspector.inspect(
request_data.get("appointment"), request_data.get("signature"), request_data.get("public_key")
)

11
pisa/inspector.py
View File

@@ -5,7 +5,6 @@ from common.constants import LOCATOR_LEN_HEX
from common.cryptographer import Cryptographer
from pisa import errors
import pisa.conf as conf
from common.logger import Logger
from common.appointment import Appointment
from pisa.block_processor import BlockProcessor
@@ -23,6 +22,9 @@ class Inspector:
The :class:`Inspector` class is in charge of verifying that the appointment data provided by the user is correct.
"""
def __init__(self, config):
self.config = config
def inspect(self, appointment_data, signature, public_key):
"""
Inspects whether the data provided by the user is correct.
@@ -221,8 +223,7 @@ class Inspector:
return rcode, message
@staticmethod
def check_to_self_delay(to_self_delay):
def check_to_self_delay(self, to_self_delay):
"""
Checks if the provided ``to_self_delay`` is correct.
@@ -255,10 +256,10 @@ class Inspector:
rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE
message = "wrong to_self_delay data type ({})".format(t)
elif to_self_delay < conf.MIN_TO_SELF_DELAY:
elif to_self_delay < self.config.get("MIN_TO_SELF_DELAY"):
rcode = errors.APPOINTMENT_FIELD_TOO_SMALL
message = "to_self_delay too small. The to_self_delay should be at least {} (current: {})".format(
conf.MIN_TO_SELF_DELAY, to_self_delay
self.config.get("MIN_TO_SELF_DELAY"), to_self_delay
)
if message is not None:

64
pisa/pisad.py
View File

@@ -2,12 +2,11 @@ from getopt import getopt
from sys import argv, exit
from signal import signal, SIGINT, SIGQUIT, SIGTERM
from pisa.conf import DB_PATH
from common.logger import Logger
from pisa.api import API
from pisa.watcher import Watcher
from pisa.builder import Builder
from pisa.conf import BTC_NETWORK, PISA_SECRET_KEY
import pisa.conf as conf
from pisa.responder import Responder
from pisa.db_manager import DBManager
from pisa.block_processor import BlockProcessor
@@ -24,6 +23,53 @@ def handle_signals(signal_received, frame):
exit(0)
def load_config(config):
"""
Looks through all of the config options to make sure they contain the right type of data and builds a config
dictionary.
Args:
config (:obj:`module`): It takes in a config module object.
Returns:
:obj:`dict` A dictionary containing the config values.
"""
conf_dict = {}
conf_fields = {
"BTC_RPC_USER": {"value": config.BTC_RPC_USER, "type": str},
"BTC_RPC_PASSWD": {"value": config.BTC_RPC_PASSWD, "type": str},
"BTC_RPC_HOST": {"value": config.BTC_RPC_HOST, "type": str},
"BTC_RPC_PORT": {"value": config.BTC_RPC_PORT, "type": int},
"BTC_NETWORK": {"value": config.BTC_NETWORK, "type": str},
"FEED_PROTOCOL": {"value": config.FEED_PROTOCOL, "type": str},
"FEED_ADDR": {"value": config.FEED_ADDR, "type": str},
"FEED_PORT": {"value": config.FEED_PORT, "type": int},
"MAX_APPOINTMENTS": {"value": config.MAX_APPOINTMENTS, "type": int},
"EXPIRY_DELTA": {"value": config.EXPIRY_DELTA, "type": int},
"MIN_TO_SELF_DELAY": {"value": config.MIN_TO_SELF_DELAY, "type": int},
"SERVER_LOG_FILE": {"value": config.SERVER_LOG_FILE, "type": str},
"PISA_SECRET_KEY": {"value": config.PISA_SECRET_KEY, "type": str},
"CLIENT_LOG_FILE": {"value": config.CLIENT_LOG_FILE, "type": str},
"TEST_LOG_FILE": {"value": config.TEST_LOG_FILE, "type": str},
"DB_PATH": {"value": config.DB_PATH, "type": str},
}
for field in conf_fields:
value = conf_fields[field]["value"]
correct_type = conf_fields[field]["type"]
if (value is not None) and isinstance(value, correct_type):
conf_dict[field] = value
else:
err_msg = "{} variable in config is of the wrong type".format(field)
logger.error(err_msg)
raise ValueError(err_msg)
return conf_dict
if __name__ == "__main__":
logger.info("Starting PISA")
@@ -36,23 +82,25 @@ if __name__ == "__main__":
# FIXME: Leaving this here for future option/arguments
pass
pisa_config = load_config(conf)
if not can_connect_to_bitcoind():
logger.error("Can't connect to bitcoind. Shutting down")
elif not in_correct_network(BTC_NETWORK):
elif not in_correct_network(pisa_config.get("BTC_NETWORK")):
logger.error("bitcoind is running on a different network, check conf.py and bitcoin.conf. Shutting down")
else:
try:
db_manager = DBManager(DB_PATH)
db_manager = DBManager(pisa_config.get("DB_PATH"))
watcher_appointments_data = db_manager.load_watcher_appointments()
responder_trackers_data = db_manager.load_responder_trackers()
with open(PISA_SECRET_KEY, "rb") as key_file:
with open(pisa_config.get("PISA_SECRET_KEY"), "rb") as key_file:
secret_key_der = key_file.read()
watcher = Watcher(db_manager, secret_key_der)
watcher = Watcher(db_manager, secret_key_der, config=pisa_config)
if len(watcher_appointments_data) == 0 and len(responder_trackers_data) == 0:
logger.info("Fresh bootstrap")
@@ -65,7 +113,7 @@ if __name__ == "__main__":
last_block_responder = db_manager.load_last_block_hash_responder()
# FIXME: 32-reorgs-offline dropped txs are not used at this point.
responder = Responder(db_manager)
responder = Responder(db_manager, pisa_config)
last_common_ancestor_responder = None
missed_blocks_responder = None
@@ -97,7 +145,7 @@ if __name__ == "__main__":
watcher.block_queue = Builder.build_block_queue(missed_blocks_watcher)
# Fire the API
API(watcher).start()
API(watcher, config=pisa_config).start()
except Exception as e:
logger.error("An error occurred: {}. Shutting down".format(e))

5
pisa/responder.py
View File

@@ -135,13 +135,14 @@ class Responder:
"""
def __init__(self, db_manager):
def __init__(self, db_manager, config):
self.trackers = dict()
self.tx_tracker_map = dict()
self.unconfirmed_txs = []
self.missed_confirmations = dict()
self.asleep = True
self.block_queue = Queue()
self.config = config
self.zmq_subscriber = None
self.db_manager = db_manager
@@ -271,7 +272,7 @@ class Responder:
from ``bitcoind``. Block ids are received trough the ``block_queue``.
"""
self.zmq_subscriber = ZMQSubscriber(parent="Responder")
self.zmq_subscriber = ZMQSubscriber(self.config, parent="Responder")
self.zmq_subscriber.handle(self.block_queue)
def do_watch(self):

6
pisa/utils/zmq_subscriber.py
View File

@@ -8,12 +8,14 @@ from pisa.conf import FEED_PROTOCOL, FEED_ADDR, FEED_PORT
class ZMQSubscriber:
""" Adapted from https://github.com/bitcoin/bitcoin/blob/master/contrib/zmq/zmq_sub.py"""
def __init__(self, parent):
def __init__(self, config, parent):
self.zmqContext = zmq.Context()
self.zmqSubSocket = self.zmqContext.socket(zmq.SUB)
self.zmqSubSocket.setsockopt(zmq.RCVHWM, 0)
self.zmqSubSocket.setsockopt_string(zmq.SUBSCRIBE, "hashblock")
self.zmqSubSocket.connect("%s://%s:%s" % (FEED_PROTOCOL, FEED_ADDR, FEED_PORT))
self.zmqSubSocket.connect(
"%s://%s:%s" % (config.get("FEED_PROTOCOL"), config.get("FEED_ADDR"), config.get("FEED_PORT"))
)
self.logger = Logger("ZMQSubscriber-{}".format(parent))
self.terminate = False

13
pisa/watcher.py
View File

@@ -10,7 +10,6 @@ from pisa.cleaner import Cleaner
from pisa.responder import Responder
from pisa.block_processor import BlockProcessor
from pisa.utils.zmq_subscriber import ZMQSubscriber
from pisa.conf import EXPIRY_DELTA, MAX_APPOINTMENTS
logger = Logger("Watcher")
@@ -58,18 +57,18 @@ class Watcher:
"""
def __init__(self, db_manager, sk_der, responder=None, max_appointments=MAX_APPOINTMENTS):
def __init__(self, db_manager, sk_der, config, responder=None):
self.appointments = dict()
self.locator_uuid_map = dict()
self.asleep = True
self.block_queue = Queue()
self.max_appointments = max_appointments
self.config = config
self.zmq_subscriber = None
self.db_manager = db_manager
self.signing_key = Cryptographer.load_private_key_der(sk_der)
if not isinstance(responder, Responder):
self.responder = Responder(db_manager)
self.responder = Responder(db_manager, self.config)
def add_appointment(self, appointment):
"""
@@ -101,7 +100,7 @@ class Watcher:
"""
if len(self.appointments) < self.max_appointments:
if len(self.appointments) < self.config.get("MAX_APPOINTMENTS"):
uuid = uuid4().hex
self.appointments[uuid] = appointment
@@ -143,7 +142,7 @@ class Watcher:
trough the ``block_queue``.
"""
self.zmq_subscriber = ZMQSubscriber(parent="Watcher")
self.zmq_subscriber = ZMQSubscriber(self.config, parent="Watcher")
self.zmq_subscriber.handle(self.block_queue)
def do_watch(self):
@@ -168,7 +167,7 @@ class Watcher:
expired_appointments = [
uuid
for uuid, appointment in self.appointments.items()
if block["height"] > appointment.end_time + EXPIRY_DELTA
if block["height"] > appointment.end_time + self.config.get("EXPIRY_DELTA")
]
Cleaner.delete_expired_appointment(

23
test/pisa/unit/conftest.py
View File

@@ -148,3 +148,26 @@ def generate_dummy_tracker():
)
return TransactionTracker.from_dict(tracker_data)
def get_config():
config = {
"BTC_RPC_USER": "username",
"BTC_RPC_PASSWD": "password",
"BTC_RPC_HOST": "localhost",
"BTC_RPC_PORT": 8332,
"BTC_NETWORK": "regtest",
"FEED_PROTOCOL": "tcp",
"FEED_ADDR": "127.0.0.1",
"FEED_PORT": 28332,
"MAX_APPOINTMENTS": 100,
"EXPIRY_DELTA": 6,
"MIN_TO_SELF_DELAY": 20,
"SERVER_LOG_FILE": "pisa.log",
"PISA_SECRET_KEY": "pisa_sk.der",
"CLIENT_LOG_FILE": "pisa.log",
"TEST_LOG_FILE": "test.log",
"DB_PATH": "appointments",
}
return config

10
test/pisa/unit/test_api.py
View File

@@ -9,7 +9,6 @@ from pisa.api import API
from pisa.watcher import Watcher
from pisa.tools import bitcoin_cli
from pisa import HOST, PORT
from pisa.conf import MAX_APPOINTMENTS
from test.pisa.unit.conftest import (
generate_block,
@@ -17,6 +16,7 @@ from test.pisa.unit.conftest import (
get_random_value_hex,
generate_dummy_appointment_data,
generate_keypair,
get_config,
)
from common.constants import LOCATOR_LEN_BYTES
@@ -28,6 +28,8 @@ MULTIPLE_APPOINTMENTS = 10
appointments = []
locator_dispute_tx_map = {}
config = get_config()
@pytest.fixture(scope="module")
def run_api(db_manager):
@@ -37,9 +39,9 @@ def run_api(db_manager):
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
)
watcher = Watcher(db_manager, sk_der)
watcher = Watcher(db_manager, sk_der, get_config())
api_thread = Thread(target=API(watcher).start)
api_thread = Thread(target=API(watcher, config).start)
api_thread.daemon = True
api_thread.start()
@@ -102,7 +104,7 @@ def test_request_multiple_appointments_same_locator(new_appt_data, n=MULTIPLE_AP
def test_add_too_many_appointment(new_appt_data):
for _ in range(MAX_APPOINTMENTS - len(appointments)):
for _ in range(config.get("MAX_APPOINTMENTS") - len(appointments)):
r = add_appointment(new_appt_data)
assert r.status_code == 200

12
test/pisa/unit/test_inspector.py
View File

@@ -10,13 +10,13 @@ from common.appointment import Appointment
from pisa.block_processor import BlockProcessor
from pisa.conf import MIN_TO_SELF_DELAY
from test.pisa.unit.conftest import get_random_value_hex, generate_dummy_appointment_data, generate_keypair
from test.pisa.unit.conftest import get_random_value_hex, generate_dummy_appointment_data, generate_keypair, get_config
from common.constants import LOCATOR_LEN_BYTES, LOCATOR_LEN_HEX
from common.cryptographer import Cryptographer
inspector = Inspector()
inspector = Inspector(get_config())
APPOINTMENT_OK = (0, None)
NO_HEX_STRINGS = [
@@ -126,21 +126,21 @@ def test_check_to_self_delay():
# Right value, right format
to_self_delays = [MIN_TO_SELF_DELAY, MIN_TO_SELF_DELAY + 1, MIN_TO_SELF_DELAY + 1000]
for to_self_delay in to_self_delays:
assert Inspector.check_to_self_delay(to_self_delay) == APPOINTMENT_OK
assert inspector.check_to_self_delay(to_self_delay) == APPOINTMENT_OK
# to_self_delay too small
to_self_delays = [MIN_TO_SELF_DELAY - 1, MIN_TO_SELF_DELAY - 2, 0, -1, -1000]
for to_self_delay in to_self_delays:
assert Inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_FIELD_TOO_SMALL
assert inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_FIELD_TOO_SMALL
# Empty field
to_self_delay = None
assert Inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_EMPTY_FIELD
assert inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_EMPTY_FIELD
# Wrong data type
to_self_delays = WRONG_TYPES
for to_self_delay in to_self_delays:
assert Inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_WRONG_FIELD_TYPE
assert inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_WRONG_FIELD_TYPE
def test_check_blob():

52
test/pisa/unit/test_pisad.py Normal file
View File

@@ -0,0 +1,52 @@
import importlib
import os
import pytest
from pathlib import Path
from shutil import copyfile
from pisa.pisad import load_config
test_conf_file_path = os.getcwd() + "/test/pisa/unit/test_conf.py"
def test_load_config():
# Copy the sample-conf.py file to use as a test config file.
copyfile(os.getcwd() + "/pisa/sample_conf.py", test_conf_file_path)
import test.pisa.unit.test_conf as conf
# If the file has all the correct fields and data, it should return a dict.
conf_dict = load_config(conf)
assert type(conf_dict) == dict
# Delete the file.
os.remove(test_conf_file_path)
def test_bad_load_config():
# Create a messed up version of the file that should throw an error.
with open(test_conf_file_path, "w") as f:
f.write('# bitcoind\nBTC_RPC_USER = 0000\nBTC_RPC_PASSWD = "password"\nBTC_RPC_HOST = 000')
import test.pisa.unit.test_conf as conf
importlib.reload(conf)
with pytest.raises(Exception):
conf_dict = load_config(conf)
os.remove(test_conf_file_path)
def test_empty_load_config():
# Create an empty version of the file that should throw an error.
open(test_conf_file_path, "a")
import test.pisa.unit.test_conf as conf
importlib.reload(conf)
with pytest.raises(Exception):
conf_dict = load_config(conf)
os.remove(test_conf_file_path)

15
test/pisa/unit/test_responder.py
View File

@@ -17,12 +17,12 @@ from common.tools import check_sha256_hex_format
from bitcoind_mock.utils import sha256d
from bitcoind_mock.transaction import TX
from test.pisa.unit.conftest import generate_block, generate_blocks, get_random_value_hex
from test.pisa.unit.conftest import generate_block, generate_blocks, get_random_value_hex, get_config
@pytest.fixture(scope="module")
def responder(db_manager):
return Responder(db_manager)
return Responder(db_manager, get_config())
@pytest.fixture()
@@ -151,11 +151,12 @@ def test_init_responder(responder):
assert type(responder.missed_confirmations) is dict and len(responder.missed_confirmations) == 0
assert responder.block_queue.empty()
assert responder.asleep is True
assert type(responder.config) is dict
assert responder.zmq_subscriber is None
def test_handle_breach(db_manager):
responder = Responder(db_manager)
responder = Responder(db_manager, get_config())
uuid = uuid4().hex
tracker = create_dummy_tracker()
@@ -295,7 +296,7 @@ def test_do_subscribe(responder):
def test_do_watch(temp_db_manager):
responder = Responder(temp_db_manager)
responder = Responder(temp_db_manager, get_config())
responder.block_queue = Queue()
zmq_thread = Thread(target=responder.do_subscribe)
@@ -351,7 +352,7 @@ def test_do_watch(temp_db_manager):
def test_check_confirmations(temp_db_manager):
responder = Responder(temp_db_manager)
responder = Responder(temp_db_manager, get_config())
responder.block_queue = Queue()
zmq_thread = Thread(target=responder.do_subscribe)
@@ -414,7 +415,7 @@ def test_get_completed_trackers(db_manager):
initial_height = bitcoin_cli().getblockcount()
# Let's use a fresh responder for this to make it easier to compare the results
responder = Responder(db_manager)
responder = Responder(db_manager, get_config())
# A complete tracker is a tracker that has reached the appointment end with enough confirmations (> MIN_CONFIRMATIONS)
# We'll create three type of transactions: end reached + enough conf, end reached + no enough conf, end not reached
@@ -462,7 +463,7 @@ def test_get_completed_trackers(db_manager):
def test_rebroadcast(db_manager):
responder = Responder(db_manager)
responder = Responder(db_manager, get_config())
responder.asleep = False
txs_to_rebroadcast = []

5
test/pisa/unit/test_watcher.py
View File

@@ -13,6 +13,7 @@ from test.pisa.unit.conftest import (
generate_dummy_appointment,
get_random_value_hex,
generate_keypair,
get_config,
)
from pisa.conf import EXPIRY_DELTA, MAX_APPOINTMENTS
@@ -36,7 +37,7 @@ sk_der = signing_key.private_bytes(
@pytest.fixture(scope="module")
def watcher(db_manager):
return Watcher(db_manager, sk_der)
return Watcher(db_manager, sk_der, get_config())
@pytest.fixture(scope="module")
@@ -72,7 +73,7 @@ def test_init(watcher):
assert type(watcher.locator_uuid_map) is dict and len(watcher.locator_uuid_map) == 0
assert watcher.block_queue.empty()
assert watcher.asleep is True
assert watcher.max_appointments == MAX_APPOINTMENTS
assert type(watcher.config) is dict
assert watcher.zmq_subscriber is None
assert type(watcher.responder) is Responder