From 6ee04bd3032d60ae05bc5911c494ce31c5b6581f Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 24 Mar 2020 19:03:41 +0100 Subject: [PATCH 01/62] Sets add_appointment_endpoint from / to /add_appointment for consistency Also passes the base_url to add_appointment and get_appointment and builds the full endpoint inside (also for consistency) --- cli/teos_cli.py | 23 ++++++++---------- teos/api.py | 2 +- test/cli/unit/test_teos_cli.py | 43 +++++++++++++++------------------ test/teos/e2e/test_basic_e2e.py | 29 +++++++++------------- test/teos/unit/test_api.py | 18 ++++++++------ 5 files changed, 54 insertions(+), 61 deletions(-) diff --git a/cli/teos_cli.py b/cli/teos_cli.py index f3c2e8a..f33daa8 100644 --- a/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -109,9 +109,6 @@ def add_appointment(args, teos_url, config): error occurs during the process. """ - # Currently the base_url is the same as the add_appointment_endpoint - add_appointment_endpoint = teos_url - teos_pk, cli_sk, cli_pk_der = load_keys( config.get("TEOS_PUBLIC_KEY"), config.get("CLI_PRIVATE_KEY"), config.get("CLI_PUBLIC_KEY") ) @@ -159,7 +156,7 @@ def add_appointment(args, teos_url, config): data = {"appointment": appointment.to_dict(), "signature": signature, "public_key": hex_pk_der.decode("utf-8")} # Send appointment to the server. - server_response = post_appointment(data, add_appointment_endpoint) + server_response = post_appointment(data, teos_url) if server_response is None: return False @@ -233,20 +230,22 @@ def parse_add_appointment_args(args): return appointment_data -def post_appointment(data, add_appointment_endpoint): +def post_appointment(data, teos_url): """ Sends appointment data to add_appointment endpoint to be processed by the tower. Args: data (:obj:`dict`): a dictionary containing three fields: an appointment, the client-side signature, and the der-encoded client public key. - add_appointment_endpoint (:obj:`str`): the teos endpoint where to send appointments to. + teos_url (:obj:`str`): the teos base url. Returns: :obj:`dict` or ``None``: a json-encoded dictionary with the server response if the data can be posted. None otherwise. """ + add_appointment_endpoint = "{}/add_appointment".format(teos_url) + logger.info("Sending appointment to the Eye of Satoshi") try: @@ -342,29 +341,28 @@ def save_appointment_receipt(appointment, signature, config): return False -def get_appointment(locator, get_appointment_endpoint): +def get_appointment(locator, teos_url): """ Gets information about an appointment from the tower. Args: locator (:obj:`str`): the appointment locator used to identify it. - get_appointment_endpoint (:obj:`str`): the teos endpoint where to get appointments from. + teos_url (:obj:`str`): the teos base url. Returns: :obj:`dict` or :obj:`None`: a dictionary containing thew appointment data if the locator is valid and the tower responds. ``None`` otherwise. """ + get_appointment_endpoint = "{}/get_appointment".format(teos_url) valid_locator = check_locator_format(locator) if not valid_locator: logger.error("The provided locator is not valid", locator=locator) return None - parameters = "?locator={}".format(locator) - try: - r = requests.get(url=get_appointment_endpoint + parameters, timeout=5) + r = requests.get(url="{}?locator={}".format(get_appointment_endpoint, locator), timeout=5) return r.json() except ConnectTimeout: @@ -414,8 +412,7 @@ def main(args, command_line_conf): if arg_opt in ["-h", "--help"]: sys.exit(help_get_appointment()) - get_appointment_endpoint = "{}/get_appointment".format(teos_url) - appointment_data = get_appointment(arg_opt, get_appointment_endpoint) + appointment_data = get_appointment(arg_opt, teos_url) if appointment_data: print(appointment_data) diff --git a/teos/api.py b/teos/api.py index 97fcf6d..31f612c 100644 --- a/teos/api.py +++ b/teos/api.py @@ -189,7 +189,7 @@ class API: """ routes = { - "/": (self.add_appointment, ["POST"]), + "/add_appointment": (self.add_appointment, ["POST"]), "/get_appointment": (self.get_appointment, ["GET"]), "/get_all_appointments": (self.get_all_appointments, ["GET"]), } diff --git a/test/cli/unit/test_teos_cli.py b/test/cli/unit/test_teos_cli.py index fb4e053..ec34714 100644 --- a/test/cli/unit/test_teos_cli.py +++ b/test/cli/unit/test_teos_cli.py @@ -24,11 +24,9 @@ dummy_sk = PrivateKey() dummy_pk = dummy_sk.public_key another_sk = PrivateKey() - -# Replace the key in the module with a key we control for the tests -teos_cli.teos_public_key = dummy_pk -# Replace endpoint with dummy one -teos_endpoint = "http://{}:{}/".format(config.get("TEOS_SERVER"), config.get("TEOS_PORT")) +teos_url = "http://{}:{}".format(config.get("TEOS_SERVER"), config.get("TEOS_PORT")) +add_appointment_endpoint = "{}/add_appointment".format(teos_url) +get_appointment_endpoint = "{}/get_appointment".format(teos_url) dummy_appointment_request = { "tx": get_random_value_hex(192), @@ -102,16 +100,17 @@ def test_load_keys(): # TODO: 90-add-more-add-appointment-tests @responses.activate def test_add_appointment(monkeypatch): + # Simulate a request to add_appointment for dummy_appointment, make sure that the right endpoint is requested # and the return value is True monkeypatch.setattr(teos_cli, "load_keys", load_dummy_keys) response = {"locator": dummy_appointment.locator, "signature": get_dummy_signature()} - responses.add(responses.POST, teos_endpoint, json=response, status=200) - result = teos_cli.add_appointment([json.dumps(dummy_appointment_request)], teos_endpoint, config) + responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) + result = teos_cli.add_appointment([json.dumps(dummy_appointment_request)], teos_url, config) assert len(responses.calls) == 1 - assert responses.calls[0].request.url == teos_endpoint + assert responses.calls[0].request.url == add_appointment_endpoint assert result @@ -128,8 +127,8 @@ def test_add_appointment_with_invalid_signature(monkeypatch): "signature": get_bad_signature(), # Sign with a bad key } - responses.add(responses.POST, teos_endpoint, json=response, status=200) - result = teos_cli.add_appointment([json.dumps(dummy_appointment_request)], teos_endpoint, config) + responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) + result = teos_cli.add_appointment([json.dumps(dummy_appointment_request)], teos_url, config) shutil.rmtree(config.get("APPOINTMENTS_FOLDER_NAME")) @@ -166,11 +165,11 @@ def test_post_appointment(): "signature": Cryptographer.sign(dummy_appointment.serialize(), dummy_pk), } - responses.add(responses.POST, teos_endpoint, json=response, status=200) - response = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_endpoint) + responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) + response = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_url) assert len(responses.calls) == 1 - assert responses.calls[0].request.url == teos_endpoint + assert responses.calls[0].request.url == add_appointment_endpoint assert response @@ -183,18 +182,18 @@ def test_process_post_appointment_response(): } # A 200 OK with a correct json response should return the json of the response - responses.add(responses.POST, teos_endpoint, json=response, status=200) - r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_endpoint) + responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) + r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_url) assert teos_cli.process_post_appointment_response(r) == r.json() # If we modify the response code tor a rejection (lets say 404) we should get None - responses.replace(responses.POST, teos_endpoint, json=response, status=404) - r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_endpoint) + responses.replace(responses.POST, add_appointment_endpoint, json=response, status=404) + r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_url) assert teos_cli.process_post_appointment_response(r) is None # The same should happen if the response is not in json - responses.replace(responses.POST, teos_endpoint, status=404) - r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_endpoint) + responses.replace(responses.POST, add_appointment_endpoint, status=404) + r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_url) assert teos_cli.process_post_appointment_response(r) is None @@ -219,11 +218,10 @@ def test_get_appointment(): # Response of get_appointment endpoint is an appointment with status added to it. dummy_appointment_full["status"] = "being_watched" response = dummy_appointment_full - get_appointment_endpoint = teos_endpoint + "get_appointment" request_url = "{}?locator={}".format(get_appointment_endpoint, response.get("locator")) responses.add(responses.GET, request_url, json=response, status=200) - result = teos_cli.get_appointment(response.get("locator"), get_appointment_endpoint) + result = teos_cli.get_appointment(response.get("locator"), teos_url) assert len(responses.calls) == 1 assert responses.calls[0].request.url == request_url @@ -233,10 +231,9 @@ def test_get_appointment(): @responses.activate def test_get_appointment_err(): locator = get_random_value_hex(16) - get_appointment_endpoint = teos_endpoint + "get_appointment" # Test that get_appointment handles a connection error appropriately. request_url = "{}?locator={}".format(get_appointment_endpoint, locator) responses.add(responses.GET, request_url, body=ConnectionError()) - assert not teos_cli.get_appointment(locator, get_appointment_endpoint) + assert not teos_cli.get_appointment(locator, teos_url) diff --git a/test/teos/e2e/test_basic_e2e.py b/test/teos/e2e/test_basic_e2e.py index 88b3702..1b19258 100644 --- a/test/teos/e2e/test_basic_e2e.py +++ b/test/teos/e2e/test_basic_e2e.py @@ -24,14 +24,9 @@ from test.teos.e2e.conftest import ( cli_config = get_config(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF) common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix="") -# # We'll use teos_cli to add appointments. The expected input format is a list of arguments with a json-encoded -# # appointment -# teos_cli.teos_api_server = "http://{}".format(HOST) -# teos_cli.teos_api_port = PORT - teos_base_endpoint = "http://{}:{}".format(cli_config.get("TEOS_SERVER"), cli_config.get("TEOS_PORT")) -teos_add_appointment_endpoint = teos_base_endpoint -teos_get_appointment_endpoint = teos_base_endpoint + "/get_appointment" +teos_add_appointment_endpoint = "{}/add_appointment".format(teos_base_endpoint) +teos_get_appointment_endpoint = "{}/get_appointment".format(teos_base_endpoint) # Run teosd teosd_process = run_teosd() @@ -46,7 +41,7 @@ def broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, addr): def get_appointment_info(locator): # Check that the justice has been triggered (the appointment has moved from Watcher to Responder) sleep(1) # Let's add a bit of delay so the state can be updated - return teos_cli.get_appointment(locator, teos_get_appointment_endpoint) + return teos_cli.get_appointment(locator, teos_base_endpoint) def test_appointment_life_cycle(bitcoin_cli, create_txs): @@ -55,7 +50,7 @@ def test_appointment_life_cycle(bitcoin_cli, create_txs): appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) locator = compute_locator(commitment_tx_id) - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_add_appointment_endpoint, cli_config) is True + assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True appointment_info = get_appointment_info(locator) assert appointment_info is not None @@ -105,7 +100,7 @@ def test_appointment_malformed_penalty(bitcoin_cli, create_txs): appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, mod_penalty_tx.hex()) locator = compute_locator(commitment_tx_id) - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_add_appointment_endpoint, cli_config) is True + assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True # Broadcast the commitment transaction and mine a block new_addr = bitcoin_cli.getnewaddress() @@ -143,7 +138,7 @@ def test_appointment_wrong_key(bitcoin_cli, create_txs): data = {"appointment": appointment.to_dict(), "signature": signature, "public_key": hex_pk_der.decode("utf-8")} # Send appointment to the server. - response = teos_cli.post_appointment(data, teos_add_appointment_endpoint) + response = teos_cli.post_appointment(data, teos_base_endpoint) response_json = teos_cli.process_post_appointment_response(response) # Check that the server has accepted the appointment @@ -179,8 +174,8 @@ def test_two_identical_appointments(bitcoin_cli, create_txs): locator = compute_locator(commitment_tx_id) # Send the appointment twice - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_add_appointment_endpoint, cli_config) is True - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_add_appointment_endpoint, cli_config) is True + assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True + assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True # Broadcast the commitment transaction and mine a block new_addr = bitcoin_cli.getnewaddress() @@ -213,8 +208,8 @@ def test_two_appointment_same_locator_different_penalty(bitcoin_cli, create_txs) appointment2_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx2) locator = compute_locator(commitment_tx_id) - assert teos_cli.add_appointment([json.dumps(appointment1_data)], teos_add_appointment_endpoint, cli_config) is True - assert teos_cli.add_appointment([json.dumps(appointment2_data)], teos_add_appointment_endpoint, cli_config) is True + assert teos_cli.add_appointment([json.dumps(appointment1_data)], teos_base_endpoint, cli_config) is True + assert teos_cli.add_appointment([json.dumps(appointment2_data)], teos_base_endpoint, cli_config) is True # Broadcast the commitment transaction and mine a block new_addr = bitcoin_cli.getnewaddress() @@ -241,7 +236,7 @@ def test_appointment_shutdown_teos_trigger_back_online(create_txs, bitcoin_cli): appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) locator = compute_locator(commitment_tx_id) - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_add_appointment_endpoint, cli_config) is True + assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True # Restart teos teosd_process.terminate() @@ -279,7 +274,7 @@ def test_appointment_shutdown_teos_trigger_while_offline(create_txs, bitcoin_cli appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) locator = compute_locator(commitment_tx_id) - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_add_appointment_endpoint, cli_config) is True + assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True # Check that the appointment is still in the Watcher appointment_info = get_appointment_info(locator) diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 2273087..ea65b4f 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -28,6 +28,10 @@ from common.constants import LOCATOR_LEN_BYTES TEOS_API = "http://{}:{}".format(HOST, PORT) +add_appointment_endpoint = "{}/add_appointment".format(TEOS_API) +get_appointment_endpoint = "{}/get_appointment".format(TEOS_API) +get_all_appointment_endpoint = "{}/get_all_appointments".format(TEOS_API) + MULTIPLE_APPOINTMENTS = 10 appointments = [] @@ -68,7 +72,7 @@ def new_appt_data(): def add_appointment(new_appt_data): - r = requests.post(url=TEOS_API, json=json.dumps(new_appt_data), timeout=5) + r = requests.post(url=add_appointment_endpoint, json=json.dumps(new_appt_data), timeout=5) if r.status_code == 200: appointments.append(new_appt_data["appointment"]) @@ -88,7 +92,7 @@ def test_add_appointment(run_api, run_bitcoind, new_appt_data): def test_request_random_appointment(): - r = requests.get(url=TEOS_API + "/get_appointment?locator=" + get_random_value_hex(LOCATOR_LEN_BYTES)) + r = requests.get(url="{}?locator={}".format(get_appointment_endpoint, get_random_value_hex(LOCATOR_LEN_BYTES))) assert r.status_code == 200 received_appointments = json.loads(r.content) @@ -123,7 +127,7 @@ def test_add_too_many_appointment(new_appt_data): def test_get_all_appointments_watcher(): - r = requests.get(url=TEOS_API + "/get_all_appointments") + r = requests.get(url=get_all_appointment_endpoint) assert r.status_code == 200 and r.reason == "OK" received_appointments = json.loads(r.content) @@ -147,7 +151,7 @@ def test_get_all_appointments_responder(): generate_blocks(6) # Get all appointments - r = requests.get(url=TEOS_API + "/get_all_appointments") + r = requests.get(url=get_all_appointment_endpoint) received_appointments = json.loads(r.content) # Make sure there is not pending locator in the watcher @@ -164,7 +168,7 @@ def test_request_appointment_watcher(new_appt_data): assert r.status_code == 200 # Next we can request it - r = requests.get(url=TEOS_API + "/get_appointment?locator=" + new_appt_data["appointment"]["locator"]) + r = requests.get(url="{}?locator={}".format(get_appointment_endpoint, new_appt_data["appointment"]["locator"])) assert r.status_code == 200 # Each locator may point to multiple appointments, check them all @@ -173,7 +177,7 @@ def test_request_appointment_watcher(new_appt_data): # Take the status out and leave the received appointments ready to compare appointment_status = [appointment.pop("status") for appointment in received_appointments] - # Check that the appointment is within the received appoints + # Check that the appointment is within the received appointments assert new_appt_data["appointment"] in received_appointments # Check that all the appointments are being watched @@ -191,7 +195,7 @@ def test_request_appointment_responder(new_appt_data): # Generate a block to trigger the watcher generate_block() - r = requests.get(url=TEOS_API + "/get_appointment?locator=" + new_appt_data["appointment"]["locator"]) + r = requests.get(url="{}?locator={}".format(get_appointment_endpoint, new_appt_data["appointment"]["locator"])) assert r.status_code == 200 received_appointments = json.loads(r.content) From dd53ad68fb299ec4e2ffc37ff30be8bc3ef88719 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 24 Mar 2020 19:55:41 +0100 Subject: [PATCH 02/62] Fixes bug when dealing with empty JSON requests or empty appointment field When posting a request via requests.post the json field was dumped to json, but it shouldn't have been since requests deals with this internally. That meant that the requests made by the code didn't match proper JSON. In line with this, the API was only parsing this type POST requests correctly, making add_appointment to fail if a proper formatted JSON was passed. On top of that, empty appointments were not checked in the Inspector before trying to get data from them, making it crash if a JSON was posted to add_appointment not containing the `appointment` field. Unit tests for this should be added. --- cli/teos_cli.py | 2 +- teos/api.py | 3 +-- teos/inspector.py | 3 +++ test/teos/unit/test_api.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/teos_cli.py b/cli/teos_cli.py index f33daa8..a4609d3 100644 --- a/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -249,7 +249,7 @@ def post_appointment(data, teos_url): logger.info("Sending appointment to the Eye of Satoshi") try: - return requests.post(url=add_appointment_endpoint, json=json.dumps(data), timeout=5) + return requests.post(url=add_appointment_endpoint, json=data, timeout=5) except ConnectTimeout: logger.error("Can't connect to the Eye of Satoshi's API. Connection timeout") diff --git a/teos/api.py b/teos/api.py index 31f612c..6aecb10 100644 --- a/teos/api.py +++ b/teos/api.py @@ -1,5 +1,4 @@ import os -import json import logging from flask import Flask, request, abort, jsonify @@ -55,7 +54,7 @@ class API: if request.is_json: # Check content type once if properly defined - request_data = json.loads(request.get_json()) + request_data = request.get_json() appointment = self.inspector.inspect( request_data.get("appointment"), request_data.get("signature"), request_data.get("public_key") ) diff --git a/teos/inspector.py b/teos/inspector.py index ed288fc..60a83fc 100644 --- a/teos/inspector.py +++ b/teos/inspector.py @@ -54,6 +54,9 @@ class Inspector: Errors are defined in :mod:`Errors `. """ + if appointment_data is None: + return errors.APPOINTMENT_EMPTY_FIELD, "empty appointment received" + block_height = self.block_processor.get_block_count() if block_height is not None: diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index ea65b4f..6cbf15f 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -72,7 +72,7 @@ def new_appt_data(): def add_appointment(new_appt_data): - r = requests.post(url=add_appointment_endpoint, json=json.dumps(new_appt_data), timeout=5) + r = requests.post(url=add_appointment_endpoint, json=new_appt_data, timeout=5) if r.status_code == 200: appointments.append(new_appt_data["appointment"]) From 519caec29af492d9fdc75f28ea0829f9e0e9e7c3 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 25 Mar 2020 12:14:12 +0100 Subject: [PATCH 03/62] Adds basic register logic - Adds register endpoint in the API - Adds the Gatekeeper to keep track of registered user and allow/reject access - Adds registration errors - Updates API unit tests - Refactors some methods of the API to reduce code replication --- teos/api.py | 93 ++++++++++++++++++++++++++++---------- teos/errors.py | 6 ++- teos/gatekeeper.py | 32 +++++++++++++ teos/teosd.py | 3 +- test/teos/unit/test_api.py | 5 +- 5 files changed, 113 insertions(+), 26 deletions(-) create mode 100644 teos/gatekeeper.py diff --git a/teos/api.py b/teos/api.py index 6aecb10..8abf790 100644 --- a/teos/api.py +++ b/teos/api.py @@ -2,10 +2,11 @@ import os import logging from flask import Flask, request, abort, jsonify +import teos.errors as errors from teos import HOST, PORT, LOG_PREFIX + from common.logger import Logger from common.appointment import Appointment - from common.constants import HTTP_OK, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, LOCATOR_LEN_HEX @@ -14,6 +15,24 @@ app = Flask(__name__) logger = Logger(actor="API", log_name_prefix=LOG_PREFIX) +# TODO: UNITTEST +def get_remote_addr(): + """ + Gets the remote client ip address. The HTTP_X_REAL_IP field is tried first in case the server is behind a reverse + proxy. + + Returns: + :obj:`str`: the IP address of the client. + """ + + # Getting the real IP if the server is behind a reverse proxy + remote_addr = request.environ.get("HTTP_X_REAL_IP") + if not remote_addr: + remote_addr = request.environ.get("REMOTE_ADDR") + + return remote_addr + + class API: """ The :class:`API` is in charge of the interface between the user and the tower. It handles and server user requests. @@ -24,9 +43,48 @@ class API: watcher (:obj:`Watcher `): a ``Watcher`` instance to pass the requests to. """ - def __init__(self, inspector, watcher): + # TODO: UNITTEST + def __init__(self, inspector, watcher, gatekeeper): self.inspector = inspector self.watcher = watcher + self.gatekeeper = gatekeeper + + # TODO: UNITTEST + def register(self): + remote_addr = get_remote_addr() + + logger.info("Received register request", from_addr="{}".format(remote_addr)) + + if request.is_json: + request_data = request.get_json() + client_pk = request_data.get("public_key") + + if client_pk: + try: + rcode = HTTP_OK + available_slots = self.gatekeeper.add_update_user(client_pk) + response = {"public_key": client_pk, "available_slots": available_slots} + + except ValueError as e: + rcode = HTTP_BAD_REQUEST + error = "Error {}: {}".format(errors.REGISTRATION_MISSING_FIELD, str(e)) + response = {"error": error} + + else: + rcode = HTTP_BAD_REQUEST + error = "Error {}: public_key not found in register message".format( + errors.REGISTRATION_WRONG_FIELD_FORMAT + ) + response = {"error": error} + + else: + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Request is not json encoded" + response = {"error": error} + + logger.info("Sending response and disconnecting", from_addr="{}".format(remote_addr), response=response) + + return jsonify(response), rcode def add_appointment(self): """ @@ -43,15 +101,10 @@ class API: """ # Getting the real IP if the server is behind a reverse proxy - remote_addr = request.environ.get("HTTP_X_REAL_IP") - if not remote_addr: - remote_addr = request.environ.get("REMOTE_ADDR") + remote_addr = get_remote_addr() logger.info("Received add_appointment request", from_addr="{}".format(remote_addr)) - # FIXME: Logging every request so we can get better understanding of bugs in the alpha - logger.debug("Request details", data="{}".format(request.data)) - if request.is_json: # Check content type once if properly defined request_data = request.get_json() @@ -59,9 +112,6 @@ class API: request_data.get("appointment"), request_data.get("signature"), request_data.get("public_key") ) - error = None - response = None - if type(appointment) == Appointment: appointment_added, signature = self.watcher.add_appointment(appointment) @@ -72,29 +122,26 @@ class API: else: rcode = HTTP_SERVICE_UNAVAILABLE error = "appointment rejected" + response = {"error": error} elif type(appointment) == tuple: rcode = HTTP_BAD_REQUEST error = "appointment rejected. Error {}: {}".format(appointment[0], appointment[1]) + response = {"error": error} else: # We should never end up here, since inspect only returns appointments or tuples. Just in case. rcode = HTTP_BAD_REQUEST error = "appointment rejected. Request does not match the standard" + response = {"error": error} else: rcode = HTTP_BAD_REQUEST error = "appointment rejected. Request is not json encoded" - response = None + response = {"error": error} - logger.info( - "Sending response and disconnecting", from_addr="{}".format(remote_addr), response=response, error=error - ) - - if error is None: - return jsonify(response), rcode - else: - return jsonify({"error": error}), rcode + logger.info("Sending response and disconnecting", from_addr="{}".format(remote_addr), response=response) + return jsonify(response), rcode # FIXME: THE NEXT TWO API ENDPOINTS ARE FOR TESTING AND SHOULD BE REMOVED / PROPERLY MANAGED BEFORE PRODUCTION! # ToDo: #17-add-api-keys @@ -116,9 +163,7 @@ class API: """ # Getting the real IP if the server is behind a reverse proxy - remote_addr = request.environ.get("HTTP_X_REAL_IP") - if not remote_addr: - remote_addr = request.environ.get("REMOTE_ADDR") + remote_addr = get_remote_addr() locator = request.args.get("locator") response = [] @@ -182,12 +227,14 @@ class API: return response + # TODO: UNITTEST def start(self): """ This function starts the Flask server used to run the API. Adds all the routes to the functions listed above. """ routes = { + "/register": (self.register, ["POST"]), "/add_appointment": (self.add_appointment, ["POST"]), "/get_appointment": (self.get_appointment, ["GET"]), "/get_all_appointments": (self.get_all_appointments, ["GET"]), diff --git a/teos/errors.py b/teos/errors.py index 747103b..c4b5b25 100644 --- a/teos/errors.py +++ b/teos/errors.py @@ -1,4 +1,4 @@ -# Appointment errors +# Appointment errors [-1, -64] APPOINTMENT_EMPTY_FIELD = -1 APPOINTMENT_WRONG_FIELD_TYPE = -2 APPOINTMENT_WRONG_FIELD_SIZE = -3 @@ -8,6 +8,10 @@ APPOINTMENT_FIELD_TOO_BIG = -6 APPOINTMENT_WRONG_FIELD = -7 APPOINTMENT_INVALID_SIGNATURE = -8 +# Registration errors [-65, -128] +REGISTRATION_MISSING_FIELD = -65 +REGISTRATION_WRONG_FIELD_FORMAT = -66 + # Custom RPC errors RPC_TX_REORGED_AFTER_BROADCAST = -98 diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py new file mode 100644 index 0000000..75d71f7 --- /dev/null +++ b/teos/gatekeeper.py @@ -0,0 +1,32 @@ +import re + +SUBSCRIPTION_SLOTS = 100 + +# TODO: UNITTEST +class Gatekeeper: + def __init__(self): + self.registered_users = {} + + @staticmethod + def check_user_pk(user_pk): + """ + Checks if a given value is a 33-byte hex encoded string. + + Args: + user_pk(:mod:`str`): the value to be checked. + + Returns: + :obj:`bool`: Whether or not the value matches the format. + """ + return isinstance(user_pk, str) and re.match(r"^[0-9A-Fa-f]{66}$", user_pk) is not None + + def add_update_user(self, user_pk): + if not self.check_user_pk(user_pk): + raise ValueError("provided public key does not match expected format (33-byte hex string)") + + if user_pk not in self.registered_users: + self.registered_users[user_pk] = SUBSCRIPTION_SLOTS + else: + self.registered_users[user_pk] += SUBSCRIPTION_SLOTS + + return self.registered_users[user_pk] diff --git a/teos/teosd.py b/teos/teosd.py index dd63bd6..2f46341 100644 --- a/teos/teosd.py +++ b/teos/teosd.py @@ -17,6 +17,7 @@ from teos.carrier import Carrier from teos.inspector import Inspector from teos.responder import Responder from teos.db_manager import DBManager +from teos.gatekeeper import Gatekeeper from teos.chain_monitor import ChainMonitor from teos.block_processor import BlockProcessor from teos.tools import can_connect_to_bitcoind, in_correct_network @@ -150,7 +151,7 @@ def main(command_line_conf): # Fire the API and the ChainMonitor # FIXME: 92-block-data-during-bootstrap-db chain_monitor.monitor_chain() - API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher).start() + API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, Gatekeeper()).start() except Exception as e: logger.error("An error occurred: {}. Shutting down".format(e)) exit(1) diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 6cbf15f..c1d4786 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -10,6 +10,7 @@ from teos.watcher import Watcher from teos.tools import bitcoin_cli from teos.inspector import Inspector from teos.responder import Responder +from teos.gatekeeper import Gatekeeper from teos.chain_monitor import ChainMonitor from test.teos.unit.conftest import ( @@ -55,7 +56,9 @@ def run_api(db_manager, carrier, block_processor): watcher.awake() chain_monitor.monitor_chain() - api_thread = Thread(target=API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher).start) + api_thread = Thread( + target=API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, Gatekeeper()).start + ) api_thread.daemon = True api_thread.start() From 83b3913cb5ee59cc52b287b3462bcd23c2d08c33 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 25 Mar 2020 17:13:35 +0100 Subject: [PATCH 04/62] Adds basic authentication logic. + Users need to be registered in order to send appointments (free registration for now) + The tower gives them a number of appointments to work with + Non-registered users and users with no enough appoitnemnts slots return the same error (to avoid proving) - Authentication does not cover get_* requests yet - No tests - No docs --- common/constants.py | 3 ++ common/cryptographer.py | 19 ++++++++++++ teos/api.py | 52 ++++++++++++++++++++++++------- teos/errors.py | 2 +- teos/gatekeeper.py | 69 +++++++++++++++++++++++++++++++++++++++-- teos/inspector.py | 62 ++---------------------------------- 6 files changed, 133 insertions(+), 74 deletions(-) diff --git a/common/constants.py b/common/constants.py index b577044..bf119aa 100644 --- a/common/constants.py +++ b/common/constants.py @@ -6,3 +6,6 @@ LOCATOR_LEN_BYTES = LOCATOR_LEN_HEX // 2 HTTP_OK = 200 HTTP_BAD_REQUEST = 400 HTTP_SERVICE_UNAVAILABLE = 503 + +# Temporary constants, may be changed +ENCRYPTED_BLOB_MAX_SIZE_HEX = 2 * 2048 diff --git a/common/cryptographer.py b/common/cryptographer.py index 3e65099..ecd47a8 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -315,3 +315,22 @@ class Cryptographer: """ return pk.point() == rpk.point() + + # TODO: UNITTEST + @staticmethod + def get_compressed_pk(pk): + """ + Computes a compressed, hex encoded, public key given a ``PublicKey`` object. + + Args: + pk(:obj:`PublicKey`): a given public key. + + Returns: + :obj:`str`: A compressed, hex encoded, public key (33-byte long) + """ + + if not isinstance(pk, PublicKey): + logger.error("The received data is not a PublicKey object") + return None + + return hexlify(pk.format(compressed=True)).decode("utf-8") diff --git a/teos/api.py b/teos/api.py index 8abf790..79a5317 100644 --- a/teos/api.py +++ b/teos/api.py @@ -1,5 +1,6 @@ import os import logging +from math import ceil from flask import Flask, request, abort, jsonify import teos.errors as errors @@ -7,7 +8,13 @@ from teos import HOST, PORT, LOG_PREFIX from common.logger import Logger from common.appointment import Appointment -from common.constants import HTTP_OK, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, LOCATOR_LEN_HEX +from common.constants import ( + HTTP_OK, + HTTP_BAD_REQUEST, + HTTP_SERVICE_UNAVAILABLE, + LOCATOR_LEN_HEX, + ENCRYPTED_BLOB_MAX_SIZE_HEX, +) # ToDo: #5-add-async-to-api @@ -34,6 +41,7 @@ def get_remote_addr(): class API: + # FIXME: DOCS """ The :class:`API` is in charge of the interface between the user and the tower. It handles and server user requests. @@ -49,7 +57,7 @@ class API: self.watcher = watcher self.gatekeeper = gatekeeper - # TODO: UNITTEST + # TODO: UNITTEST, DOCS def register(self): remote_addr = get_remote_addr() @@ -86,6 +94,7 @@ class API: return jsonify(response), rcode + # FIXME: UNITTEST def add_appointment(self): """ Main endpoint of the Watchtower. @@ -108,20 +117,41 @@ class API: if request.is_json: # Check content type once if properly defined request_data = request.get_json() - appointment = self.inspector.inspect( - request_data.get("appointment"), request_data.get("signature"), request_data.get("public_key") + + rcode, message = self.gatekeeper.identify_user( + request_data.get("appointment"), request_data.get("signature") ) - if type(appointment) == Appointment: - appointment_added, signature = self.watcher.add_appointment(appointment) + if rcode: + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Error {}: {}".format(rcode, message) + return jsonify({"error": error}), rcode - if appointment_added: - rcode = HTTP_OK - response = {"locator": appointment.locator, "signature": signature} + else: + user_pk = message + + appointment = self.inspector.inspect(request_data.get("appointment")) + + if type(appointment) == Appointment: + # An appointment will fill 1 slot per ENCRYPTED_BLOB_MAX_SIZE_HEX block. + required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) + + if self.gatekeeper.get_slots(user_pk) >= required_slots: + appointment_added, signature = self.watcher.add_appointment(appointment) + + if appointment_added: + rcode = HTTP_OK + response = {"locator": appointment.locator, "signature": signature} + self.gatekeeper.fill_subscription_slots(user_pk, required_slots) + + else: + rcode = HTTP_SERVICE_UNAVAILABLE + error = "appointment rejected" + response = {"error": error} else: - rcode = HTTP_SERVICE_UNAVAILABLE - error = "appointment rejected" + rcode = errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS + error = "invalid signature or the user does not have enough slots available" response = {"error": error} elif type(appointment) == tuple: diff --git a/teos/errors.py b/teos/errors.py index c4b5b25..fb7ef1d 100644 --- a/teos/errors.py +++ b/teos/errors.py @@ -6,7 +6,7 @@ APPOINTMENT_WRONG_FIELD_FORMAT = -4 APPOINTMENT_FIELD_TOO_SMALL = -5 APPOINTMENT_FIELD_TOO_BIG = -6 APPOINTMENT_WRONG_FIELD = -7 -APPOINTMENT_INVALID_SIGNATURE = -8 +APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS = -8 # Registration errors [-65, -128] REGISTRATION_MISSING_FIELD = -65 diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index 75d71f7..79eb34c 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -1,8 +1,13 @@ import re -SUBSCRIPTION_SLOTS = 100 +import teos.errors as errors -# TODO: UNITTEST +from common.appointment import Appointment +from common.cryptographer import Cryptographer + +SUBSCRIPTION_SLOTS = 1 + +# TODO: UNITTEST, DOCS class Gatekeeper: def __init__(self): self.registered_users = {} @@ -30,3 +35,63 @@ class Gatekeeper: self.registered_users[user_pk] += SUBSCRIPTION_SLOTS return self.registered_users[user_pk] + + def fill_subscription_slots(self, user_pk, n): + slots = self.registered_users.get(user_pk) + + # FIXME: This looks pretty dangerous. I'm guessing race conditions can happen here. + if slots == n: + self.registered_users.pop(user_pk) + else: + self.registered_users[user_pk] -= n + + def identify_user(self, appointment_data, signature): + """ + Checks if the provided user signature is comes from a registered user with available appointment slots. + + Args: + appointment_data (:obj:`dict`): the appointment that was signed by the user. + signature (:obj:`str`): the user's signature (hex encoded). + + Returns: + :obj:`tuple`: A tuple (return code, message) as follows: + + - ``(0, None)`` if the user can be identified (recovered pk belongs to a registered user) and the user has + available slots. + - ``!= (0, None)`` otherwise. + + The possible return errors are: ``APPOINTMENT_EMPTY_FIELD`` and ``APPOINTMENT_INVALID_SIGNATURE``. + """ + + if signature is None: + rcode = errors.APPOINTMENT_EMPTY_FIELD + message = "empty signature received" + + else: + appointment = Appointment.from_dict(appointment_data) + rpk = Cryptographer.recover_pk(appointment.serialize(), signature) + compressed_user_pk = Cryptographer.get_compressed_pk(rpk) + + if compressed_user_pk and compressed_user_pk in self.registered_users: + rcode = 0 + message = compressed_user_pk + + else: + rcode = errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS + message = "invalid signature or the user does not have enough slots available" + + return rcode, message + + def get_slots(self, user_pk): + """ + Returns the number os available slots for a given user. + + Args: + user_pk(:mod:`str`): the public key that identifies the user (33-bytes hex str) + + Returns: + :obj:`int`: the number of available slots. + + """ + slots = self.registered_users.get(user_pk) + return slots if slots is not None else 0 diff --git a/teos/inspector.py b/teos/inspector.py index 60a83fc..d00cc08 100644 --- a/teos/inspector.py +++ b/teos/inspector.py @@ -1,9 +1,7 @@ import re -from binascii import unhexlify import common.cryptographer from common.constants import LOCATOR_LEN_HEX -from common.cryptographer import Cryptographer, PublicKey from teos import errors, LOG_PREFIX from common.logger import Logger @@ -19,7 +17,6 @@ common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_ BLOCKS_IN_A_MONTH = 4320 # 4320 = roughly a month in blocks -ENCRYPTED_BLOB_MAX_SIZE_HEX = 2 * 2048 class Inspector: @@ -36,14 +33,13 @@ class Inspector: self.block_processor = block_processor self.min_to_self_delay = min_to_self_delay - def inspect(self, appointment_data, signature, public_key): + def inspect(self, appointment_data): """ Inspects whether the data provided by the user is correct. Args: appointment_data (:obj:`dict`): a dictionary containing the appointment data. - signature (:obj:`str`): the appointment signature provided by the user (hex encoded). - public_key (:obj:`str`): the user's public key (hex encoded). + Returns: :obj:`Appointment ` or :obj:`tuple`: An appointment initialized with the @@ -72,8 +68,6 @@ class Inspector: rcode, message = self.check_to_self_delay(appointment_data.get("to_self_delay")) if rcode == 0: rcode, message = self.check_blob(appointment_data.get("encrypted_blob")) - if rcode == 0: - rcode, message = self.check_appointment_signature(appointment_data, signature, public_key) if rcode == 0: r = Appointment.from_dict(appointment_data) @@ -330,10 +324,6 @@ class Inspector: rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE message = "wrong encrypted_blob data type ({})".format(t) - elif len(encrypted_blob) > ENCRYPTED_BLOB_MAX_SIZE_HEX: - rcode = errors.APPOINTMENT_FIELD_TOO_BIG - message = "encrypted_blob has to be 2Kib at most (current {})".format(len(encrypted_blob) // 2) - elif re.search(r"^[0-9A-Fa-f]+$", encrypted_blob) is None: rcode = errors.APPOINTMENT_WRONG_FIELD_FORMAT message = "wrong encrypted_blob format ({})".format(encrypted_blob) @@ -342,51 +332,3 @@ class Inspector: logger.error(message) return rcode, message - - @staticmethod - # Verifies that the appointment signature is a valid signature with public key - def check_appointment_signature(appointment_data, signature, pk): - """ - Checks if the provided user signature is correct. - - Args: - appointment_data (:obj:`dict`): the appointment that was signed by the user. - signature (:obj:`str`): the user's signature (hex encoded). - pk (:obj:`str`): the user's public key (hex encoded). - - Returns: - :obj:`tuple`: A tuple (return code, message) as follows: - - - ``(0, None)`` if the ``signature`` is correct. - - ``!= (0, None)`` otherwise. - - The possible return errors are: ``APPOINTMENT_EMPTY_FIELD``, ``APPOINTMENT_WRONG_FIELD_TYPE``, and - ``APPOINTMENT_WRONG_FIELD_FORMAT``. - """ - - message = None - rcode = 0 - - if signature is None: - rcode = errors.APPOINTMENT_EMPTY_FIELD - message = "empty signature received" - - elif pk is None: - rcode = errors.APPOINTMENT_EMPTY_FIELD - message = "empty public key received" - - elif re.match(r"^[0-9A-Fa-f]{66}$", pk) is None: - rcode = errors.APPOINTMENT_WRONG_FIELD - message = "public key must be a hex encoded 33-byte long value" - - else: - appointment = Appointment.from_dict(appointment_data) - rpk = Cryptographer.recover_pk(appointment.serialize(), signature) - pk = PublicKey(unhexlify(pk)) - valid_sig = Cryptographer.verify_rpk(pk, rpk) - - if not valid_sig: - rcode = errors.APPOINTMENT_INVALID_SIGNATURE - message = "invalid signature" - - return rcode, message From f7260bc1ce35fac2f305af9dc7500dfaed0b5226 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 25 Mar 2020 21:10:58 +0100 Subject: [PATCH 05/62] Improves and simplifies add_appointment --- teos/api.py | 73 ++++++++++++++++++++++++++-------------------- teos/gatekeeper.py | 55 +++++++++++++++------------------- 2 files changed, 65 insertions(+), 63 deletions(-) diff --git a/teos/api.py b/teos/api.py index 79a5317..24016a5 100644 --- a/teos/api.py +++ b/teos/api.py @@ -4,6 +4,7 @@ from math import ceil from flask import Flask, request, abort, jsonify import teos.errors as errors +from teos.gatekeeper import NotEnoughSlots from teos import HOST, PORT, LOG_PREFIX from common.logger import Logger @@ -117,52 +118,60 @@ class API: if request.is_json: # Check content type once if properly defined request_data = request.get_json() + user_pk = self.gatekeeper.identify_user(request_data.get("appointment"), request_data.get("signature")) - rcode, message = self.gatekeeper.identify_user( - request_data.get("appointment"), request_data.get("signature") - ) + if user_pk: + appointment = self.inspector.inspect(request_data.get("appointment")) - if rcode: - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Error {}: {}".format(rcode, message) - return jsonify({"error": error}), rcode + if type(appointment) == Appointment: + # An appointment will fill 1 slot per ENCRYPTED_BLOB_MAX_SIZE_HEX block. + required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) - else: - user_pk = message + # Temporarily taking out slots to avoid abusing this via race conditions + # DISCUSS: It may be worth using signals here to avoid race conditions anyway + try: + self.gatekeeper.fill_slots(user_pk, required_slots) - appointment = self.inspector.inspect(request_data.get("appointment")) + appointment_added, signature = self.watcher.add_appointment(appointment) - if type(appointment) == Appointment: - # An appointment will fill 1 slot per ENCRYPTED_BLOB_MAX_SIZE_HEX block. - required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) + if appointment_added: + rcode = HTTP_OK + response = {"locator": appointment.locator, "signature": signature} - if self.gatekeeper.get_slots(user_pk) >= required_slots: - appointment_added, signature = self.watcher.add_appointment(appointment) + else: + # Adding back the slots since they were not used + self.gatekeeper.free_slots(user_pk, required_slots) + rcode = HTTP_SERVICE_UNAVAILABLE + error = "appointment rejected" + response = {"error": error} - if appointment_added: - rcode = HTTP_OK - response = {"locator": appointment.locator, "signature": signature} - self.gatekeeper.fill_subscription_slots(user_pk, required_slots) - - else: - rcode = HTTP_SERVICE_UNAVAILABLE - error = "appointment rejected" + except NotEnoughSlots: + # Adding back the slots since they were not used + self.gatekeeper.free_slots(user_pk, required_slots) + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Error {}: {}".format( + errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS, + "Invalid signature or the user does not have enough slots available", + ) response = {"error": error} - else: - rcode = errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS - error = "invalid signature or the user does not have enough slots available" + elif type(appointment) == tuple: + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Error {}: {}".format(appointment[0], appointment[1]) response = {"error": error} - elif type(appointment) == tuple: - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Error {}: {}".format(appointment[0], appointment[1]) - response = {"error": error} + else: + # We should never end up here, since inspect only returns appointments or tuples. Just in case. + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Request does not match the standard" + response = {"error": error} else: - # We should never end up here, since inspect only returns appointments or tuples. Just in case. rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Request does not match the standard" + error = "appointment rejected. Error {}: {}".format( + errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS, + "Invalid signature or the user does not have enough slots available", + ) response = {"error": error} else: diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index 79eb34c..67dbdc2 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -1,13 +1,18 @@ import re -import teos.errors as errors - from common.appointment import Appointment from common.cryptographer import Cryptographer SUBSCRIPTION_SLOTS = 1 + # TODO: UNITTEST, DOCS +class NotEnoughSlots(ValueError): + """Raise this when trying to subtract more slots than a user has available""" + + pass + + class Gatekeeper: def __init__(self): self.registered_users = {} @@ -36,15 +41,6 @@ class Gatekeeper: return self.registered_users[user_pk] - def fill_subscription_slots(self, user_pk, n): - slots = self.registered_users.get(user_pk) - - # FIXME: This looks pretty dangerous. I'm guessing race conditions can happen here. - if slots == n: - self.registered_users.pop(user_pk) - else: - self.registered_users[user_pk] -= n - def identify_user(self, appointment_data, signature): """ Checks if the provided user signature is comes from a registered user with available appointment slots. @@ -54,33 +50,21 @@ class Gatekeeper: signature (:obj:`str`): the user's signature (hex encoded). Returns: - :obj:`tuple`: A tuple (return code, message) as follows: - - - ``(0, None)`` if the user can be identified (recovered pk belongs to a registered user) and the user has - available slots. - - ``!= (0, None)`` otherwise. - - The possible return errors are: ``APPOINTMENT_EMPTY_FIELD`` and ``APPOINTMENT_INVALID_SIGNATURE``. + :obj:`str` or `None`: a compressed key if it can be recovered from the signature and matches a registered + user. ``None`` otherwise. """ - if signature is None: - rcode = errors.APPOINTMENT_EMPTY_FIELD - message = "empty signature received" + user_pk = None - else: + if signature is not None: appointment = Appointment.from_dict(appointment_data) rpk = Cryptographer.recover_pk(appointment.serialize(), signature) - compressed_user_pk = Cryptographer.get_compressed_pk(rpk) + compressed_pk = Cryptographer.get_compressed_pk(rpk) - if compressed_user_pk and compressed_user_pk in self.registered_users: - rcode = 0 - message = compressed_user_pk + if compressed_pk in self.registered_users and self.registered_users.get(compressed_pk) > 0: + user_pk = compressed_pk - else: - rcode = errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS - message = "invalid signature or the user does not have enough slots available" - - return rcode, message + return user_pk def get_slots(self, user_pk): """ @@ -95,3 +79,12 @@ class Gatekeeper: """ slots = self.registered_users.get(user_pk) return slots if slots is not None else 0 + + def fill_slots(self, user_pk, n): + if n >= self.registered_users.get(user_pk): + self.registered_users[user_pk] -= n + else: + raise NotEnoughSlots("No enough empty slots") + + def free_slots(self, user_pk, n): + self.registered_users[user_pk] += n From 4fad6b7b6f5079ecf5bbc6c6674d81a2521a645f Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 17:09:15 +0100 Subject: [PATCH 06/62] Fixes docstring --- common/cryptographer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/cryptographer.py b/common/cryptographer.py index ecd47a8..4a1dbeb 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -265,7 +265,7 @@ class Cryptographer: Recovers an ECDSA public key from a given message and zbase32 signature. Args: - message(:obj:`bytes`): the data to be signed. + message(:obj:`bytes`): original message from where the signature was generated. zb32_sig(:obj:`str`): the zbase32 signature of the message. Returns: From b3d67b4ce36ae47191c27b6035dc131b4962a024 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 17:10:16 +0100 Subject: [PATCH 07/62] Refactors Inspector so it returns exceptions for invalid data instead of error codes Defines InspectionFailed, and exception that it's raised for invalid parameters. The design looks much cleaner now. --- teos/inspector.py | 275 ++++++++++++++++------------------------------ 1 file changed, 95 insertions(+), 180 deletions(-) diff --git a/teos/inspector.py b/teos/inspector.py index d00cc08..c5fa9af 100644 --- a/teos/inspector.py +++ b/teos/inspector.py @@ -1,11 +1,11 @@ import re import common.cryptographer +from common.logger import Logger from common.constants import LOCATOR_LEN_HEX +from common.appointment import Appointment from teos import errors, LOG_PREFIX -from common.logger import Logger -from common.appointment import Appointment logger = Logger(actor="Inspector", log_name_prefix=LOG_PREFIX) common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX) @@ -19,6 +19,14 @@ common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_ BLOCKS_IN_A_MONTH = 4320 # 4320 = roughly a month in blocks +class InspectionFailed(Exception): + """Raise this the inspector finds a problem with any of the appointment fields""" + + def __init__(self, erno, reason): + self.erno = erno + self.reason = reason + + class Inspector: """ The :class:`Inspector` class is in charge of verifying that the appointment data provided by the user is correct. @@ -42,43 +50,29 @@ class Inspector: Returns: - :obj:`Appointment ` or :obj:`tuple`: An appointment initialized with the - provided data if it is correct. + :obj:`Appointment `: An appointment initialized with the + provided data. - Returns a tuple ``(return code, message)`` describing the error otherwise. - - Errors are defined in :mod:`Errors `. + Raises: + :obj:`InspectionFailed `: if any of the fields is wrong. """ if appointment_data is None: - return errors.APPOINTMENT_EMPTY_FIELD, "empty appointment received" + raise InspectionFailed(errors.APPOINTMENT_EMPTY_FIELD, "empty appointment received") + elif not isinstance(appointment_data, dict): + raise InspectionFailed(errors.APPOINTMENT_WRONG_FIELD, "wrong appointment format") block_height = self.block_processor.get_block_count() + if block_height is None: + raise InspectionFailed(errors.UNKNOWN_JSON_RPC_EXCEPTION, "Unexpected error occurred") - if block_height is not None: - rcode, message = self.check_locator(appointment_data.get("locator")) + self.check_locator(appointment_data.get("locator")) + self.check_start_time(appointment_data.get("start_time"), block_height) + self.check_end_time(appointment_data.get("end_time"), appointment_data.get("start_time"), block_height) + self.check_to_self_delay(appointment_data.get("to_self_delay")) + self.check_blob(appointment_data.get("encrypted_blob")) - if rcode == 0: - rcode, message = self.check_start_time(appointment_data.get("start_time"), block_height) - if rcode == 0: - rcode, message = self.check_end_time( - appointment_data.get("end_time"), appointment_data.get("start_time"), block_height - ) - if rcode == 0: - rcode, message = self.check_to_self_delay(appointment_data.get("to_self_delay")) - if rcode == 0: - rcode, message = self.check_blob(appointment_data.get("encrypted_blob")) - - if rcode == 0: - r = Appointment.from_dict(appointment_data) - else: - r = (rcode, message) - - else: - # In case of an unknown exception, assign a special rcode and reason. - r = (errors.UNKNOWN_JSON_RPC_EXCEPTION, "Unexpected error occurred") - - return r + return Appointment.from_dict(appointment_data) @staticmethod def check_locator(locator): @@ -90,40 +84,23 @@ class Inspector: Args: locator (:obj:`str`): the locator to be checked. - Returns: - :obj:`tuple`: A tuple (return code, message) as follows: - - - ``(0, None)`` if the ``locator`` is correct. - - ``!= (0, None)`` otherwise. - - The possible return errors are: ``APPOINTMENT_EMPTY_FIELD``, ``APPOINTMENT_WRONG_FIELD_TYPE``, - ``APPOINTMENT_WRONG_FIELD_SIZE``, and ``APPOINTMENT_WRONG_FIELD_FORMAT``. + Raises: + :obj:`InspectionFailed `: if any of the fields is wrong. """ - message = None - rcode = 0 - if locator is None: - rcode = errors.APPOINTMENT_EMPTY_FIELD - message = "empty locator received" + raise InspectionFailed(errors.APPOINTMENT_EMPTY_FIELD, "empty locator received") elif type(locator) != str: - rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE - message = "wrong locator data type ({})".format(type(locator)) + raise InspectionFailed( + errors.APPOINTMENT_WRONG_FIELD_TYPE, "wrong locator data type ({})".format(type(locator)) + ) elif len(locator) != LOCATOR_LEN_HEX: - rcode = errors.APPOINTMENT_WRONG_FIELD_SIZE - message = "wrong locator size ({})".format(len(locator)) - # TODO: #12-check-txid-regexp + raise InspectionFailed(errors.APPOINTMENT_WRONG_FIELD_SIZE, "wrong locator size ({})".format(len(locator))) elif re.search(r"^[0-9A-Fa-f]+$", locator) is None: - rcode = errors.APPOINTMENT_WRONG_FIELD_FORMAT - message = "wrong locator format ({})".format(locator) - - if message is not None: - logger.error(message) - - return rcode, message + raise InspectionFailed(errors.APPOINTMENT_WRONG_FIELD_FORMAT, "wrong locator format ({})".format(locator)) @staticmethod def check_start_time(start_time, block_height): @@ -136,50 +113,35 @@ class Inspector: start_time (:obj:`int`): the block height at which the tower is requested to start watching for breaches. block_height (:obj:`int`): the chain height. - Returns: - :obj:`tuple`: A tuple (return code, message) as follows: - - - ``(0, None)`` if the ``start_time`` is correct. - - ``!= (0, None)`` otherwise. - - The possible return errors are: ``APPOINTMENT_EMPTY_FIELD``, ``APPOINTMENT_WRONG_FIELD_TYPE``, and - ``APPOINTMENT_FIELD_TOO_SMALL``. + Raises: + :obj:`InspectionFailed `: if any of the fields is wrong. """ - message = None - rcode = 0 - # TODO: What's too close to the current height is not properly defined. Right now any appointment that is in the # future will be accepted (even if it's only one block away). - t = type(start_time) - if start_time is None: - rcode = errors.APPOINTMENT_EMPTY_FIELD - message = "empty start_time received" + raise InspectionFailed(errors.APPOINTMENT_EMPTY_FIELD, "empty start_time received") - elif t != int: - rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE - message = "wrong start_time data type ({})".format(t) + elif type(start_time) != int: + raise InspectionFailed( + errors.APPOINTMENT_WRONG_FIELD_TYPE, "wrong start_time data type ({})".format(type(start_time)) + ) - elif start_time <= block_height: - rcode = errors.APPOINTMENT_FIELD_TOO_SMALL - if start_time < block_height: - message = "start_time is in the past" - else: - message = ( - "start_time is too close to current height. " - "Accepted times are: [current_height+1, current_height+6]" - ) + elif start_time < block_height: + raise InspectionFailed(errors.APPOINTMENT_FIELD_TOO_SMALL, "start_time is in the past") + + elif start_time == block_height: + raise InspectionFailed( + errors.APPOINTMENT_FIELD_TOO_SMALL, + "start_time is too close to current height. Accepted times are: [current_height+1, current_height+6]", + ) elif start_time > block_height + 6: - rcode = errors.APPOINTMENT_FIELD_TOO_BIG - message = "start_time is too far in the future. Accepted start times are up to 6 blocks in the future" - - if message is not None: - logger.error(message) - - return rcode, message + raise InspectionFailed( + errors.APPOINTMENT_FIELD_TOO_BIG, + "start_time is too far in the future. Accepted start times are up to 6 blocks in the future", + ) @staticmethod def check_end_time(end_time, start_time, block_height): @@ -193,54 +155,36 @@ class Inspector: start_time (:obj:`int`): the block height at which the tower is requested to start watching for breaches. block_height (:obj:`int`): the chain height. - Returns: - :obj:`tuple`: A tuple (return code, message) as follows: - - - ``(0, None)`` if the ``end_time`` is correct. - - ``!= (0, None)`` otherwise. - - The possible return errors are: ``APPOINTMENT_EMPTY_FIELD``, ``APPOINTMENT_WRONG_FIELD_TYPE``, and - ``APPOINTMENT_FIELD_TOO_SMALL``. + Raises: + :obj:`InspectionFailed `: if any of the fields is wrong. """ - message = None - rcode = 0 - # TODO: What's too close to the current height is not properly defined. Right now any appointment that ends in # the future will be accepted (even if it's only one block away). - t = type(end_time) - if end_time is None: - rcode = errors.APPOINTMENT_EMPTY_FIELD - message = "empty end_time received" + raise InspectionFailed(errors.APPOINTMENT_EMPTY_FIELD, "empty end_time received") - elif t != int: - rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE - message = "wrong end_time data type ({})".format(t) + elif type(end_time) != int: + raise InspectionFailed( + errors.APPOINTMENT_WRONG_FIELD_TYPE, "wrong end_time data type ({})".format(type(end_time)) + ) elif end_time > block_height + BLOCKS_IN_A_MONTH: # 4320 = roughly a month in blocks - rcode = errors.APPOINTMENT_FIELD_TOO_BIG - message = "end_time should be within the next month (<= current_height + 4320)" + raise InspectionFailed( + errors.APPOINTMENT_FIELD_TOO_BIG, "end_time should be within the next month (<= current_height + 4320)" + ) + elif start_time > end_time: + raise InspectionFailed(errors.APPOINTMENT_FIELD_TOO_SMALL, "end_time is smaller than start_time") - elif start_time >= end_time: - rcode = errors.APPOINTMENT_FIELD_TOO_SMALL - if start_time > end_time: - message = "end_time is smaller than start_time" - else: - message = "end_time is equal to start_time" + elif start_time == end_time: + raise InspectionFailed(errors.APPOINTMENT_FIELD_TOO_SMALL, "end_time is equal to start_time") - elif block_height >= end_time: - rcode = errors.APPOINTMENT_FIELD_TOO_SMALL - if block_height > end_time: - message = "end_time is in the past" - else: - message = "end_time is too close to current height" + elif block_height > end_time: + raise InspectionFailed(errors.APPOINTMENT_FIELD_TOO_SMALL, "end_time is in the past") - if message is not None: - logger.error(message) - - return rcode, message + elif block_height == end_time: + raise InspectionFailed(errors.APPOINTMENT_FIELD_TOO_SMALL, "end_time is too close to current height") def check_to_self_delay(self, to_self_delay): """ @@ -252,46 +196,32 @@ class Inspector: to_self_delay (:obj:`int`): The ``to_self_delay`` encoded in the ``csv`` of the ``htlc`` that this appointment is covering. - Returns: - :obj:`tuple`: A tuple (return code, message) as follows: - - - ``(0, None)`` if the ``to_self_delay`` is correct. - - ``!= (0, None)`` otherwise. - - The possible return errors are: ``APPOINTMENT_EMPTY_FIELD``, ``APPOINTMENT_WRONG_FIELD_TYPE``, and - ``APPOINTMENT_FIELD_TOO_SMALL``. + Raises: + :obj:`InspectionFailed `: if any of the fields is wrong. """ - message = None - rcode = 0 - - t = type(to_self_delay) - if to_self_delay is None: - rcode = errors.APPOINTMENT_EMPTY_FIELD - message = "empty to_self_delay received" + raise InspectionFailed(errors.APPOINTMENT_EMPTY_FIELD, "empty to_self_delay received") - elif t != int: - rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE - message = "wrong to_self_delay data type ({})".format(t) + elif type(to_self_delay) != int: + raise InspectionFailed( + errors.APPOINTMENT_WRONG_FIELD_TYPE, "wrong to_self_delay data type ({})".format(type(to_self_delay)) + ) elif to_self_delay > pow(2, 32): - rcode = errors.APPOINTMENT_FIELD_TOO_BIG - message = "to_self_delay must fit the transaction nLockTime field ({} > {})".format( - to_self_delay, pow(2, 32) + raise InspectionFailed( + errors.APPOINTMENT_FIELD_TOO_BIG, + "to_self_delay must fit the transaction nLockTime field ({} > {})".format(to_self_delay, pow(2, 32)), ) elif to_self_delay < self.min_to_self_delay: - rcode = errors.APPOINTMENT_FIELD_TOO_SMALL - message = "to_self_delay too small. The to_self_delay should be at least {} (current: {})".format( - self.min_to_self_delay, to_self_delay + raise InspectionFailed( + errors.APPOINTMENT_FIELD_TOO_SMALL, + "to_self_delay too small. The to_self_delay should be at least {} (current: {})".format( + self.min_to_self_delay, to_self_delay + ), ) - if message is not None: - logger.error(message) - - return rcode, message - # ToDo: #6-define-checks-encrypted-blob @staticmethod def check_blob(encrypted_blob): @@ -301,34 +231,19 @@ class Inspector: Args: encrypted_blob (:obj:`str`): the encrypted blob to be checked (hex encoded). - Returns: - :obj:`tuple`: A tuple (return code, message) as follows: - - - ``(0, None)`` if the ``encrypted_blob`` is correct. - - ``!= (0, None)`` otherwise. - - The possible return errors are: ``APPOINTMENT_EMPTY_FIELD``, ``APPOINTMENT_WRONG_FIELD_TYPE``, and - ``APPOINTMENT_WRONG_FIELD_FORMAT``. + Raises: + :obj:`InspectionFailed `: if any of the fields is wrong. """ - message = None - rcode = 0 - - t = type(encrypted_blob) - if encrypted_blob is None: - rcode = errors.APPOINTMENT_EMPTY_FIELD - message = "empty encrypted_blob received" + raise InspectionFailed(errors.APPOINTMENT_EMPTY_FIELD, "empty encrypted_blob received") - elif t != str: - rcode = errors.APPOINTMENT_WRONG_FIELD_TYPE - message = "wrong encrypted_blob data type ({})".format(t) + elif type(encrypted_blob) != str: + raise InspectionFailed( + errors.APPOINTMENT_WRONG_FIELD_TYPE, "wrong encrypted_blob data type ({})".format(type(encrypted_blob)) + ) elif re.search(r"^[0-9A-Fa-f]+$", encrypted_blob) is None: - rcode = errors.APPOINTMENT_WRONG_FIELD_FORMAT - message = "wrong encrypted_blob format ({})".format(encrypted_blob) - - if message is not None: - logger.error(message) - - return rcode, message + raise InspectionFailed( + errors.APPOINTMENT_WRONG_FIELD_FORMAT, "wrong encrypted_blob format ({})".format(encrypted_blob) + ) From bbe93ae3f141dd483c2407c105c1215928407283 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 17:13:38 +0100 Subject: [PATCH 08/62] Adds RIPMED160 hash function --- common/cryptographer.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/common/cryptographer.py b/common/cryptographer.py index 4a1dbeb..f5427e7 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -1,5 +1,5 @@ import pyzbase32 -from hashlib import sha256 +from hashlib import sha256, new from binascii import unhexlify, hexlify from coincurve.utils import int_to_bytes from coincurve import PrivateKey, PublicKey @@ -25,6 +25,24 @@ def sha256d(message): return sha256(sha256(message).digest()).digest() +def hash_160(message): + """ Calculates the RIPEMD-160 hash of a given message. + + Args: + message (:obj:`str`) the message to be hashed. + + Returns: + :obj:`str`: the ripemd160 hash of the given message. + """ + + # Calculate the RIPEMD-160 hash of the given data. + md = new("ripemd160") + md.update(unhexlify(message)) + h160 = md.hexdigest() + + return h160 + + def sigrec_encode(rsig_rid): """ Encodes a pk-recoverable signature to be used in LN. ```rsig_rid`` can be obtained trough From 9bc3bf2e6ec8fd3700e88fbfa6423be27e5145c3 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 17:21:18 +0100 Subject: [PATCH 09/62] Refactors and improves the gatekeeper Following the same apporach as the Inspector, the gatekeeper now raises exceptions depending on the error encountered. Also generalises `identify_user` so it can be used with any kind of user-signed message. Finally lifts the restriction of having available slots on `identify_user` (that only applied to `add_appointent`). --- teos/gatekeeper.py | 102 ++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 33 deletions(-) diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index 67dbdc2..6810014 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -1,19 +1,37 @@ import re -from common.appointment import Appointment from common.cryptographer import Cryptographer SUBSCRIPTION_SLOTS = 1 -# TODO: UNITTEST, DOCS +# TODO: UNITTEST class NotEnoughSlots(ValueError): - """Raise this when trying to subtract more slots than a user has available""" + """Raise this when trying to subtract more slots than a user has available.""" + + def __init__(self, user_pk, requested_slots): + self.user_pk = user_pk + self.requested_slots = requested_slots + + +class IdentificationFailure(Exception): + """ + Raise this when a user can not be identified. Either the user public key cannot be recovered or the user is + not found within the registered ones. + """ pass class Gatekeeper: + """ + The Gatekeeper is in charge of managing the access to the tower. Only registered users are allowed to perform + actions. + + Attributes: + registered_users (:obj:`dict`): a map of user_pk:appointment_slots. + """ + def __init__(self): self.registered_users = {} @@ -23,14 +41,25 @@ class Gatekeeper: Checks if a given value is a 33-byte hex encoded string. Args: - user_pk(:mod:`str`): the value to be checked. + user_pk(:obj:`str`): the value to be checked. Returns: :obj:`bool`: Whether or not the value matches the format. """ + return isinstance(user_pk, str) and re.match(r"^[0-9A-Fa-f]{66}$", user_pk) is not None def add_update_user(self, user_pk): + """ + Adds a new user or updates the subscription of an existing one, by adding additional slots. + + Args: + user_pk(:obj:`str`): the public key that identifies the user (33-bytes hex str). + + Returns: + :obj:`int`: the number of avaiable slots in the user subscription. + """ + if not self.check_user_pk(user_pk): raise ValueError("provided public key does not match expected format (33-byte hex string)") @@ -41,50 +70,57 @@ class Gatekeeper: return self.registered_users[user_pk] - def identify_user(self, appointment_data, signature): + def identify_user(self, message, signature): """ - Checks if the provided user signature is comes from a registered user with available appointment slots. + Checks if the provided user signature comes from a registered user. Args: - appointment_data (:obj:`dict`): the appointment that was signed by the user. + message (:obj:`bytes`): byte representation of the original message from where the signature was generated. signature (:obj:`str`): the user's signature (hex encoded). Returns: - :obj:`str` or `None`: a compressed key if it can be recovered from the signature and matches a registered - user. ``None`` otherwise. + :obj:`str`: a compressed key recovered from the signature and matching a registered user. + + Raises: + :obj:``: if the user cannot be identified. """ - user_pk = None - - if signature is not None: - appointment = Appointment.from_dict(appointment_data) - rpk = Cryptographer.recover_pk(appointment.serialize(), signature) + if isinstance(message, bytes) and isinstance(signature, str): + rpk = Cryptographer.recover_pk(message, signature) compressed_pk = Cryptographer.get_compressed_pk(rpk) - if compressed_pk in self.registered_users and self.registered_users.get(compressed_pk) > 0: - user_pk = compressed_pk + if compressed_pk in self.registered_users: + return compressed_pk + else: + raise IdentificationFailure("User not found.") - return user_pk - - def get_slots(self, user_pk): - """ - Returns the number os available slots for a given user. - - Args: - user_pk(:mod:`str`): the public key that identifies the user (33-bytes hex str) - - Returns: - :obj:`int`: the number of available slots. - - """ - slots = self.registered_users.get(user_pk) - return slots if slots is not None else 0 + else: + raise IdentificationFailure("Wrong message or signature.") def fill_slots(self, user_pk, n): - if n >= self.registered_users.get(user_pk): + """ + Fills a given number os slots of the user subscription. + + Args: + user_pk(:obj:`str`): the public key that identifies the user (33-bytes hex str). + n: the number of slots to fill. + + Raises: + :obj:``: if the user subscription does not have enough slots. + """ + + if n <= self.registered_users.get(user_pk): self.registered_users[user_pk] -= n else: - raise NotEnoughSlots("No enough empty slots") + raise NotEnoughSlots(user_pk, n) def free_slots(self, user_pk, n): + """ + Frees some slots of a user subscription. + + Args: + user_pk(:obj:`str`): the public key that identifies the user (33-bytes hex str). + n: the number of slots to free. + """ + self.registered_users[user_pk] += n From 460a98d42f28b92deb492c27e2337b3467fea778 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 17:36:37 +0100 Subject: [PATCH 10/62] Improves and simplifies the API by using the exceptions provided from the Inspector and Gatekeeper. - Refactors and simplifies `add_appointment` - Modifies `get_appointment` as a POST interface and to require user signatures. - Defines a new way of creating uuids to avoid storing user_pu:uuids maps. - The uuids are now defined as RIPMED160(locator|user_pk) instead of using `uuid4`. - Add missing docs and fixes the existing ones --- teos/api.py | 169 +++++++++++++++++++++++++++------------------------- 1 file changed, 87 insertions(+), 82 deletions(-) diff --git a/teos/api.py b/teos/api.py index 24016a5..efe9f81 100644 --- a/teos/api.py +++ b/teos/api.py @@ -4,18 +4,13 @@ from math import ceil from flask import Flask, request, abort, jsonify import teos.errors as errors -from teos.gatekeeper import NotEnoughSlots from teos import HOST, PORT, LOG_PREFIX +from teos.inspector import InspectionFailed +from teos.gatekeeper import NotEnoughSlots, IdentificationFailure from common.logger import Logger -from common.appointment import Appointment -from common.constants import ( - HTTP_OK, - HTTP_BAD_REQUEST, - HTTP_SERVICE_UNAVAILABLE, - LOCATOR_LEN_HEX, - ENCRYPTED_BLOB_MAX_SIZE_HEX, -) +from common.cryptographer import hash_160 +from common.constants import HTTP_OK, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, ENCRYPTED_BLOB_MAX_SIZE_HEX # ToDo: #5-add-async-to-api @@ -42,7 +37,6 @@ def get_remote_addr(): class API: - # FIXME: DOCS """ The :class:`API` is in charge of the interface between the user and the tower. It handles and server user requests. @@ -50,6 +44,7 @@ class API: inspector (:obj:`Inspector `): an ``Inspector`` instance to check the correctness of the received data. watcher (:obj:`Watcher `): a ``Watcher`` instance to pass the requests to. + gatekeeper (:obj:`Watcher `): a `Gatekeeper` instance in charge to gatekeep the API. """ # TODO: UNITTEST @@ -58,8 +53,22 @@ class API: self.watcher = watcher self.gatekeeper = gatekeeper - # TODO: UNITTEST, DOCS + # TODO: UNITTEST def register(self): + """ + Registers a user by creating a subscription. + + The user is identified by public key. + + Currently subscriptions are free. + + Returns: + :obj:`tuple`: A tuple containing the response (``json``) and response code (``int``). For accepted requests, + the ``rcode`` is always 200 and the response contains a json with the public key and number of slots in the + subscription. For rejected requests, the ``rcode`` is a 404 and the value contains an application specific + error, and an error message. Error messages can be found at :mod:`Errors `. + """ + remote_addr = get_remote_addr() logger.info("Received register request", from_addr="{}".format(remote_addr)) @@ -105,9 +114,9 @@ class API: Returns: :obj:`tuple`: A tuple containing the response (``json``) and response code (``int``). For accepted - appointments, the ``rcode`` is always 0 and the response contains the signed receipt. For rejected - appointments, the ``rcode`` is a negative value and the response contains the error message. Error messages - can be found at :mod:`Errors `. + appointments, the ``rcode`` is always 200 and the response contains the receipt signature. For rejected + appointments, the ``rcode`` is a 404 and the value contains an application specific error, and an error + message. Error messages can be found at :mod:`Errors `. """ # Getting the real IP if the server is behind a reverse proxy @@ -116,57 +125,44 @@ class API: logger.info("Received add_appointment request", from_addr="{}".format(remote_addr)) if request.is_json: - # Check content type once if properly defined request_data = request.get_json() - user_pk = self.gatekeeper.identify_user(request_data.get("appointment"), request_data.get("signature")) - if user_pk: + # We kind of have the chicken an the egg problem here. Data must be verified and the signature must be + # checked: + # + # - If we verify the data first, we may encounter that the signature is wrong and wasted some time. + # - If we check the signature first, we may need to verify some of the information or expose to build + # appointments with potentially wrong data, which may be exploitable. + # + # The first approach seems safer since it only implies a bunch of pretty quick checks. + + try: appointment = self.inspector.inspect(request_data.get("appointment")) + user_pk = self.gatekeeper.identify_user(appointment.serialize(), request_data.get("signature")) - if type(appointment) == Appointment: - # An appointment will fill 1 slot per ENCRYPTED_BLOB_MAX_SIZE_HEX block. - required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) + # An appointment will fill 1 slot per ENCRYPTED_BLOB_MAX_SIZE_HEX block. + # Temporarily taking out slots to avoid abusing this via race conditions. + # DISCUSS: It may be worth using signals here to avoid race conditions anyway. + required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) + self.gatekeeper.fill_slots(user_pk, required_slots) + appointment_added, signature = self.watcher.add_appointment(appointment, user_pk) - # Temporarily taking out slots to avoid abusing this via race conditions - # DISCUSS: It may be worth using signals here to avoid race conditions anyway - try: - self.gatekeeper.fill_slots(user_pk, required_slots) - - appointment_added, signature = self.watcher.add_appointment(appointment) - - if appointment_added: - rcode = HTTP_OK - response = {"locator": appointment.locator, "signature": signature} - - else: - # Adding back the slots since they were not used - self.gatekeeper.free_slots(user_pk, required_slots) - rcode = HTTP_SERVICE_UNAVAILABLE - error = "appointment rejected" - response = {"error": error} - - except NotEnoughSlots: - # Adding back the slots since they were not used - self.gatekeeper.free_slots(user_pk, required_slots) - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Error {}: {}".format( - errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS, - "Invalid signature or the user does not have enough slots available", - ) - response = {"error": error} - - elif type(appointment) == tuple: - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Error {}: {}".format(appointment[0], appointment[1]) - response = {"error": error} + if appointment_added: + rcode = HTTP_OK + response = {"locator": appointment.locator, "signature": signature} else: - # We should never end up here, since inspect only returns appointments or tuples. Just in case. - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Request does not match the standard" - response = {"error": error} + # Adding back the slots since they were not used + self.gatekeeper.free_slots(user_pk, required_slots) + rcode = HTTP_SERVICE_UNAVAILABLE + response = {"error": "appointment rejected"} - else: + except InspectionFailed as e: + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Error {}: {}".format(e.erno, e.reason) + response = {"error": error} + + except (IdentificationFailure, NotEnoughSlots) as e: rcode = HTTP_BAD_REQUEST error = "appointment rejected. Error {}: {}".format( errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS, @@ -204,40 +200,49 @@ class API: # Getting the real IP if the server is behind a reverse proxy remote_addr = get_remote_addr() - locator = request.args.get("locator") - response = [] + if request.is_json: + request_data = request.get_json() + locator = request_data.get("locator") - logger.info("Received get_appointment request", from_addr="{}".format(remote_addr), locator=locator) + try: + self.inspector.check_locator(locator) + logger.info("Received get_appointment request", from_addr="{}".format(remote_addr), locator=locator) - # ToDo: #15-add-system-monitor - if not isinstance(locator, str) or len(locator) != LOCATOR_LEN_HEX: - response.append({"locator": locator, "status": "not_found"}) - return jsonify(response) + message = "get appointment {}".format(locator).encode() + signature = request_data.get("signature") + user_pk = self.gatekeeper.identify_user(message, signature) - locator_map = self.watcher.db_manager.load_locator_map(locator) - triggered_appointments = self.watcher.db_manager.load_all_triggered_flags() + triggered_appointments = self.watcher.db_manager.load_all_triggered_flags() + uuid = hash_160("{}{}".format(locator, user_pk)) - if locator_map is not None: - for uuid in locator_map: - if uuid not in triggered_appointments: - appointment_data = self.watcher.db_manager.load_watcher_appointment(uuid) + # If the appointment has been triggered, it should be in the locator (default else just in case). + if uuid in triggered_appointments: + response = self.watcher.db_manager.load_responder_tracker(uuid) + if response: + response["status"] = "dispute_responded" + else: + response = {"locator": locator, "status": "not_found"} - if appointment_data is not None: - appointment_data["status"] = "being_watched" - response.append(appointment_data) + # Otherwise it should be either in the watcher, or not in the system. + else: + response = self.watcher.db_manager.load_watcher_appointment(uuid) + if response: + response["status"] = "being_watched" + else: + response = {"locator": locator, "status": "not_found"} - tracker_data = self.watcher.db_manager.load_responder_tracker(uuid) + except (InspectionFailed, IdentificationFailure): + response = {"locator": locator, "status": "not_found"} - if tracker_data is not None: - tracker_data["status"] = "dispute_responded" - response.append(tracker_data) + finally: + rcode = HTTP_OK else: - response.append({"locator": locator, "status": "not_found"}) + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Request is not json encoded" + response = {"error": error} - response = jsonify(response) - - return response + return jsonify(response), rcode def get_all_appointments(self): """ @@ -275,7 +280,7 @@ class API: routes = { "/register": (self.register, ["POST"]), "/add_appointment": (self.add_appointment, ["POST"]), - "/get_appointment": (self.get_appointment, ["GET"]), + "/get_appointment": (self.get_appointment, ["POST"]), "/get_all_appointments": (self.get_all_appointments, ["GET"]), } From 3d8c0a949882a53da4d64167c92f9d698a24bbde Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 17:42:10 +0100 Subject: [PATCH 11/62] Applies new uuid apporach to the Watcher --- teos/watcher.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/teos/watcher.py b/teos/watcher.py index 33efa3c..79cb2b6 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -1,4 +1,3 @@ -from uuid import uuid4 from queue import Queue from threading import Thread @@ -6,7 +5,7 @@ import common.cryptographer from common.logger import Logger from common.tools import compute_locator from common.appointment import Appointment -from common.cryptographer import Cryptographer +from common.cryptographer import Cryptographer, hash_160 from teos import LOG_PREFIX from teos.cleaner import Cleaner @@ -77,7 +76,7 @@ class Watcher: return watcher_thread - def add_appointment(self, appointment): + def add_appointment(self, appointment, user_pk): """ Adds a new appointment to the ``appointments`` dictionary if ``max_appointments`` has not been reached. @@ -96,6 +95,7 @@ class Watcher: Args: appointment (:obj:`Appointment `): the appointment to be added to the :obj:`Watcher`. + user_pk(:obj:`str`): the public key that identifies the user who sent the appointment (33-bytes hex str). Returns: :obj:`tuple`: A tuple signaling if the appointment has been added or not (based on ``max_appointments``). @@ -108,7 +108,10 @@ class Watcher: if len(self.appointments) < self.max_appointments: - uuid = uuid4().hex + # The uuids are generated as the RIPMED160(locator||user_pubkey), that way the tower does not need to know + # anything about the user from this point on (no need to store user_pk in the database). + # If an appointment is requested by the user the uuid can be recomputed and queried straightaway (no maps). + uuid = hash_160("{}{}".format(appointment.locator, user_pk)) self.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} if appointment.locator in self.locator_uuid_map: From 4a8bd921442fa7b1d02568c517b42f0293fc6b7b Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 18:34:40 +0100 Subject: [PATCH 12/62] Addapt Inspector unit tests to handle exceptions --- test/teos/unit/test_inspector.py | 218 ++++++++++++++++++++++--------- 1 file changed, 155 insertions(+), 63 deletions(-) diff --git a/test/teos/unit/test_inspector.py b/test/teos/unit/test_inspector.py index d0c35ec..6f02e4c 100644 --- a/test/teos/unit/test_inspector.py +++ b/test/teos/unit/test_inspector.py @@ -1,27 +1,20 @@ +import pytest from binascii import unhexlify from teos.errors import * from teos import LOG_PREFIX -from teos.inspector import Inspector from teos.block_processor import BlockProcessor +from teos.inspector import Inspector, InspectionFailed import common.cryptographer from common.logger import Logger from common.appointment import Appointment -from common.cryptographer import Cryptographer from common.constants import LOCATOR_LEN_BYTES, LOCATOR_LEN_HEX -from test.teos.unit.conftest import ( - get_random_value_hex, - generate_dummy_appointment_data, - generate_keypair, - bitcoind_connect_params, - get_config, -) +from test.teos.unit.conftest import get_random_value_hex, generate_keypair, bitcoind_connect_params, get_config common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX) -APPOINTMENT_OK = (0, None) NO_HEX_STRINGS = [ "R" * LOCATOR_LEN_HEX, get_random_value_hex(LOCATOR_LEN_BYTES - 1) + "PP", @@ -51,30 +44,60 @@ inspector = Inspector(block_processor, MIN_TO_SELF_DELAY) def test_check_locator(): # Right appointment type, size and format locator = get_random_value_hex(LOCATOR_LEN_BYTES) - assert Inspector.check_locator(locator) == APPOINTMENT_OK + assert inspector.check_locator(locator) is None # Wrong size (too big) locator = get_random_value_hex(LOCATOR_LEN_BYTES + 1) - assert Inspector.check_locator(locator)[0] == APPOINTMENT_WRONG_FIELD_SIZE + with pytest.raises(InspectionFailed): + try: + inspector.check_locator(locator) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_SIZE + raise e # Wrong size (too small) locator = get_random_value_hex(LOCATOR_LEN_BYTES - 1) - assert Inspector.check_locator(locator)[0] == APPOINTMENT_WRONG_FIELD_SIZE + with pytest.raises(InspectionFailed): + try: + inspector.check_locator(locator) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_SIZE + raise e # Empty locator = None - assert Inspector.check_locator(locator)[0] == APPOINTMENT_EMPTY_FIELD + with pytest.raises(InspectionFailed): + try: + inspector.check_locator(locator) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_EMPTY_FIELD + raise e # Wrong type (several types tested, it should do for anything that is not a string) locators = [[], -1, 3.2, 0, 4, (), object, {}, object()] for locator in locators: - assert Inspector.check_locator(locator)[0] == APPOINTMENT_WRONG_FIELD_TYPE + with pytest.raises(InspectionFailed): + try: + inspector.check_locator(locator) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + raise e # Wrong format (no hex) locators = NO_HEX_STRINGS for locator in locators: - assert Inspector.check_locator(locator)[0] == APPOINTMENT_WRONG_FIELD_FORMAT + with pytest.raises(InspectionFailed): + try: + inspector.check_locator(locator) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_FORMAT + raise e def test_check_start_time(): @@ -83,21 +106,39 @@ def test_check_start_time(): # Right format and right value (start time in the future) start_time = 101 - assert Inspector.check_start_time(start_time, current_time) == APPOINTMENT_OK + assert inspector.check_start_time(start_time, current_time) is None # Start time too small (either same block or block in the past) start_times = [100, 99, 98, -1] for start_time in start_times: - assert Inspector.check_start_time(start_time, current_time)[0] == APPOINTMENT_FIELD_TOO_SMALL + with pytest.raises(InspectionFailed): + try: + inspector.check_start_time(start_time, current_time) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_FIELD_TOO_SMALL + raise e # Empty field start_time = None - assert Inspector.check_start_time(start_time, current_time)[0] == APPOINTMENT_EMPTY_FIELD + with pytest.raises(InspectionFailed): + try: + inspector.check_start_time(start_time, current_time) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_EMPTY_FIELD + raise e # Wrong data type start_times = WRONG_TYPES for start_time in start_times: - assert Inspector.check_start_time(start_time, current_time)[0] == APPOINTMENT_WRONG_FIELD_TYPE + with pytest.raises(InspectionFailed): + try: + inspector.check_start_time(start_time, current_time) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + raise e def test_check_end_time(): @@ -107,54 +148,96 @@ def test_check_end_time(): # Right format and right value (start time before end and end in the future) end_time = 121 - assert Inspector.check_end_time(end_time, start_time, current_time) == APPOINTMENT_OK + assert inspector.check_end_time(end_time, start_time, current_time) is None # End time too small (start time after end time) end_times = [120, 119, 118, -1] for end_time in end_times: - assert Inspector.check_end_time(end_time, start_time, current_time)[0] == APPOINTMENT_FIELD_TOO_SMALL + with pytest.raises(InspectionFailed): + try: + inspector.check_end_time(end_time, start_time, current_time) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_FIELD_TOO_SMALL + raise e # End time too small (either same height as current block or in the past) current_time = 130 end_times = [130, 129, 128, -1] for end_time in end_times: - assert Inspector.check_end_time(end_time, start_time, current_time)[0] == APPOINTMENT_FIELD_TOO_SMALL + with pytest.raises(InspectionFailed): + try: + inspector.check_end_time(end_time, start_time, current_time) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_FIELD_TOO_SMALL + raise e # Empty field end_time = None - assert Inspector.check_end_time(end_time, start_time, current_time)[0] == APPOINTMENT_EMPTY_FIELD + with pytest.raises(InspectionFailed): + try: + inspector.check_end_time(end_time, start_time, current_time) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_EMPTY_FIELD + raise e # Wrong data type end_times = WRONG_TYPES for end_time in end_times: - assert Inspector.check_end_time(end_time, start_time, current_time)[0] == APPOINTMENT_WRONG_FIELD_TYPE + with pytest.raises(InspectionFailed): + try: + inspector.check_end_time(end_time, start_time, current_time) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + raise e def test_check_to_self_delay(): # Right value, right format to_self_delays = [MIN_TO_SELF_DELAY, MIN_TO_SELF_DELAY + 1, MIN_TO_SELF_DELAY + 1000] for to_self_delay in to_self_delays: - assert inspector.check_to_self_delay(to_self_delay) == APPOINTMENT_OK + assert inspector.check_to_self_delay(to_self_delay) is None # to_self_delay too small to_self_delays = [MIN_TO_SELF_DELAY - 1, MIN_TO_SELF_DELAY - 2, 0, -1, -1000] for to_self_delay in to_self_delays: - assert inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_FIELD_TOO_SMALL + with pytest.raises(InspectionFailed): + try: + inspector.check_to_self_delay(to_self_delay) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_FIELD_TOO_SMALL + raise e # Empty field to_self_delay = None - assert inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_EMPTY_FIELD + with pytest.raises(InspectionFailed): + try: + inspector.check_to_self_delay(to_self_delay) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_EMPTY_FIELD + raise e # Wrong data type to_self_delays = WRONG_TYPES for to_self_delay in to_self_delays: - assert inspector.check_to_self_delay(to_self_delay)[0] == APPOINTMENT_WRONG_FIELD_TYPE + with pytest.raises(InspectionFailed): + try: + inspector.check_to_self_delay(to_self_delay) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + raise e def test_check_blob(): # Right format and length encrypted_blob = get_random_value_hex(120) - assert Inspector.check_blob(encrypted_blob) == APPOINTMENT_OK + assert inspector.check_blob(encrypted_blob) is None # # Wrong content # # FIXME: There is not proper defined format for this yet. It should be restricted by size at least, and check it @@ -163,47 +246,37 @@ def test_check_blob(): # Wrong type encrypted_blobs = WRONG_TYPES_NO_STR for encrypted_blob in encrypted_blobs: - assert Inspector.check_blob(encrypted_blob)[0] == APPOINTMENT_WRONG_FIELD_TYPE + with pytest.raises(InspectionFailed): + try: + inspector.check_blob(encrypted_blob) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + raise e # Empty field encrypted_blob = None - assert Inspector.check_blob(encrypted_blob)[0] == APPOINTMENT_EMPTY_FIELD + with pytest.raises(InspectionFailed): + try: + inspector.check_blob(encrypted_blob) + + except InspectionFailed as e: + assert e.erno == APPOINTMENT_EMPTY_FIELD + raise e # Wrong format (no hex) encrypted_blobs = NO_HEX_STRINGS for encrypted_blob in encrypted_blobs: - assert Inspector.check_blob(encrypted_blob)[0] == APPOINTMENT_WRONG_FIELD_FORMAT + with pytest.raises(InspectionFailed): + try: + inspector.check_blob(encrypted_blob) - -def test_check_appointment_signature(): - # The inspector receives the public key as hex - client_sk, client_pk = generate_keypair() - client_pk_hex = client_pk.format().hex() - - dummy_appointment_data, _ = generate_dummy_appointment_data(real_height=False) - assert Inspector.check_appointment_signature( - dummy_appointment_data["appointment"], dummy_appointment_data["signature"], dummy_appointment_data["public_key"] - ) - - fake_sk, _ = generate_keypair() - - # Create a bad signature to make sure inspector rejects it - bad_signature = Cryptographer.sign( - Appointment.from_dict(dummy_appointment_data["appointment"]).serialize(), fake_sk - ) - assert ( - Inspector.check_appointment_signature(dummy_appointment_data["appointment"], bad_signature, client_pk_hex)[0] - == APPOINTMENT_INVALID_SIGNATURE - ) + except InspectionFailed as e: + assert e.erno == APPOINTMENT_WRONG_FIELD_FORMAT + raise e def test_inspect(run_bitcoind): - # At this point every single check function has been already tested, let's test inspect with an invalid and a valid - # appointments. - - client_sk, client_pk = generate_keypair() - client_pk_hex = client_pk.format().hex() - # Valid appointment locator = get_random_value_hex(LOCATOR_LEN_BYTES) start_time = block_processor.get_block_count() + 5 @@ -219,9 +292,7 @@ def test_inspect(run_bitcoind): "encrypted_blob": encrypted_blob, } - signature = Cryptographer.sign(Appointment.from_dict(appointment_data).serialize(), client_sk) - - appointment = inspector.inspect(appointment_data, signature, client_pk_hex) + appointment = inspector.inspect(appointment_data) assert ( type(appointment) == Appointment @@ -231,3 +302,24 @@ def test_inspect(run_bitcoind): and appointment.to_self_delay == to_self_delay and appointment.encrypted_blob.data == encrypted_blob ) + + +def test_inspect_wrong(run_bitcoind): + # Wrong types (taking out empty dict, since that's a different error) + wrong_types = WRONG_TYPES.pop(WRONG_TYPES.index({})) + for data in wrong_types: + with pytest.raises(InspectionFailed): + try: + inspector.inspect(data) + except InspectionFailed as e: + print(data) + assert e.erno == APPOINTMENT_WRONG_FIELD + raise e + + # None data + with pytest.raises(InspectionFailed): + try: + inspector.inspect(None) + except InspectionFailed as e: + assert e.erno == APPOINTMENT_EMPTY_FIELD + raise e From a6f9cbf44ea1adbd644a6d2dca4e6567bab684ea Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 18:52:42 +0100 Subject: [PATCH 13/62] Prevents adding the same uuid twice in the map (appointment update) --- teos/watcher.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/teos/watcher.py b/teos/watcher.py index 79cb2b6..feb2f71 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -115,7 +115,9 @@ class Watcher: self.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} if appointment.locator in self.locator_uuid_map: - self.locator_uuid_map[appointment.locator].append(uuid) + # If the uuid is already in the map it means this is an update. + if uuid not in self.locator_uuid_map[appointment.locator]: + self.locator_uuid_map[appointment.locator].append(uuid) else: self.locator_uuid_map[appointment.locator] = [uuid] From 5d46049f31188b82c83a59931d04210a71c9b1fd Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 18:54:06 +0100 Subject: [PATCH 14/62] Adapts Watcher unit tests to the new uuid apporach --- test/teos/unit/test_watcher.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/test/teos/unit/test_watcher.py b/test/teos/unit/test_watcher.py index de72298..b219094 100644 --- a/test/teos/unit/test_watcher.py +++ b/test/teos/unit/test_watcher.py @@ -120,7 +120,9 @@ def test_add_appointment(watcher): appointment, dispute_tx = generate_dummy_appointment( start_time_offset=START_TIME_OFFSET, end_time_offset=END_TIME_OFFSET ) - added_appointment, sig = watcher.add_appointment(appointment) + user_pk = get_random_value_hex(33) + + added_appointment, sig = watcher.add_appointment(appointment, user_pk) assert added_appointment is True assert Cryptographer.verify_rpk( @@ -128,13 +130,25 @@ def test_add_appointment(watcher): ) # Check that we can also add an already added appointment (same locator) - added_appointment, sig = watcher.add_appointment(appointment) + added_appointment, sig = watcher.add_appointment(appointment, user_pk) assert added_appointment is True assert Cryptographer.verify_rpk( watcher.signing_key.public_key, Cryptographer.recover_pk(appointment.serialize(), sig) ) + # If two appointments with the same locator from the same user are added, they are overwritten, but if they come + # from different users, they are kept. + assert len(watcher.locator_uuid_map[appointment.locator]) == 1 + + different_user_pk = get_random_value_hex(33) + added_appointment, sig = watcher.add_appointment(appointment, different_user_pk) + assert added_appointment is True + assert Cryptographer.verify_rpk( + watcher.signing_key.public_key, Cryptographer.recover_pk(appointment.serialize(), sig) + ) + assert len(watcher.locator_uuid_map[appointment.locator]) == 2 + def test_add_too_many_appointments(watcher): # Any appointment on top of those should fail @@ -144,7 +158,9 @@ def test_add_too_many_appointments(watcher): appointment, dispute_tx = generate_dummy_appointment( start_time_offset=START_TIME_OFFSET, end_time_offset=END_TIME_OFFSET ) - added_appointment, sig = watcher.add_appointment(appointment) + user_pk = get_random_value_hex(33) + + added_appointment, sig = watcher.add_appointment(appointment, user_pk) assert added_appointment is True assert Cryptographer.verify_rpk( @@ -154,7 +170,8 @@ def test_add_too_many_appointments(watcher): appointment, dispute_tx = generate_dummy_appointment( start_time_offset=START_TIME_OFFSET, end_time_offset=END_TIME_OFFSET ) - added_appointment, sig = watcher.add_appointment(appointment) + user_pk = get_random_value_hex(33) + added_appointment, sig = watcher.add_appointment(appointment, user_pk) assert added_appointment is False assert sig is None From 2fcdc7eacc3ef269bd06c33753cbcfff90f10715 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 26 Mar 2020 18:54:25 +0100 Subject: [PATCH 15/62] Removes public key from add_appointment requests --- test/teos/unit/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/teos/unit/conftest.py b/test/teos/unit/conftest.py index 367d28e..0b621f5 100644 --- a/test/teos/unit/conftest.py +++ b/test/teos/unit/conftest.py @@ -121,7 +121,6 @@ def generate_dummy_appointment_data(real_height=True, start_time_offset=5, end_t # dummy keys for this test client_sk, client_pk = generate_keypair() - client_pk_hex = client_pk.format().hex() locator = compute_locator(dispute_txid) blob = Blob(dummy_appointment_data.get("tx")) @@ -138,7 +137,7 @@ def generate_dummy_appointment_data(real_height=True, start_time_offset=5, end_t signature = Cryptographer.sign(Appointment.from_dict(appointment_data).serialize(), client_sk) - data = {"appointment": appointment_data, "signature": signature, "public_key": client_pk_hex} + data = {"appointment": appointment_data, "signature": signature} return data, dispute_tx.hex() From 9e798916d142aad19830ae56ba1f7526df6a5a87 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 12:07:26 +0100 Subject: [PATCH 16/62] Fixes API return HTTP return types and messages --- common/constants.py | 1 + teos/api.py | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/common/constants.py b/common/constants.py index bf119aa..904db90 100644 --- a/common/constants.py +++ b/common/constants.py @@ -5,6 +5,7 @@ LOCATOR_LEN_BYTES = LOCATOR_LEN_HEX // 2 # HTTP HTTP_OK = 200 HTTP_BAD_REQUEST = 400 +HTTP_NOT_FOUND = 404 HTTP_SERVICE_UNAVAILABLE = 503 # Temporary constants, may be changed diff --git a/teos/api.py b/teos/api.py index efe9f81..3839327 100644 --- a/teos/api.py +++ b/teos/api.py @@ -10,7 +10,13 @@ from teos.gatekeeper import NotEnoughSlots, IdentificationFailure from common.logger import Logger from common.cryptographer import hash_160 -from common.constants import HTTP_OK, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, ENCRYPTED_BLOB_MAX_SIZE_HEX +from common.constants import ( + HTTP_OK, + HTTP_BAD_REQUEST, + HTTP_SERVICE_UNAVAILABLE, + HTTP_NOT_FOUND, + ENCRYPTED_BLOB_MAX_SIZE_HEX, +) # ToDo: #5-add-async-to-api @@ -162,11 +168,11 @@ class API: error = "appointment rejected. Error {}: {}".format(e.erno, e.reason) response = {"error": error} - except (IdentificationFailure, NotEnoughSlots) as e: + except (IdentificationFailure, NotEnoughSlots): rcode = HTTP_BAD_REQUEST error = "appointment rejected. Error {}: {}".format( errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS, - "Invalid signature or the user does not have enough slots available", + "Invalid signature or user does not have enough slots available", ) response = {"error": error} @@ -219,24 +225,26 @@ class API: if uuid in triggered_appointments: response = self.watcher.db_manager.load_responder_tracker(uuid) if response: + rcode = HTTP_OK response["status"] = "dispute_responded" else: + rcode = HTTP_NOT_FOUND response = {"locator": locator, "status": "not_found"} # Otherwise it should be either in the watcher, or not in the system. else: response = self.watcher.db_manager.load_watcher_appointment(uuid) if response: + rcode = HTTP_OK response["status"] = "being_watched" else: + rcode = HTTP_NOT_FOUND response = {"locator": locator, "status": "not_found"} except (InspectionFailed, IdentificationFailure): + rcode = HTTP_NOT_FOUND response = {"locator": locator, "status": "not_found"} - finally: - rcode = HTTP_OK - else: rcode = HTTP_BAD_REQUEST error = "appointment rejected. Request is not json encoded" From f9a3315cec7bc5a3f0b3f5bf9f015a0118fac014 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 12:07:55 +0100 Subject: [PATCH 17/62] Updates API unit tests --- test/teos/unit/conftest.py | 19 +-- test/teos/unit/test_api.py | 284 ++++++++++++++++++++++++++----------- 2 files changed, 203 insertions(+), 100 deletions(-) diff --git a/test/teos/unit/conftest.py b/test/teos/unit/conftest.py index 0b621f5..52d95f7 100644 --- a/test/teos/unit/conftest.py +++ b/test/teos/unit/conftest.py @@ -100,7 +100,7 @@ def fork(block_hash): requests.post(fork_endpoint, json={"parent": block_hash}) -def generate_dummy_appointment_data(real_height=True, start_time_offset=5, end_time_offset=30): +def generate_dummy_appointment(real_height=True, start_time_offset=5, end_time_offset=30): if real_height: current_height = bitcoin_cli(bitcoind_connect_params).getblockcount() @@ -119,9 +119,6 @@ def generate_dummy_appointment_data(real_height=True, start_time_offset=5, end_t "to_self_delay": 20, } - # dummy keys for this test - client_sk, client_pk = generate_keypair() - locator = compute_locator(dispute_txid) blob = Blob(dummy_appointment_data.get("tx")) @@ -135,19 +132,7 @@ def generate_dummy_appointment_data(real_height=True, start_time_offset=5, end_t "encrypted_blob": encrypted_blob, } - signature = Cryptographer.sign(Appointment.from_dict(appointment_data).serialize(), client_sk) - - data = {"appointment": appointment_data, "signature": signature} - - return data, dispute_tx.hex() - - -def generate_dummy_appointment(real_height=True, start_time_offset=5, end_time_offset=30): - appointment_data, dispute_tx = generate_dummy_appointment_data( - real_height=real_height, start_time_offset=start_time_offset, end_time_offset=end_time_offset - ) - - return Appointment.from_dict(appointment_data["appointment"]), dispute_tx + return Appointment.from_dict(appointment_data), dispute_tx.hex() def generate_dummy_tracker(): diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index c1d4786..75ef96e 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -2,10 +2,12 @@ import json import pytest import requests from time import sleep +from binascii import hexlify from threading import Thread from teos.api import API from teos import HOST, PORT +import teos.errors as errors from teos.watcher import Watcher from teos.tools import bitcoin_cli from teos.inspector import Inspector @@ -17,15 +19,16 @@ from test.teos.unit.conftest import ( generate_block, generate_blocks, get_random_value_hex, - generate_dummy_appointment_data, + generate_dummy_appointment, generate_keypair, get_config, bitcoind_connect_params, bitcoind_feed_params, ) - -from common.constants import LOCATOR_LEN_BYTES +from common.blob import Blob +from common.cryptographer import Cryptographer +from common.constants import HTTP_OK, HTTP_NOT_FOUND, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, LOCATOR_LEN_BYTES TEOS_API = "http://{}:{}".format(HOST, PORT) @@ -41,8 +44,12 @@ locator_dispute_tx_map = {} config = get_config() +client_sk, client_pk = generate_keypair() +client_pk_hex = hexlify(client_pk.format(compressed=True)).decode("utf-8") + + @pytest.fixture(scope="module") -def run_api(db_manager, carrier, block_processor): +def api(db_manager, carrier, block_processor): sk, pk = generate_keypair() responder = Responder(db_manager, carrier, block_processor) @@ -56,82 +63,220 @@ def run_api(db_manager, carrier, block_processor): watcher.awake() chain_monitor.monitor_chain() - api_thread = Thread( - target=API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, Gatekeeper()).start - ) + api = API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, Gatekeeper()) + api_thread = Thread(target=api.start) api_thread.daemon = True api_thread.start() # It takes a little bit of time to start the API (otherwise the requests are sent too early and they fail) sleep(0.1) + return api + @pytest.fixture -def new_appt_data(): - appt_data, dispute_tx = generate_dummy_appointment_data() - locator_dispute_tx_map[appt_data["appointment"]["locator"]] = dispute_tx +def appointment(): + appointment, dispute_tx = generate_dummy_appointment() + locator_dispute_tx_map[appointment.locator] = dispute_tx - return appt_data + return appointment -def add_appointment(new_appt_data): - r = requests.post(url=add_appointment_endpoint, json=new_appt_data, timeout=5) +def add_appointment(appointment_data): + r = requests.post(url=add_appointment_endpoint, json=appointment_data, timeout=5) - if r.status_code == 200: - appointments.append(new_appt_data["appointment"]) + if r.status_code == HTTP_OK: + appointments.append(appointment_data["appointment"]) return r -def test_add_appointment(run_api, run_bitcoind, new_appt_data): +def test_add_appointment(api, run_bitcoind, appointment): + # Simulate the user registration + api.gatekeeper.registered_users[client_pk_hex] = 1 + # Properly formatted appointment - r = add_appointment(new_appt_data) - assert r.status_code == 200 + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK + + +def test_add_appointment_wrong(api, appointment): + # Simulate the user registration + api.gatekeeper.registered_users[client_pk_hex] = 1 # Incorrect appointment - new_appt_data["appointment"]["to_self_delay"] = 0 - r = add_appointment(new_appt_data) - assert r.status_code == 400 + appointment.to_self_delay = 0 + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_BAD_REQUEST + assert "Error {}:".format(errors.APPOINTMENT_FIELD_TOO_SMALL) in r.json().get("error") -def test_request_random_appointment(): - r = requests.get(url="{}?locator={}".format(get_appointment_endpoint, get_random_value_hex(LOCATOR_LEN_BYTES))) - assert r.status_code == 200 - - received_appointments = json.loads(r.content) - appointment_status = [appointment.pop("status") for appointment in received_appointments] - - assert all([status == "not_found" for status in appointment_status]) +def test_add_appointment_not_registered(api, appointment): + # Properly formatted appointment + tmp_sk, tmp_pk = generate_keypair() + appointment_signature = Cryptographer.sign(appointment.serialize(), tmp_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_BAD_REQUEST + assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json().get("error") -def test_add_appointment_multiple_times(new_appt_data, n=MULTIPLE_APPOINTMENTS): +def test_add_appointment_registered_no_free_slots(api, appointment): + # Empty the user slots + api.gatekeeper.registered_users[client_pk_hex] = 0 + + # Properly formatted appointment + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_BAD_REQUEST + assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json().get("error") + + +def test_add_appointment_registered_not_enough_free_slots(api, appointment): + # Empty the user slots + api.gatekeeper.registered_users[client_pk_hex] = 1 + + # Properly formatted appointment + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + + # Let's create a big blob + for _ in range(10): + appointment.encrypted_blob.data += appointment.encrypted_blob.data + + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_BAD_REQUEST + assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json().get("error") + + +def test_add_appointment_multiple_times_same_user(api, appointment, n=MULTIPLE_APPOINTMENTS): # Multiple appointments with the same locator should be valid + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + + # Simulate registering enough slots + api.gatekeeper.registered_users[client_pk_hex] = n # DISCUSS: #34-store-identical-appointments for _ in range(n): - r = add_appointment(new_appt_data) - assert r.status_code == 200 + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK + + # Since all updates came from the same user, only the last one is stored + assert len(api.watcher.locator_uuid_map[appointment.locator]) == 1 -def test_request_multiple_appointments_same_locator(new_appt_data, n=MULTIPLE_APPOINTMENTS): - for _ in range(n): - r = add_appointment(new_appt_data) - assert r.status_code == 200 +def test_add_appointment_multiple_times_different_users(api, appointment, n=MULTIPLE_APPOINTMENTS): + # Create user keys and appointment signatures + user_keys = [generate_keypair() for _ in range(n)] + signatures = [Cryptographer.sign(appointment.serialize(), key[0]) for key in user_keys] - test_request_appointment_watcher(new_appt_data) + # Add one slot per public key + for pair in user_keys: + api.gatekeeper.registered_users[hexlify(pair[1].format(compressed=True)).decode("utf-8")] = 1 + + # Send the appointments + for signature in signatures: + r = add_appointment({"appointment": appointment.to_dict(), "signature": signature}) + assert r.status_code == HTTP_OK + + # Check that all the appointments have been added and that there are no duplicates + assert len(set(api.watcher.locator_uuid_map[appointment.locator])) == n -def test_add_too_many_appointment(new_appt_data): - for _ in range(config.get("MAX_APPOINTMENTS") - len(appointments)): - r = add_appointment(new_appt_data) - assert r.status_code == 200 +def test_request_random_appointment_registered_user(user_sk=client_sk): + locator = get_random_value_hex(LOCATOR_LEN_BYTES) + message = "get appointment {}".format(locator) + signature = Cryptographer.sign(message.encode("utf-8"), client_sk) - r = add_appointment(new_appt_data) - assert r.status_code == 503 + data = {"locator": locator, "signature": signature} + r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) + + # We should get a 404 not found since we are using a made up locator + received_appointment = r.json() + assert r.status_code == HTTP_NOT_FOUND + assert received_appointment.get("status") == "not_found" + + +def test_request_appointment_not_registered_user(): + # Not registered users have no associated appointments, so this should fail + tmp_sk, tmp_pk = generate_keypair() + + # The tower is designed so a not found appointment and a request from a non-registered user return the same error to + # prevent proving. + test_request_random_appointment_registered_user(tmp_sk) + + +def test_request_appointment_in_watcher(api, appointment): + # Give slots to the user + api.gatekeeper.registered_users[client_pk_hex] = 1 + + # Add an appointment + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK + + message = "get appointment {}".format(appointment.locator) + signature = Cryptographer.sign(message.encode("utf-8"), client_sk) + data = {"locator": appointment.locator, "signature": signature} + + # Next we can request it + r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) + assert r.status_code == HTTP_OK + + appointment_data = json.loads(r.content) + # Check that the appointment is on the watcher + status = appointment_data.pop("status") + assert status == "being_watched" + + # Check the the sent appointment matches the received one + assert appointment.to_dict() == appointment_data + + +def test_request_appointment_in_responder(api, appointment): + # Give slots to the user + api.gatekeeper.registered_users[client_pk_hex] = 1 + + # Let's do something similar to what we did with the watcher but now we'll send the dispute tx to the network + dispute_tx = locator_dispute_tx_map.pop(appointment.locator) + bitcoin_cli(bitcoind_connect_params).sendrawtransaction(dispute_tx) + + # Add an appointment + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK + + # Generate a block to trigger the watcher + generate_block() + + # Request back the data + message = "get appointment {}".format(appointment.locator) + signature = Cryptographer.sign(message.encode("utf-8"), client_sk) + data = {"locator": appointment.locator, "signature": signature} + + # Next we can request it + r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) + assert r.status_code == HTTP_OK + + appointment_data = json.loads(r.content) + # Check that the appointment is on the watcher + status = appointment_data.pop("status") + assert status == "dispute_responded" + + # Check the the sent appointment matches the received one + assert appointment.locator == appointment_data.get("locator") + assert appointment.encrypted_blob.data == Cryptographer.encrypt( + Blob(appointment_data.get("penalty_rawtx")), appointment_data.get("dispute_txid") + ) + + # Delete appointment so it does not mess up with future tests + appointments.pop() + uuids = api.watcher.responder.tx_tracker_map.pop(appointment_data.get("penalty_txid")) + api.watcher.responder.db_manager.delete_responder_tracker(uuids[0]) + # api.watcher.responder.trackers.pop(uuids[0]) def test_get_all_appointments_watcher(): r = requests.get(url=get_all_appointment_endpoint) - assert r.status_code == 200 and r.reason == "OK" + assert r.status_code == HTTP_OK received_appointments = json.loads(r.content) @@ -165,45 +310,18 @@ def test_get_all_appointments_responder(): assert len(received_appointments["watcher_appointments"]) == 0 -def test_request_appointment_watcher(new_appt_data): - # First we need to add an appointment - r = add_appointment(new_appt_data) - assert r.status_code == 200 +def test_add_too_many_appointment(api): + # Give slots to the user + api.gatekeeper.registered_users[client_pk_hex] = 100 - # Next we can request it - r = requests.get(url="{}?locator={}".format(get_appointment_endpoint, new_appt_data["appointment"]["locator"])) - assert r.status_code == 200 + for i in range(config.get("MAX_APPOINTMENTS") - len(appointments)): + appointment, dispute_tx = generate_dummy_appointment() + locator_dispute_tx_map[appointment.locator] = dispute_tx - # Each locator may point to multiple appointments, check them all - received_appointments = json.loads(r.content) + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) - # Take the status out and leave the received appointments ready to compare - appointment_status = [appointment.pop("status") for appointment in received_appointments] - - # Check that the appointment is within the received appointments - assert new_appt_data["appointment"] in received_appointments - - # Check that all the appointments are being watched - assert all([status == "being_watched" for status in appointment_status]) - - -def test_request_appointment_responder(new_appt_data): - # Let's do something similar to what we did with the watcher but now we'll send the dispute tx to the network - dispute_tx = locator_dispute_tx_map[new_appt_data["appointment"]["locator"]] - bitcoin_cli(bitcoind_connect_params).sendrawtransaction(dispute_tx) - - r = add_appointment(new_appt_data) - assert r.status_code == 200 - - # Generate a block to trigger the watcher - generate_block() - - r = requests.get(url="{}?locator={}".format(get_appointment_endpoint, new_appt_data["appointment"]["locator"])) - assert r.status_code == 200 - - received_appointments = json.loads(r.content) - appointment_status = [appointment.pop("status") for appointment in received_appointments] - appointment_locators = [appointment["locator"] for appointment in received_appointments] - - assert new_appt_data["appointment"]["locator"] in appointment_locators and len(received_appointments) == 1 - assert all([status == "dispute_responded" for status in appointment_status]) and len(appointment_status) == 1 + if i != config.get("MAX_APPOINTMENTS") - len(appointments): + assert r.status_code == HTTP_OK + else: + assert r.status_code == HTTP_SERVICE_UNAVAILABLE From a19e4f2c2e432f6ab7a793447753ea15299ca299 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 13:05:59 +0100 Subject: [PATCH 18/62] Updates docs and comments in api --- teos/api.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/teos/api.py b/teos/api.py index 3839327..2e2997a 100644 --- a/teos/api.py +++ b/teos/api.py @@ -24,7 +24,7 @@ app = Flask(__name__) logger = Logger(actor="API", log_name_prefix=LOG_PREFIX) -# TODO: UNITTEST +# NOTCOVERED: not sure how to monkey path this one. May be related to #77 def get_remote_addr(): """ Gets the remote client ip address. The HTTP_X_REAL_IP field is tried first in case the server is behind a reverse @@ -53,7 +53,6 @@ class API: gatekeeper (:obj:`Watcher `): a `Gatekeeper` instance in charge to gatekeep the API. """ - # TODO: UNITTEST def __init__(self, inspector, watcher, gatekeeper): self.inspector = inspector self.watcher = watcher @@ -64,9 +63,11 @@ class API: """ Registers a user by creating a subscription. - The user is identified by public key. + Registration is pretty straightforward for now, since it does not require payments. + The amount of slots cannot be requested by the user yet either. This is linked to the previous point. + Users register by sending a public key to the proper endpoint. This is exploitable atm, but will be solved when + payments are introduced. - Currently subscriptions are free. Returns: :obj:`tuple`: A tuple containing the response (``json``) and response code (``int``). For accepted requests, @@ -110,7 +111,6 @@ class API: return jsonify(response), rcode - # FIXME: UNITTEST def add_appointment(self): """ Main endpoint of the Watchtower. @@ -184,8 +184,6 @@ class API: logger.info("Sending response and disconnecting", from_addr="{}".format(remote_addr), response=response) return jsonify(response), rcode - # FIXME: THE NEXT TWO API ENDPOINTS ARE FOR TESTING AND SHOULD BE REMOVED / PROPERLY MANAGED BEFORE PRODUCTION! - # ToDo: #17-add-api-keys def get_appointment(self): """ Gives information about a given appointment state in the Watchtower. @@ -279,7 +277,6 @@ class API: return response - # TODO: UNITTEST def start(self): """ This function starts the Flask server used to run the API. Adds all the routes to the functions listed above. @@ -295,7 +292,7 @@ class API: for url, params in routes.items(): app.add_url_rule(url, view_func=params[0], methods=params[1]) - # Setting Flask log to ERROR only so it does not mess with out logging. Also disabling flask initial messages + # Setting Flask log to ERROR only so it does not mess with our logging. Also disabling flask initial messages logging.getLogger("werkzeug").setLevel(logging.ERROR) os.environ["WERKZEUG_RUN_MAIN"] = "true" From 50c35a5ac17df2a9727806c85fca6846cc1ae78d Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 13:59:23 +0100 Subject: [PATCH 19/62] Makes sure the received data in posts have the proper format and content --- teos/api.py | 210 ++++++++++++++++++++++++++++------------------------ 1 file changed, 115 insertions(+), 95 deletions(-) diff --git a/teos/api.py b/teos/api.py index 2e2997a..87838ac 100644 --- a/teos/api.py +++ b/teos/api.py @@ -42,6 +42,31 @@ def get_remote_addr(): return remote_addr +# NOTCOVERED: not sure how to monkey path this one. May be related to #77 +def get_request_data_json(request): + """ + Gets the content of a json POST request and makes sure ir decodes to a Python dictionary. + + Args: + request (:obj:`Request`): the request sent by the user. + + Returns: + :obj:`dict`: the dictionary parsed from the json request. + + Raises: + :obj:`TypeError`: if the request is not json encoded or it does not decodes to a Python dictionary. + """ + + if request.is_json: + request_data = request.get_json() + if isinstance(request_data, dict): + return request_data + else: + raise TypeError("Invalid request content") + else: + raise TypeError("Request is not json encoded") + + class API: """ The :class:`API` is in charge of the interface between the user and the tower. It handles and server user requests. @@ -77,34 +102,32 @@ class API: """ remote_addr = get_remote_addr() - logger.info("Received register request", from_addr="{}".format(remote_addr)) - if request.is_json: - request_data = request.get_json() - client_pk = request_data.get("public_key") + # Check that data type and content are correct. Abort otherwise. + try: + request_data = get_request_data_json(request) - if client_pk: - try: - rcode = HTTP_OK - available_slots = self.gatekeeper.add_update_user(client_pk) - response = {"public_key": client_pk, "available_slots": available_slots} + except TypeError as e: + logger.info("Received invalid get_appointment request", from_addr="{}".format(remote_addr)) + return abort(HTTP_BAD_REQUEST, e) - except ValueError as e: - rcode = HTTP_BAD_REQUEST - error = "Error {}: {}".format(errors.REGISTRATION_MISSING_FIELD, str(e)) - response = {"error": error} + client_pk = request_data.get("public_key") - else: + if client_pk: + try: + rcode = HTTP_OK + available_slots = self.gatekeeper.add_update_user(client_pk) + response = {"public_key": client_pk, "available_slots": available_slots} + + except ValueError as e: rcode = HTTP_BAD_REQUEST - error = "Error {}: public_key not found in register message".format( - errors.REGISTRATION_WRONG_FIELD_FORMAT - ) + error = "Error {}: {}".format(errors.REGISTRATION_MISSING_FIELD, str(e)) response = {"error": error} else: rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Request is not json encoded" + error = "Error {}: public_key not found in register message".format(errors.REGISTRATION_WRONG_FIELD_FORMAT) response = {"error": error} logger.info("Sending response and disconnecting", from_addr="{}".format(remote_addr), response=response) @@ -127,58 +150,54 @@ class API: # Getting the real IP if the server is behind a reverse proxy remote_addr = get_remote_addr() - logger.info("Received add_appointment request", from_addr="{}".format(remote_addr)) - if request.is_json: - request_data = request.get_json() + # Check that data type and content are correct. Abort otherwise. + try: + request_data = get_request_data_json(request) - # We kind of have the chicken an the egg problem here. Data must be verified and the signature must be - # checked: - # - # - If we verify the data first, we may encounter that the signature is wrong and wasted some time. - # - If we check the signature first, we may need to verify some of the information or expose to build - # appointments with potentially wrong data, which may be exploitable. - # - # The first approach seems safer since it only implies a bunch of pretty quick checks. + except TypeError as e: + return abort(HTTP_BAD_REQUEST, e) - try: - appointment = self.inspector.inspect(request_data.get("appointment")) - user_pk = self.gatekeeper.identify_user(appointment.serialize(), request_data.get("signature")) + # We kind of have the chicken an the egg problem here. Data must be verified and the signature must be checked: + # - If we verify the data first, we may encounter that the signature is wrong and wasted some time. + # - If we check the signature first, we may need to verify some of the information or expose to build + # appointments with potentially wrong data, which may be exploitable. + # + # The first approach seems safer since it only implies a bunch of pretty quick checks. - # An appointment will fill 1 slot per ENCRYPTED_BLOB_MAX_SIZE_HEX block. - # Temporarily taking out slots to avoid abusing this via race conditions. - # DISCUSS: It may be worth using signals here to avoid race conditions anyway. - required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) - self.gatekeeper.fill_slots(user_pk, required_slots) - appointment_added, signature = self.watcher.add_appointment(appointment, user_pk) + try: + appointment = self.inspector.inspect(request_data.get("appointment")) + user_pk = self.gatekeeper.identify_user(appointment.serialize(), request_data.get("signature")) - if appointment_added: - rcode = HTTP_OK - response = {"locator": appointment.locator, "signature": signature} + # An appointment will fill 1 slot per ENCRYPTED_BLOB_MAX_SIZE_HEX block. + # Temporarily taking out slots to avoid abusing this via race conditions. + # DISCUSS: It may be worth using signals here to avoid race conditions anyway. + required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) + self.gatekeeper.fill_slots(user_pk, required_slots) + appointment_added, signature = self.watcher.add_appointment(appointment, user_pk) - else: - # Adding back the slots since they were not used - self.gatekeeper.free_slots(user_pk, required_slots) - rcode = HTTP_SERVICE_UNAVAILABLE - response = {"error": "appointment rejected"} + if appointment_added: + rcode = HTTP_OK + response = {"locator": appointment.locator, "signature": signature} - except InspectionFailed as e: - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Error {}: {}".format(e.erno, e.reason) - response = {"error": error} + else: + # Adding back the slots since they were not used + self.gatekeeper.free_slots(user_pk, required_slots) + rcode = HTTP_SERVICE_UNAVAILABLE + response = {"error": "appointment rejected"} - except (IdentificationFailure, NotEnoughSlots): - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Error {}: {}".format( - errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS, - "Invalid signature or user does not have enough slots available", - ) - response = {"error": error} - - else: + except InspectionFailed as e: rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Request is not json encoded" + error = "appointment rejected. Error {}: {}".format(e.erno, e.reason) + response = {"error": error} + + except (IdentificationFailure, NotEnoughSlots): + rcode = HTTP_BAD_REQUEST + error = "appointment rejected. Error {}: {}".format( + errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS, + "Invalid signature or user does not have enough slots available", + ) response = {"error": error} logger.info("Sending response and disconnecting", from_addr="{}".format(remote_addr), response=response) @@ -204,49 +223,50 @@ class API: # Getting the real IP if the server is behind a reverse proxy remote_addr = get_remote_addr() - if request.is_json: - request_data = request.get_json() - locator = request_data.get("locator") + # Check that data type and content are correct. Abort otherwise. + try: + request_data = get_request_data_json(request) - try: - self.inspector.check_locator(locator) - logger.info("Received get_appointment request", from_addr="{}".format(remote_addr), locator=locator) + except TypeError as e: + logger.info("Received invalid get_appointment request", from_addr="{}".format(remote_addr)) + return abort(HTTP_BAD_REQUEST, e) - message = "get appointment {}".format(locator).encode() - signature = request_data.get("signature") - user_pk = self.gatekeeper.identify_user(message, signature) + locator = request_data.get("locator") - triggered_appointments = self.watcher.db_manager.load_all_triggered_flags() - uuid = hash_160("{}{}".format(locator, user_pk)) + try: + self.inspector.check_locator(locator) + logger.info("Received get_appointment request", from_addr="{}".format(remote_addr), locator=locator) - # If the appointment has been triggered, it should be in the locator (default else just in case). - if uuid in triggered_appointments: - response = self.watcher.db_manager.load_responder_tracker(uuid) - if response: - rcode = HTTP_OK - response["status"] = "dispute_responded" - else: - rcode = HTTP_NOT_FOUND - response = {"locator": locator, "status": "not_found"} + message = "get appointment {}".format(locator).encode() + signature = request_data.get("signature") + user_pk = self.gatekeeper.identify_user(message, signature) - # Otherwise it should be either in the watcher, or not in the system. + triggered_appointments = self.watcher.db_manager.load_all_triggered_flags() + uuid = hash_160("{}{}".format(locator, user_pk)) + + # If the appointment has been triggered, it should be in the locator (default else just in case). + if uuid in triggered_appointments: + response = self.watcher.db_manager.load_responder_tracker(uuid) + if response: + rcode = HTTP_OK + response["status"] = "dispute_responded" else: - response = self.watcher.db_manager.load_watcher_appointment(uuid) - if response: - rcode = HTTP_OK - response["status"] = "being_watched" - else: - rcode = HTTP_NOT_FOUND - response = {"locator": locator, "status": "not_found"} + rcode = HTTP_NOT_FOUND + response = {"locator": locator, "status": "not_found"} - except (InspectionFailed, IdentificationFailure): - rcode = HTTP_NOT_FOUND - response = {"locator": locator, "status": "not_found"} + # Otherwise it should be either in the watcher, or not in the system. + else: + response = self.watcher.db_manager.load_watcher_appointment(uuid) + if response: + rcode = HTTP_OK + response["status"] = "being_watched" + else: + rcode = HTTP_NOT_FOUND + response = {"locator": locator, "status": "not_found"} - else: - rcode = HTTP_BAD_REQUEST - error = "appointment rejected. Request is not json encoded" - response = {"error": error} + except (InspectionFailed, IdentificationFailure): + rcode = HTTP_NOT_FOUND + response = {"locator": locator, "status": "not_found"} return jsonify(response), rcode From 4c21326974fec4ba6cdb30725481a612ee5a345b Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 14:45:20 +0100 Subject: [PATCH 20/62] Moves slots added on register from gatekeeper to config --- teos/gatekeeper.py | 11 +++++------ teos/teosd.py | 3 ++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index 6810014..4807be3 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -2,8 +2,6 @@ import re from common.cryptographer import Cryptographer -SUBSCRIPTION_SLOTS = 1 - # TODO: UNITTEST class NotEnoughSlots(ValueError): @@ -32,7 +30,8 @@ class Gatekeeper: registered_users (:obj:`dict`): a map of user_pk:appointment_slots. """ - def __init__(self): + def __init__(self, default_slots): + self.default_slots = default_slots self.registered_users = {} @staticmethod @@ -57,16 +56,16 @@ class Gatekeeper: user_pk(:obj:`str`): the public key that identifies the user (33-bytes hex str). Returns: - :obj:`int`: the number of avaiable slots in the user subscription. + :obj:`int`: the number of available slots in the user subscription. """ if not self.check_user_pk(user_pk): raise ValueError("provided public key does not match expected format (33-byte hex string)") if user_pk not in self.registered_users: - self.registered_users[user_pk] = SUBSCRIPTION_SLOTS + self.registered_users[user_pk] = self.default_slots else: - self.registered_users[user_pk] += SUBSCRIPTION_SLOTS + self.registered_users[user_pk] += self.default_slots return self.registered_users[user_pk] diff --git a/teos/teosd.py b/teos/teosd.py index 2f46341..c33e42b 100644 --- a/teos/teosd.py +++ b/teos/teosd.py @@ -151,7 +151,8 @@ def main(command_line_conf): # Fire the API and the ChainMonitor # FIXME: 92-block-data-during-bootstrap-db chain_monitor.monitor_chain() - API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, Gatekeeper()).start() + gatekeeper = Gatekeeper(config.get("DEFAULT_SLOTS")) + API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, gatekeeper).start() except Exception as e: logger.error("An error occurred: {}. Shutting down".format(e)) exit(1) From 3946599f910a73dc85958037fe11b8b63ee2a471 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 14:46:31 +0100 Subject: [PATCH 21/62] Sets a more realistic valut for max_appointments and add default_slots in config --- teos/__init__.py | 3 ++- teos/template.conf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/teos/__init__.py b/teos/__init__.py index 707c7ec..780e3cc 100644 --- a/teos/__init__.py +++ b/teos/__init__.py @@ -17,7 +17,8 @@ DEFAULT_CONF = { "FEED_PROTOCOL": {"value": "tcp", "type": str}, "FEED_CONNECT": {"value": "127.0.0.1", "type": str}, "FEED_PORT": {"value": 28332, "type": int}, - "MAX_APPOINTMENTS": {"value": 100, "type": int}, + "MAX_APPOINTMENTS": {"value": 1000000, "type": int}, + "DEFAULT_SLOTS": {"value": 100, "type": int}, "EXPIRY_DELTA": {"value": 6, "type": int}, "MIN_TO_SELF_DELAY": {"value": 20, "type": int}, "LOG_FILE": {"value": "teos.log", "type": str, "path": True}, diff --git a/teos/template.conf b/teos/template.conf index 88e4df5..46c61e8 100644 --- a/teos/template.conf +++ b/teos/template.conf @@ -11,7 +11,8 @@ feed_connect = 127.0.0.1 feed_port = 28332 [teos] -max_appointments = 100 +subscription_slots = 100 +max_appointments = 1000000 expiry_delta = 6 min_to_self_delay = 20 From 772f634d1f7824315b58ed5aad5f42bfc2da1f8f Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 14:47:59 +0100 Subject: [PATCH 22/62] Sets MAX_APPOINTMENTS for Watcher unit tests to a small testable value (100) --- test/teos/unit/test_watcher.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/teos/unit/test_watcher.py b/test/teos/unit/test_watcher.py index b219094..7645c80 100644 --- a/test/teos/unit/test_watcher.py +++ b/test/teos/unit/test_watcher.py @@ -40,6 +40,9 @@ config = get_config() signing_key, public_key = generate_keypair() +# Reduce the maximum number of appointments to something we can test faster +MAX_APPOINTMENTS = 100 + @pytest.fixture(scope="session") def temp_db_manager(): @@ -59,12 +62,7 @@ def watcher(db_manager): responder = Responder(db_manager, carrier, block_processor) watcher = Watcher( - db_manager, - block_processor, - responder, - signing_key.to_der(), - config.get("MAX_APPOINTMENTS"), - config.get("EXPIRY_DELTA"), + db_manager, block_processor, responder, signing_key.to_der(), MAX_APPOINTMENTS, config.get("EXPIRY_DELTA") ) chain_monitor = ChainMonitor( @@ -154,7 +152,7 @@ def test_add_too_many_appointments(watcher): # Any appointment on top of those should fail watcher.appointments = dict() - for _ in range(config.get("MAX_APPOINTMENTS")): + for _ in range(MAX_APPOINTMENTS): appointment, dispute_tx = generate_dummy_appointment( start_time_offset=START_TIME_OFFSET, end_time_offset=END_TIME_OFFSET ) From 7e6a4849e63a3836c7824c5ec8918afa58b918ef Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 14:50:03 +0100 Subject: [PATCH 23/62] Adds register unit tests, missing add and get appointment tests and sets MAX_APPOINTMENTS to 100 for testing --- teos/api.py | 1 - test/teos/unit/test_api.py | 96 ++++++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/teos/api.py b/teos/api.py index 87838ac..5ad1214 100644 --- a/teos/api.py +++ b/teos/api.py @@ -83,7 +83,6 @@ class API: self.watcher = watcher self.gatekeeper = gatekeeper - # TODO: UNITTEST def register(self): """ Registers a user by creating a subscription. diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 75ef96e..9af8c81 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -32,10 +32,14 @@ from common.constants import HTTP_OK, HTTP_NOT_FOUND, HTTP_BAD_REQUEST, HTTP_SER TEOS_API = "http://{}:{}".format(HOST, PORT) +register_endpoint = "{}/register".format(TEOS_API) add_appointment_endpoint = "{}/add_appointment".format(TEOS_API) get_appointment_endpoint = "{}/get_appointment".format(TEOS_API) get_all_appointment_endpoint = "{}/get_all_appointments".format(TEOS_API) +# Reduce the maximum number of appointments to something we can test faster +MAX_APPOINTMENTS = 100 + MULTIPLE_APPOINTMENTS = 10 appointments = [] @@ -50,12 +54,11 @@ client_pk_hex = hexlify(client_pk.format(compressed=True)).decode("utf-8") @pytest.fixture(scope="module") def api(db_manager, carrier, block_processor): + sk, pk = generate_keypair() responder = Responder(db_manager, carrier, block_processor) - watcher = Watcher( - db_manager, block_processor, responder, sk.to_der(), config.get("MAX_APPOINTMENTS"), config.get("EXPIRY_DELTA") - ) + watcher = Watcher(db_manager, block_processor, responder, sk.to_der(), MAX_APPOINTMENTS, config.get("EXPIRY_DELTA")) chain_monitor = ChainMonitor( watcher.block_queue, watcher.responder.block_queue, block_processor, bitcoind_feed_params @@ -63,7 +66,8 @@ def api(db_manager, carrier, block_processor): watcher.awake() chain_monitor.monitor_chain() - api = API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, Gatekeeper()) + gatekeeper = Gatekeeper(config.get("DEFAULT_SLOTS")) + api = API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, gatekeeper) api_thread = Thread(target=api.start) api_thread.daemon = True api_thread.start() @@ -91,7 +95,51 @@ def add_appointment(appointment_data): return r -def test_add_appointment(api, run_bitcoind, appointment): +def test_register(api, run_bitcoind): + data = {"public_key": client_pk_hex} + r = requests.post(url=register_endpoint, json=data, timeout=5) + assert r.status_code == HTTP_OK + assert r.json().get("public_key") == client_pk_hex + assert r.json().get("available_slots") == config.get("DEFAULT_SLOTS") + + +def test_register_top_up(run_bitcoind): + # Calling register more than once will give us DEFAULT_SLOTS * number_of_calls slots + temp_sk, tmp_pk = generate_keypair() + tmp_pk_hex = hexlify(tmp_pk.format(compressed=True)).decode("utf-8") + + data = {"public_key": tmp_pk_hex} + + for i in range(10): + r = requests.post(url=register_endpoint, json=data, timeout=5) + assert r.status_code == HTTP_OK + assert r.json().get("public_key") == tmp_pk_hex + assert r.json().get("available_slots") == config.get("DEFAULT_SLOTS") * (i + 1) + + +def test_register_no_client_pk(run_bitcoind): + data = {"public_key": client_pk_hex + client_pk_hex} + r = requests.post(url=register_endpoint, json=data, timeout=5) + assert r.status_code == HTTP_BAD_REQUEST + + +def test_register_wrong_client_pk(run_bitcoind): + data = {} + r = requests.post(url=register_endpoint, json=data, timeout=5) + assert r.status_code == HTTP_BAD_REQUEST + + +def test_register_no_json(api, appointment): + r = requests.post(url=register_endpoint, data="random_message", timeout=5) + assert r.status_code == HTTP_BAD_REQUEST + + +def test_register_json_no_inner_dict(api, appointment): + r = requests.post(url=register_endpoint, json="random_message", timeout=5) + assert r.status_code == HTTP_BAD_REQUEST + + +def test_add_appointment(api, appointment): # Simulate the user registration api.gatekeeper.registered_users[client_pk_hex] = 1 @@ -101,6 +149,24 @@ def test_add_appointment(api, run_bitcoind, appointment): assert r.status_code == HTTP_OK +def test_add_appointment_no_json(api, appointment): + # Simulate the user registration + api.gatekeeper.registered_users[client_pk_hex] = 1 + + # Properly formatted appointment + r = requests.post(url=add_appointment_endpoint, data="random_message", timeout=5) + assert r.status_code == HTTP_BAD_REQUEST + + +def test_add_appointment_json_no_inner_dict(api, appointment): + # Simulate the user registration + api.gatekeeper.registered_users[client_pk_hex] = 1 + + # Properly formatted appointment + r = requests.post(url=add_appointment_endpoint, json="random_message", timeout=5) + assert r.status_code == HTTP_BAD_REQUEST + + def test_add_appointment_wrong(api, appointment): # Simulate the user registration api.gatekeeper.registered_users[client_pk_hex] = 1 @@ -182,10 +248,20 @@ def test_add_appointment_multiple_times_different_users(api, appointment, n=MULT assert len(set(api.watcher.locator_uuid_map[appointment.locator])) == n +def test_get_appointment_no_json(api, appointment): + r = requests.post(url=add_appointment_endpoint, data="random_message", timeout=5) + assert r.status_code == HTTP_BAD_REQUEST + + +def test_get_appointment_json_no_inner_dict(api, appointment): + r = requests.post(url=add_appointment_endpoint, json="random_message", timeout=5) + assert r.status_code == HTTP_BAD_REQUEST + + def test_request_random_appointment_registered_user(user_sk=client_sk): locator = get_random_value_hex(LOCATOR_LEN_BYTES) message = "get appointment {}".format(locator) - signature = Cryptographer.sign(message.encode("utf-8"), client_sk) + signature = Cryptographer.sign(message.encode("utf-8"), user_sk) data = {"locator": locator, "signature": signature} r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) @@ -312,16 +388,18 @@ def test_get_all_appointments_responder(): def test_add_too_many_appointment(api): # Give slots to the user - api.gatekeeper.registered_users[client_pk_hex] = 100 + api.gatekeeper.registered_users[client_pk_hex] = 200 - for i in range(config.get("MAX_APPOINTMENTS") - len(appointments)): + free_appointment_slots = MAX_APPOINTMENTS - len(api.watcher.appointments) + + for i in range(free_appointment_slots + 1): appointment, dispute_tx = generate_dummy_appointment() locator_dispute_tx_map[appointment.locator] = dispute_tx appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) - if i != config.get("MAX_APPOINTMENTS") - len(appointments): + if i < free_appointment_slots: assert r.status_code == HTTP_OK else: assert r.status_code == HTTP_SERVICE_UNAVAILABLE From ae68cd33da7f058340af45168db0e093c84582ca Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 16:41:19 +0100 Subject: [PATCH 24/62] Captures TypeErrors in Cryptographer.get_compressed_pk and adds unit tests --- common/cryptographer.py | 11 ++++++++--- test/common/unit/test_cryptographer.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/common/cryptographer.py b/common/cryptographer.py index f5427e7..5414fda 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -315,7 +315,7 @@ class Cryptographer: if "failed to recover ECDSA public key" in str(e): logger.error("Cannot recover public key from signature".format(type(rsig_recid))) else: - logger.error("Unknown exception", error=e) + logger.error("Unknown exception", error=str(e)) return None @@ -334,7 +334,6 @@ class Cryptographer: return pk.point() == rpk.point() - # TODO: UNITTEST @staticmethod def get_compressed_pk(pk): """ @@ -351,4 +350,10 @@ class Cryptographer: logger.error("The received data is not a PublicKey object") return None - return hexlify(pk.format(compressed=True)).decode("utf-8") + try: + compressed_pk = pk.format(compressed=True) + return hexlify(compressed_pk).decode("utf-8") + + except TypeError as e: + logger.error("PublicKey has invalid initializer", error=str(e)) + return None diff --git a/test/common/unit/test_cryptographer.py b/test/common/unit/test_cryptographer.py index d5983f5..ddbb707 100644 --- a/test/common/unit/test_cryptographer.py +++ b/test/common/unit/test_cryptographer.py @@ -255,3 +255,27 @@ def test_verify_pk_wrong(): rpk = Cryptographer.recover_pk(message, zbase32_sig) assert not Cryptographer.verify_rpk(sk2.public_key, rpk) + + +def test_get_compressed_pk(): + sk, pk = generate_keypair() + compressed_pk = Cryptographer.get_compressed_pk(pk) + + assert isinstance(compressed_pk, str) and len(compressed_pk) == 66 + assert compressed_pk[:2] in ["02", "03"] + + +def test_get_compressed_pk_wrong_key(): + # pk should be properly initialized. Initializing from int will case it to not be recoverable + pk = PublicKey(0) + compressed_pk = Cryptographer.get_compressed_pk(pk) + + assert compressed_pk is None + + +def test_get_compressed_pk_wrong_type(): + # Passing a value that is not a PublicKey will make it to fail too + pk = get_random_value_hex(33) + compressed_pk = Cryptographer.get_compressed_pk(pk) + + assert compressed_pk is None From 6547e75a19f2caae4610567d4d3335e617915e7e Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 17:26:35 +0100 Subject: [PATCH 25/62] Adds user exist restriction to fill/free slots and update comments/docs --- teos/gatekeeper.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index 4807be3..0d5131b 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -3,7 +3,6 @@ import re from common.cryptographer import Cryptographer -# TODO: UNITTEST class NotEnoughSlots(ValueError): """Raise this when trying to subtract more slots than a user has available.""" @@ -103,12 +102,16 @@ class Gatekeeper: Args: user_pk(:obj:`str`): the public key that identifies the user (33-bytes hex str). n: the number of slots to fill. + n (:obj:`int`): the number of slots to fill. Raises: :obj:``: if the user subscription does not have enough slots. """ - if n <= self.registered_users.get(user_pk): + # We are not making sure the value passed is a integer, but the value is computed by the API and rounded before + # passing it to the gatekeeper. + # DISCUSS: we may want to return a different exception if teh user does not exist + if user_pk in self.registered_users and n <= self.registered_users.get(user_pk): self.registered_users[user_pk] -= n else: raise NotEnoughSlots(user_pk, n) @@ -122,4 +125,8 @@ class Gatekeeper: n: the number of slots to free. """ - self.registered_users[user_pk] += n + # We are not making sure the value passed is a integer, but the value is computed by the API and rounded before + # passing it to the gatekeeper. + # DISCUSS: if the user does not exist we may want to log or return an exception. + if user_pk in self.registered_users: + self.registered_users[user_pk] += n From 02fd2e51494ad77dfeb1c6725469d76e81003c28 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 17:27:30 +0100 Subject: [PATCH 26/62] Adds gatekeeper unit tests --- test/teos/unit/test_gatekeeper.py | 141 ++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 test/teos/unit/test_gatekeeper.py diff --git a/test/teos/unit/test_gatekeeper.py b/test/teos/unit/test_gatekeeper.py new file mode 100644 index 0000000..eccc8c8 --- /dev/null +++ b/test/teos/unit/test_gatekeeper.py @@ -0,0 +1,141 @@ +import pytest + +from teos.gatekeeper import Gatekeeper, IdentificationFailure, NotEnoughSlots + +from common.cryptographer import Cryptographer + +from test.teos.unit.conftest import get_random_value_hex, generate_keypair + +DEFAULT_SLOTS = 42 +gatekeeper = Gatekeeper(DEFAULT_SLOTS) + + +def test_init(): + assert isinstance(gatekeeper.default_slots, int) and gatekeeper.default_slots == DEFAULT_SLOTS + assert isinstance(gatekeeper.registered_users, dict) and len(gatekeeper.registered_users) == 0 + + +def test_check_user_pk(): + # check_user_pk must only accept values that is not a 33-byte hex string + for _ in range(100): + assert gatekeeper.check_user_pk(get_random_value_hex(33)) + + +def test_check_wrong_user_pk(): + wrong_values = [None, 3, 15.23, "", {}, (), object, str, get_random_value_hex(32), get_random_value_hex(34)] + + # check_user_pk must only accept values that is not a 33-byte hex string + for value in wrong_values: + assert not gatekeeper.check_user_pk(value) + + +def test_add_update_user(): + # add_update_user adds DEFAULT_SLOTS to a given user as long as the identifier is a 33-byte hex str + user_pk = get_random_value_hex(33) + + for _ in range(10): + current_slots = gatekeeper.registered_users.get(user_pk) + current_slots = current_slots if current_slots is not None else 0 + + gatekeeper.add_update_user(user_pk) + + assert gatekeeper.registered_users.get(user_pk) == current_slots + DEFAULT_SLOTS + + # The same can be checked for multiple users + for _ in range(10): + # The user identifier is changed every call + user_pk = get_random_value_hex(33) + + gatekeeper.add_update_user(user_pk) + assert gatekeeper.registered_users.get(user_pk) == DEFAULT_SLOTS + + +def test_add_update_user_wrong_pk(): + # Passing a wrong pk defaults to the errors in check_user_pk. We can try with one. + wrong_pk = get_random_value_hex(32) + + with pytest.raises(ValueError): + gatekeeper.add_update_user(wrong_pk) + + +def test_identify_user(): + # Identify user should return a user_pk for registered users. It raises + # IdentificationFailure for invalid parameters or non-registered users. + + # Let's first register a user + sk, pk = generate_keypair() + compressed_pk = Cryptographer.get_compressed_pk(pk) + gatekeeper.add_update_user(compressed_pk) + + message = "Hey, it's me" + signature = Cryptographer.sign(message.encode(), sk) + + assert gatekeeper.identify_user(message.encode(), signature) == compressed_pk + + +def test_identify_user_non_registered(): + # Non-registered user won't be identified + sk, pk = generate_keypair() + + message = "Hey, it's me" + signature = Cryptographer.sign(message.encode(), sk) + + with pytest.raises(IdentificationFailure): + gatekeeper.identify_user(message.encode(), signature) + + +def test_identify_user_invalid_signature(): + # If the signature does not match the message given a public key, the user won't be identified + message = "Hey, it's me" + signature = get_random_value_hex(72) + + with pytest.raises(IdentificationFailure): + gatekeeper.identify_user(message.encode(), signature) + + +def test_identify_user_wrong(): + # Wrong parameters shouldn't verify either + sk, pk = generate_keypair() + + message = "Hey, it's me" + signature = Cryptographer.sign(message.encode(), sk) + + # Non-byte message and str sig + with pytest.raises(IdentificationFailure): + gatekeeper.identify_user(message, signature) + + # byte message and non-str sig + with pytest.raises(IdentificationFailure): + gatekeeper.identify_user(message.encode(), signature.encode()) + + # non-byte message and non-str sig + with pytest.raises(IdentificationFailure): + gatekeeper.identify_user(message, signature.encode()) + + +def test_fill_slots(): + # Free slots will decrease the slot count of a user as long as he has enough slots, otherwise raise NotEnoughSlots + user_pk = get_random_value_hex(33) + gatekeeper.add_update_user(user_pk) + + gatekeeper.fill_slots(user_pk, DEFAULT_SLOTS - 1) + assert gatekeeper.registered_users.get(user_pk) == 1 + + with pytest.raises(NotEnoughSlots): + gatekeeper.fill_slots(user_pk, 2) + + # NotEnoughSlots is also raised if the user does not exist + with pytest.raises(NotEnoughSlots): + gatekeeper.fill_slots(get_random_value_hex(33), 2) + + +def test_free_slots(): + # Free slots simply adds slots to the user as long as it exists. + user_pk = get_random_value_hex(33) + gatekeeper.add_update_user(user_pk) + gatekeeper.free_slots(user_pk, 33) + + assert gatekeeper.registered_users.get(user_pk) == DEFAULT_SLOTS + 33 + + # Just making sure it does not crash for non-registered user + assert gatekeeper.free_slots(get_random_value_hex(33), 10) is None From e924b57efcf2b601293ce0af7cde19d215cba91d Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 27 Mar 2020 18:16:25 +0100 Subject: [PATCH 27/62] Appointment updates only do not decrease slots if not necessary - For a given appointment, checks if it is an update and computes the difference of sizes if so. - Additional slots are only filled if the new version is bigger. Slots are freed if the update is smaller. - Adds get_appoiment_summary to get information in memory information of an appointment (so the API can check if a request is an update) - The API computes the uuid and requests it to the tower. - Size field has been added to all in memory appointments --- teos/api.py | 32 +++++++++++++++++++++++++++----- teos/watcher.py | 21 ++++++++++++++++++++- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/teos/api.py b/teos/api.py index 5ad1214..6bffe7f 100644 --- a/teos/api.py +++ b/teos/api.py @@ -169,19 +169,41 @@ class API: appointment = self.inspector.inspect(request_data.get("appointment")) user_pk = self.gatekeeper.identify_user(appointment.serialize(), request_data.get("signature")) - # An appointment will fill 1 slot per ENCRYPTED_BLOB_MAX_SIZE_HEX block. - # Temporarily taking out slots to avoid abusing this via race conditions. + # Check if the appointment is an update. Updates will return a summary. + appointment_uuid = hash_160("{}{}".format(appointment.locator, user_pk)) + appointment_summary = self.watcher.get_appointment_summary(appointment_uuid) + + # For updates we only reserve the slot difference provided the new one is bigger. + if appointment_summary: + size_diff = len(appointment.encrypted_blob.data) - appointment_summary.get("size") + slot_diff = ceil(size_diff / ENCRYPTED_BLOB_MAX_SIZE_HEX) + required_slots = slot_diff if slot_diff > 0 else 0 + + # For regular appointments 1 slot is reserved per ENCRYPTED_BLOB_MAX_SIZE_HEX block. + else: + slot_diff = 0 + required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) + + # Slots are reserved before adding the appointments to prevent race conditions. # DISCUSS: It may be worth using signals here to avoid race conditions anyway. - required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) self.gatekeeper.fill_slots(user_pk, required_slots) + appointment_added, signature = self.watcher.add_appointment(appointment, user_pk) if appointment_added: rcode = HTTP_OK - response = {"locator": appointment.locator, "signature": signature} + response = { + "locator": appointment.locator, + "signature": signature, + "available_slots": self.gatekeeper.registered_users[user_pk], + } + + # If the appointment is added and the update is smaller than the original, the difference is given back. + if slot_diff < 0: + self.gatekeeper.free_slots(slot_diff) else: - # Adding back the slots since they were not used + # If the appointment is not added the reserved slots are given back self.gatekeeper.free_slots(user_pk, required_slots) rcode = HTTP_SERVICE_UNAVAILABLE response = {"error": "appointment rejected"} diff --git a/teos/watcher.py b/teos/watcher.py index feb2f71..757ffc0 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -76,6 +76,21 @@ class Watcher: return watcher_thread + def get_appointment_summary(self, uuid): + """ + Returns the summary of an appointment. The summary consists of the data kept in memory: + locator, end_time, and size. + + Args: + uuid (:obj:`str`): a 16-byte hex string identifying the appointment. + + Returns: + :obj:`dict` or :obj:`None`: a dictionary with the appointment summary, or None if the appointment is not + found. + """ + + return self.appointments.get(uuid) + def add_appointment(self, appointment, user_pk): """ Adds a new appointment to the ``appointments`` dictionary if ``max_appointments`` has not been reached. @@ -112,7 +127,11 @@ class Watcher: # anything about the user from this point on (no need to store user_pk in the database). # If an appointment is requested by the user the uuid can be recomputed and queried straightaway (no maps). uuid = hash_160("{}{}".format(appointment.locator, user_pk)) - self.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} + self.appointments[uuid] = { + "locator": appointment.locator, + "end_time": appointment.end_time, + "size": len(appointment.encrypted_blob.data), + } if appointment.locator in self.locator_uuid_map: # If the uuid is already in the map it means this is an update. From 33966e59e11115311e06cfdd3c04cea8f4a053c8 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 12:44:48 +0200 Subject: [PATCH 28/62] Moves (and renames) check_user_pk to common so cli can use it too. Fixes the regex so it only accepts {02, 03} as prefix --- common/tools.py | 15 ++++++++++++++- teos/gatekeeper.py | 19 ++----------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/common/tools.py b/common/tools.py index b02c1c4..ccd5973 100644 --- a/common/tools.py +++ b/common/tools.py @@ -1,10 +1,23 @@ import re -import os import logging from pathlib import Path from common.constants import LOCATOR_LEN_HEX +def check_compressed_pk_format(compressed_pk): + """ + Checks if a given value is a 33-byte hex encoded string. + + Args: + compressed_pk(:obj:`str`): the value to be checked. + + Returns: + :obj:`bool`: Whether or not the value matches the format. + """ + + return isinstance(compressed_pk, str) and re.match(r"^0[2-3][0-9A-Fa-f]{64}$", compressed_pk) is not None + + def check_sha256_hex_format(value): """ Checks if a given value is a 32-byte hex encoded string. diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index 0d5131b..2b812fd 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -1,5 +1,4 @@ -import re - +from common.tools import check_compressed_pk_format from common.cryptographer import Cryptographer @@ -33,20 +32,6 @@ class Gatekeeper: self.default_slots = default_slots self.registered_users = {} - @staticmethod - def check_user_pk(user_pk): - """ - Checks if a given value is a 33-byte hex encoded string. - - Args: - user_pk(:obj:`str`): the value to be checked. - - Returns: - :obj:`bool`: Whether or not the value matches the format. - """ - - return isinstance(user_pk, str) and re.match(r"^[0-9A-Fa-f]{66}$", user_pk) is not None - def add_update_user(self, user_pk): """ Adds a new user or updates the subscription of an existing one, by adding additional slots. @@ -58,7 +43,7 @@ class Gatekeeper: :obj:`int`: the number of available slots in the user subscription. """ - if not self.check_user_pk(user_pk): + if not check_compressed_pk_format(user_pk): raise ValueError("provided public key does not match expected format (33-byte hex string)") if user_pk not in self.registered_users: From c0ada5f818ecc7d9f352f0b06ba434eb24fce57d Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 12:45:39 +0200 Subject: [PATCH 29/62] Improves logging messages --- common/cryptographer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/cryptographer.py b/common/cryptographer.py index 5414fda..fbc3cf8 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -216,7 +216,7 @@ class Cryptographer: return key except FileNotFoundError: - logger.error("Key file not found. Please check your settings") + logger.error("Key file not found at {}. Please check your settings".format(file_path)) return None except IOError as e: From 709a40dc64ec075c971a2cf5e4c3781b9053083a Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 13:08:14 +0200 Subject: [PATCH 30/62] Fixes gatekeeper and common/tools tests according to 33966e59e11115311e06cfdd3c04cea8f4a053c8 --- test/common/unit/test_tools.py | 29 +++++++++++++++++++++++++ test/teos/unit/test_gatekeeper.py | 36 +++++++++++++------------------ 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/test/common/unit/test_tools.py b/test/common/unit/test_tools.py index 57be93d..0410670 100644 --- a/test/common/unit/test_tools.py +++ b/test/common/unit/test_tools.py @@ -3,6 +3,7 @@ import logging from common.constants import LOCATOR_LEN_BYTES from common.tools import ( + check_compressed_pk_format, check_sha256_hex_format, check_locator_format, compute_locator, @@ -12,6 +13,34 @@ from common.tools import ( from test.common.unit.conftest import get_random_value_hex +def test_check_compressed_pk_format(): + wrong_values = [ + None, + 3, + 15.23, + "", + {}, + (), + object, + str, + get_random_value_hex(32), + get_random_value_hex(34), + "06" + get_random_value_hex(32), + ] + + # check_user_pk must only accept values that is not a 33-byte hex string + for i in range(100): + if i % 2: + prefix = "02" + else: + prefix = "03" + assert check_compressed_pk_format(prefix + get_random_value_hex(32)) + + # check_user_pk must only accept values that is not a 33-byte hex string + for value in wrong_values: + assert not check_compressed_pk_format(value) + + def test_check_sha256_hex_format(): # Only 32-byte hex encoded strings should pass the test wrong_inputs = [None, str(), 213, 46.67, dict(), "A" * 63, "C" * 65, bytes(), get_random_value_hex(31)] diff --git a/test/teos/unit/test_gatekeeper.py b/test/teos/unit/test_gatekeeper.py index eccc8c8..5608092 100644 --- a/test/teos/unit/test_gatekeeper.py +++ b/test/teos/unit/test_gatekeeper.py @@ -15,23 +15,9 @@ def test_init(): assert isinstance(gatekeeper.registered_users, dict) and len(gatekeeper.registered_users) == 0 -def test_check_user_pk(): - # check_user_pk must only accept values that is not a 33-byte hex string - for _ in range(100): - assert gatekeeper.check_user_pk(get_random_value_hex(33)) - - -def test_check_wrong_user_pk(): - wrong_values = [None, 3, 15.23, "", {}, (), object, str, get_random_value_hex(32), get_random_value_hex(34)] - - # check_user_pk must only accept values that is not a 33-byte hex string - for value in wrong_values: - assert not gatekeeper.check_user_pk(value) - - def test_add_update_user(): - # add_update_user adds DEFAULT_SLOTS to a given user as long as the identifier is a 33-byte hex str - user_pk = get_random_value_hex(33) + # add_update_user adds DEFAULT_SLOTS to a given user as long as the identifier is {02, 03}| 32-byte hex str + user_pk = "02" + get_random_value_hex(32) for _ in range(10): current_slots = gatekeeper.registered_users.get(user_pk) @@ -44,7 +30,7 @@ def test_add_update_user(): # The same can be checked for multiple users for _ in range(10): # The user identifier is changed every call - user_pk = get_random_value_hex(33) + user_pk = "03" + get_random_value_hex(32) gatekeeper.add_update_user(user_pk) assert gatekeeper.registered_users.get(user_pk) == DEFAULT_SLOTS @@ -58,6 +44,14 @@ def test_add_update_user_wrong_pk(): gatekeeper.add_update_user(wrong_pk) +def test_add_update_user_wrong_pk_prefix(): + # Prefixes must be 02 or 03, anything else should fail + wrong_pk = "04" + get_random_value_hex(32) + + with pytest.raises(ValueError): + gatekeeper.add_update_user(wrong_pk) + + def test_identify_user(): # Identify user should return a user_pk for registered users. It raises # IdentificationFailure for invalid parameters or non-registered users. @@ -115,7 +109,7 @@ def test_identify_user_wrong(): def test_fill_slots(): # Free slots will decrease the slot count of a user as long as he has enough slots, otherwise raise NotEnoughSlots - user_pk = get_random_value_hex(33) + user_pk = "02" + get_random_value_hex(32) gatekeeper.add_update_user(user_pk) gatekeeper.fill_slots(user_pk, DEFAULT_SLOTS - 1) @@ -131,11 +125,11 @@ def test_fill_slots(): def test_free_slots(): # Free slots simply adds slots to the user as long as it exists. - user_pk = get_random_value_hex(33) + user_pk = "03" + get_random_value_hex(32) gatekeeper.add_update_user(user_pk) - gatekeeper.free_slots(user_pk, 33) + gatekeeper.free_slots(user_pk, 42) - assert gatekeeper.registered_users.get(user_pk) == DEFAULT_SLOTS + 33 + assert gatekeeper.registered_users.get(user_pk) == DEFAULT_SLOTS + 42 # Just making sure it does not crash for non-registered user assert gatekeeper.free_slots(get_random_value_hex(33), 10) is None From 495ea3800e4c5245da7cc4c0d3cca945f99f08df Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 16:05:13 +0200 Subject: [PATCH 31/62] Fixes get_appointment response structure: {"locator": l, "status": s, "data": appointment_data} --- teos/api.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/teos/api.py b/teos/api.py index 6bffe7f..922cf43 100644 --- a/teos/api.py +++ b/teos/api.py @@ -108,7 +108,7 @@ class API: request_data = get_request_data_json(request) except TypeError as e: - logger.info("Received invalid get_appointment request", from_addr="{}".format(remote_addr)) + logger.info("Received invalid register request", from_addr="{}".format(remote_addr)) return abort(HTTP_BAD_REQUEST, e) client_pk = request_data.get("public_key") @@ -267,20 +267,20 @@ class API: # If the appointment has been triggered, it should be in the locator (default else just in case). if uuid in triggered_appointments: - response = self.watcher.db_manager.load_responder_tracker(uuid) - if response: + appointment_data = self.watcher.db_manager.load_responder_tracker(uuid) + if appointment_data: rcode = HTTP_OK - response["status"] = "dispute_responded" + response = {"locator": locator, "status": "dispute_responded", "appointment": appointment_data} else: rcode = HTTP_NOT_FOUND response = {"locator": locator, "status": "not_found"} # Otherwise it should be either in the watcher, or not in the system. else: - response = self.watcher.db_manager.load_watcher_appointment(uuid) - if response: + appointment_data = self.watcher.db_manager.load_watcher_appointment(uuid) + if appointment_data: rcode = HTTP_OK - response["status"] = "being_watched" + response = {"locator": locator, "status": "being_watched", "appointment": appointment_data} else: rcode = HTTP_NOT_FOUND response = {"locator": locator, "status": "not_found"} From f55e37b0fc831f8f3d4b64371f1bf0c9d5846343 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 16:19:04 +0200 Subject: [PATCH 32/62] Adds unit test for watcher.get_appointment_summary --- teos/watcher.py | 9 ++++++--- test/teos/unit/test_watcher.py | 13 ++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/teos/watcher.py b/teos/watcher.py index 757ffc0..24fec8b 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -39,8 +39,8 @@ class Watcher: expiry_delta (:obj:`int`): the additional time the ``Watcher`` will keep an expired appointment around. Attributes: - appointments (:obj:`dict`): a dictionary containing a simplification of the appointments (:obj:`Appointment - ` instances) accepted by the tower (``locator`` and ``end_time``). + appointments (:obj:`dict`): a dictionary containing a summary of the appointments (:obj:`Appointment + ` instances) accepted by the tower (``locator``, ``end_time``, and ``size``). It's populated trough ``add_appointment``. locator_uuid_map (:obj:`dict`): a ``locator:uuid`` map used to allow the :obj:`Watcher` to deal with several appointments with the same ``locator``. @@ -71,6 +71,10 @@ class Watcher: self.signing_key = Cryptographer.load_private_key_der(sk_der) def awake(self): + """ + Starts a new thread to monitor the blockchain for channel breaches. + """ + watcher_thread = Thread(target=self.do_watch, daemon=True) watcher_thread.start() @@ -88,7 +92,6 @@ class Watcher: :obj:`dict` or :obj:`None`: a dictionary with the appointment summary, or None if the appointment is not found. """ - return self.appointments.get(uuid) def add_appointment(self, appointment, user_pk): diff --git a/test/teos/unit/test_watcher.py b/test/teos/unit/test_watcher.py index 7645c80..e61a178 100644 --- a/test/teos/unit/test_watcher.py +++ b/test/teos/unit/test_watcher.py @@ -112,6 +112,17 @@ def test_init(run_bitcoind, watcher): assert isinstance(watcher.signing_key, PrivateKey) +def test_get_appointment_summary(watcher): + # get_appointment_summary returns an appointment summary if found, else None. + random_uuid = get_random_value_hex(16) + appointment_summary = {"locator": get_random_value_hex(16), "end_time": 10, "size": 200} + watcher.appointments[random_uuid] = appointment_summary + assert watcher.get_appointment_summary(random_uuid) == appointment_summary + + # Requesting a non-existing appointment + assert watcher.get_appointment_summary(get_random_value_hex(16)) is None + + def test_add_appointment(watcher): # We should be able to add appointments up to the limit for _ in range(10): @@ -186,7 +197,7 @@ def test_do_watch(watcher, temp_db_manager): watcher.appointments = {} for uuid, appointment in appointments.items(): - watcher.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} + watcher.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time, "size": 200} watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) watcher.db_manager.create_append_locator_map(appointment.locator, uuid) From 93a389508e68a248d7014cea65340c43cfe09236 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 16:35:45 +0200 Subject: [PATCH 33/62] Adds checks of decreasing slots for added appointments --- test/teos/unit/test_api.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 9af8c81..46ba635 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -147,6 +147,7 @@ def test_add_appointment(api, appointment): appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) assert r.status_code == HTTP_OK + assert r.json().get("available_slots") == 0 def test_add_appointment_no_json(api, appointment): @@ -225,6 +226,7 @@ def test_add_appointment_multiple_times_same_user(api, appointment, n=MULTIPLE_A for _ in range(n): r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) assert r.status_code == HTTP_OK + assert r.json().get("available_slots") == n - 1 # Since all updates came from the same user, only the last one is stored assert len(api.watcher.locator_uuid_map[appointment.locator]) == 1 @@ -237,12 +239,13 @@ def test_add_appointment_multiple_times_different_users(api, appointment, n=MULT # Add one slot per public key for pair in user_keys: - api.gatekeeper.registered_users[hexlify(pair[1].format(compressed=True)).decode("utf-8")] = 1 + api.gatekeeper.registered_users[hexlify(pair[1].format(compressed=True)).decode("utf-8")] = 2 # Send the appointments for signature in signatures: r = add_appointment({"appointment": appointment.to_dict(), "signature": signature}) assert r.status_code == HTTP_OK + assert r.json().get("available_slots") == 1 # Check that all the appointments have been added and that there are no duplicates assert len(set(api.watcher.locator_uuid_map[appointment.locator])) == n @@ -298,13 +301,13 @@ def test_request_appointment_in_watcher(api, appointment): r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) assert r.status_code == HTTP_OK - appointment_data = json.loads(r.content) + r_json = json.loads(r.content) # Check that the appointment is on the watcher - status = appointment_data.pop("status") - assert status == "being_watched" + assert r_json.get("status") == "being_watched" # Check the the sent appointment matches the received one - assert appointment.to_dict() == appointment_data + assert r_json.get("locator") == appointment.locator + assert appointment.to_dict() == r_json.get("appointment") def test_request_appointment_in_responder(api, appointment): @@ -332,22 +335,20 @@ def test_request_appointment_in_responder(api, appointment): r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) assert r.status_code == HTTP_OK - appointment_data = json.loads(r.content) + r_json = json.loads(r.content) # Check that the appointment is on the watcher - status = appointment_data.pop("status") - assert status == "dispute_responded" + assert r_json.get("status") == "dispute_responded" # Check the the sent appointment matches the received one - assert appointment.locator == appointment_data.get("locator") + assert appointment.locator == r_json.get("locator") assert appointment.encrypted_blob.data == Cryptographer.encrypt( - Blob(appointment_data.get("penalty_rawtx")), appointment_data.get("dispute_txid") + Blob(r_json.get("appointment").get("penalty_rawtx")), r_json.get("appointment").get("dispute_txid") ) # Delete appointment so it does not mess up with future tests appointments.pop() - uuids = api.watcher.responder.tx_tracker_map.pop(appointment_data.get("penalty_txid")) + uuids = api.watcher.responder.tx_tracker_map.pop(r_json.get("appointment").get("penalty_txid")) api.watcher.responder.db_manager.delete_responder_tracker(uuids[0]) - # api.watcher.responder.trackers.pop(uuids[0]) def test_get_all_appointments_watcher(): From c9b3bb625f085391c0ae1f3cc65d1129ab7df8a1 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 17:56:47 +0200 Subject: [PATCH 34/62] Fixes slots difference calculation --- teos/api.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/teos/api.py b/teos/api.py index 922cf43..06c53ae 100644 --- a/teos/api.py +++ b/teos/api.py @@ -1,6 +1,6 @@ import os import logging -from math import ceil +from math import ceil, floor from flask import Flask, request, abort, jsonify import teos.errors as errors @@ -175,8 +175,10 @@ class API: # For updates we only reserve the slot difference provided the new one is bigger. if appointment_summary: - size_diff = len(appointment.encrypted_blob.data) - appointment_summary.get("size") - slot_diff = ceil(size_diff / ENCRYPTED_BLOB_MAX_SIZE_HEX) + used_slots = ceil(appointment_summary.get("size") / ENCRYPTED_BLOB_MAX_SIZE_HEX) + required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) + slot_diff = required_slots - used_slots + required_slots = slot_diff if slot_diff > 0 else 0 # For regular appointments 1 slot is reserved per ENCRYPTED_BLOB_MAX_SIZE_HEX block. @@ -191,6 +193,10 @@ class API: appointment_added, signature = self.watcher.add_appointment(appointment, user_pk) if appointment_added: + # If the appointment is added and the update is smaller than the original, the difference is given back. + if slot_diff < 0: + self.gatekeeper.free_slots(user_pk, abs(slot_diff)) + rcode = HTTP_OK response = { "locator": appointment.locator, @@ -198,10 +204,6 @@ class API: "available_slots": self.gatekeeper.registered_users[user_pk], } - # If the appointment is added and the update is smaller than the original, the difference is given back. - if slot_diff < 0: - self.gatekeeper.free_slots(slot_diff) - else: # If the appointment is not added the reserved slots are given back self.gatekeeper.free_slots(user_pk, required_slots) From 9ecf98e0c5ab64c7610b77901926ca677889eedb Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 17:57:47 +0200 Subject: [PATCH 35/62] Adds tests for appointments updates of different sizes --- test/teos/unit/test_api.py | 68 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 46ba635..6d922cb 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -28,7 +28,14 @@ from test.teos.unit.conftest import ( from common.blob import Blob from common.cryptographer import Cryptographer -from common.constants import HTTP_OK, HTTP_NOT_FOUND, HTTP_BAD_REQUEST, HTTP_SERVICE_UNAVAILABLE, LOCATOR_LEN_BYTES +from common.constants import ( + HTTP_OK, + HTTP_NOT_FOUND, + HTTP_BAD_REQUEST, + HTTP_SERVICE_UNAVAILABLE, + LOCATOR_LEN_BYTES, + ENCRYPTED_BLOB_MAX_SIZE_HEX, +) TEOS_API = "http://{}:{}".format(HOST, PORT) @@ -201,7 +208,7 @@ def test_add_appointment_registered_no_free_slots(api, appointment): def test_add_appointment_registered_not_enough_free_slots(api, appointment): - # Empty the user slots + # Give some slots to the user api.gatekeeper.registered_users[client_pk_hex] = 1 # Properly formatted appointment @@ -217,7 +224,7 @@ def test_add_appointment_registered_not_enough_free_slots(api, appointment): def test_add_appointment_multiple_times_same_user(api, appointment, n=MULTIPLE_APPOINTMENTS): - # Multiple appointments with the same locator should be valid + # Multiple appointments with the same locator should be valid and counted as updates appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) # Simulate registering enough slots @@ -251,6 +258,61 @@ def test_add_appointment_multiple_times_different_users(api, appointment, n=MULT assert len(set(api.watcher.locator_uuid_map[appointment.locator])) == n +def test_add_appointment_update_same_size(api, appointment): + # Update an appointment by one of the same size and check that no additional slots are filled + api.gatekeeper.registered_users[client_pk_hex] = 1 + + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK and r.json().get("available_slots") == 0 + + # The user has no additional slots, but it should be able to update + # Let's just reverse the encrypted blob for example + appointment.encrypted_blob.data = appointment.encrypted_blob.data[::-1] + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK and r.json().get("available_slots") == 0 + + +def test_add_appointment_update_bigger(api, appointment): + # Update an appointment by one bigger, and check additional slots are filled + api.gatekeeper.registered_users[client_pk_hex] = 2 + + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK and r.json().get("available_slots") == 1 + + # The user has one slot, so it should be able to update as long as it only takes 1 additional slot + appointment.encrypted_blob.data = "A" * ENCRYPTED_BLOB_MAX_SIZE_HEX + "AA" + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK and r.json().get("available_slots") == 0 + + # Check that it'll fail if no enough slots are available + # Double the size from before + appointment.encrypted_blob.data = "AA" * ENCRYPTED_BLOB_MAX_SIZE_HEX + "AA" + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_BAD_REQUEST + + +def test_add_appointment_update_smaller(api, appointment): + # Update an appointment by one bigger, and check slots are freed + api.gatekeeper.registered_users[client_pk_hex] = 2 + + # This should take 2 slots + appointment.encrypted_blob.data = "A" * ENCRYPTED_BLOB_MAX_SIZE_HEX + "AA" + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK and r.json().get("available_slots") == 0 + + # Let's update with one just small enough + appointment.encrypted_blob.data = "A" * (ENCRYPTED_BLOB_MAX_SIZE_HEX - 2) + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + assert r.status_code == HTTP_OK and r.json().get("available_slots") == 1 + + def test_get_appointment_no_json(api, appointment): r = requests.post(url=add_appointment_endpoint, data="random_message", timeout=5) assert r.status_code == HTTP_BAD_REQUEST From 354724075e1160da3c4bea2751d0d1cf06945e4d Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 22:14:10 +0200 Subject: [PATCH 36/62] Moves route assignment to API constructor so it can be properly tested --- teos/api.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/teos/api.py b/teos/api.py index 06c53ae..17cd45b 100644 --- a/teos/api.py +++ b/teos/api.py @@ -82,6 +82,18 @@ class API: self.inspector = inspector self.watcher = watcher self.gatekeeper = gatekeeper + self.app = app + + # Adds all the routes to the functions listed above. + routes = { + "/register": (self.register, ["POST"]), + "/add_appointment": (self.add_appointment, ["POST"]), + "/get_appointment": (self.get_appointment, ["POST"]), + "/get_all_appointments": (self.get_all_appointments, ["GET"]), + } + + for url, params in routes.items(): + app.add_url_rule(url, view_func=params[0], methods=params[1]) def register(self): """ @@ -322,19 +334,9 @@ class API: def start(self): """ - This function starts the Flask server used to run the API. Adds all the routes to the functions listed above. + This function starts the Flask server used to run the API. """ - routes = { - "/register": (self.register, ["POST"]), - "/add_appointment": (self.add_appointment, ["POST"]), - "/get_appointment": (self.get_appointment, ["POST"]), - "/get_all_appointments": (self.get_all_appointments, ["GET"]), - } - - for url, params in routes.items(): - app.add_url_rule(url, view_func=params[0], methods=params[1]) - # Setting Flask log to ERROR only so it does not mess with our logging. Also disabling flask initial messages logging.getLogger("werkzeug").setLevel(logging.ERROR) os.environ["WERKZEUG_RUN_MAIN"] = "true" From 746a78542f53ec708e7a3bbee02e110d509b99a4 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 22:16:39 +0200 Subject: [PATCH 37/62] Adds missing tests to API and improves API testing by properly mocking the requests. Closes #77 --- test/teos/unit/test_api.py | 375 ++++++++++++++++++++----------------- 1 file changed, 206 insertions(+), 169 deletions(-) diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 6d922cb..423081c 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -1,9 +1,5 @@ -import json import pytest -import requests -from time import sleep from binascii import hexlify -from threading import Thread from teos.api import API from teos import HOST, PORT @@ -27,7 +23,7 @@ from test.teos.unit.conftest import ( ) from common.blob import Blob -from common.cryptographer import Cryptographer +from common.cryptographer import Cryptographer, hash_160 from common.constants import ( HTTP_OK, HTTP_NOT_FOUND, @@ -46,21 +42,22 @@ get_all_appointment_endpoint = "{}/get_all_appointments".format(TEOS_API) # Reduce the maximum number of appointments to something we can test faster MAX_APPOINTMENTS = 100 - MULTIPLE_APPOINTMENTS = 10 -appointments = [] +TWO_SLOTS_BLOTS = "A" * ENCRYPTED_BLOB_MAX_SIZE_HEX + "AA" + +appointments = {} locator_dispute_tx_map = {} config = get_config() client_sk, client_pk = generate_keypair() -client_pk_hex = hexlify(client_pk.format(compressed=True)).decode("utf-8") +compressed_client_pk = hexlify(client_pk.format(compressed=True)).decode("utf-8") -@pytest.fixture(scope="module") -def api(db_manager, carrier, block_processor): +@pytest.fixture(scope="module", autouse=True) +def api(db_manager, carrier, block_processor, run_bitcoind): sk, pk = generate_keypair() @@ -75,16 +72,21 @@ def api(db_manager, carrier, block_processor): gatekeeper = Gatekeeper(config.get("DEFAULT_SLOTS")) api = API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, gatekeeper) - api_thread = Thread(target=api.start) - api_thread.daemon = True - api_thread.start() - - # It takes a little bit of time to start the API (otherwise the requests are sent too early and they fail) - sleep(0.1) return api +@pytest.fixture() +def app(api): + with api.app.app_context(): + yield api.app + + +@pytest.fixture +def client(app): + return app.test_client() + + @pytest.fixture def appointment(): appointment, dispute_tx = generate_dummy_appointment() @@ -93,24 +95,26 @@ def appointment(): return appointment -def add_appointment(appointment_data): - r = requests.post(url=add_appointment_endpoint, json=appointment_data, timeout=5) +def add_appointment(client, appointment_data, user_pk): + r = client.post(add_appointment_endpoint, json=appointment_data) if r.status_code == HTTP_OK: - appointments.append(appointment_data["appointment"]) + locator = appointment_data.get("appointment").get("locator") + uuid = hash_160("{}{}".format(locator, user_pk)) + appointments[uuid] = appointment_data["appointment"] return r -def test_register(api, run_bitcoind): - data = {"public_key": client_pk_hex} - r = requests.post(url=register_endpoint, json=data, timeout=5) +def test_register(client): + data = {"public_key": compressed_client_pk} + r = client.post(register_endpoint, json=data) assert r.status_code == HTTP_OK - assert r.json().get("public_key") == client_pk_hex - assert r.json().get("available_slots") == config.get("DEFAULT_SLOTS") + assert r.json.get("public_key") == compressed_client_pk + assert r.json.get("available_slots") == config.get("DEFAULT_SLOTS") -def test_register_top_up(run_bitcoind): +def test_register_top_up(client): # Calling register more than once will give us DEFAULT_SLOTS * number_of_calls slots temp_sk, tmp_pk = generate_keypair() tmp_pk_hex = hexlify(tmp_pk.format(compressed=True)).decode("utf-8") @@ -118,241 +122,201 @@ def test_register_top_up(run_bitcoind): data = {"public_key": tmp_pk_hex} for i in range(10): - r = requests.post(url=register_endpoint, json=data, timeout=5) + r = client.post(register_endpoint, json=data) assert r.status_code == HTTP_OK - assert r.json().get("public_key") == tmp_pk_hex - assert r.json().get("available_slots") == config.get("DEFAULT_SLOTS") * (i + 1) + assert r.json.get("public_key") == tmp_pk_hex + assert r.json.get("available_slots") == config.get("DEFAULT_SLOTS") * (i + 1) -def test_register_no_client_pk(run_bitcoind): - data = {"public_key": client_pk_hex + client_pk_hex} - r = requests.post(url=register_endpoint, json=data, timeout=5) +def test_register_no_client_pk(client): + data = {"public_key": compressed_client_pk + compressed_client_pk} + r = client.post(register_endpoint, json=data) assert r.status_code == HTTP_BAD_REQUEST -def test_register_wrong_client_pk(run_bitcoind): +def test_register_wrong_client_pk(client): data = {} - r = requests.post(url=register_endpoint, json=data, timeout=5) + r = client.post(register_endpoint, json=data) assert r.status_code == HTTP_BAD_REQUEST -def test_register_no_json(api, appointment): - r = requests.post(url=register_endpoint, data="random_message", timeout=5) +def test_register_no_json(client): + r = client.post(register_endpoint, data="random_message") assert r.status_code == HTTP_BAD_REQUEST -def test_register_json_no_inner_dict(api, appointment): - r = requests.post(url=register_endpoint, json="random_message", timeout=5) +def test_register_json_no_inner_dict(client): + r = client.post(register_endpoint, json="random_message") assert r.status_code == HTTP_BAD_REQUEST -def test_add_appointment(api, appointment): +def test_add_appointment(api, client, appointment): # Simulate the user registration - api.gatekeeper.registered_users[client_pk_hex] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = 1 # Properly formatted appointment appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) assert r.status_code == HTTP_OK - assert r.json().get("available_slots") == 0 + assert r.json.get("available_slots") == 0 -def test_add_appointment_no_json(api, appointment): +def test_add_appointment_no_json(api, client, appointment): # Simulate the user registration - api.gatekeeper.registered_users[client_pk_hex] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = 1 # Properly formatted appointment - r = requests.post(url=add_appointment_endpoint, data="random_message", timeout=5) + r = client.post(add_appointment_endpoint, data="random_message") assert r.status_code == HTTP_BAD_REQUEST -def test_add_appointment_json_no_inner_dict(api, appointment): +def test_add_appointment_json_no_inner_dict(api, client, appointment): # Simulate the user registration - api.gatekeeper.registered_users[client_pk_hex] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = 1 # Properly formatted appointment - r = requests.post(url=add_appointment_endpoint, json="random_message", timeout=5) + r = client.post(add_appointment_endpoint, json="random_message") assert r.status_code == HTTP_BAD_REQUEST -def test_add_appointment_wrong(api, appointment): +def test_add_appointment_wrong(api, client, appointment): # Simulate the user registration - api.gatekeeper.registered_users[client_pk_hex] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = 1 # Incorrect appointment appointment.to_self_delay = 0 appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) assert r.status_code == HTTP_BAD_REQUEST - assert "Error {}:".format(errors.APPOINTMENT_FIELD_TOO_SMALL) in r.json().get("error") + assert "Error {}:".format(errors.APPOINTMENT_FIELD_TOO_SMALL) in r.json.get("error") -def test_add_appointment_not_registered(api, appointment): +def test_add_appointment_not_registered(api, client, appointment): # Properly formatted appointment tmp_sk, tmp_pk = generate_keypair() + tmp_compressed_pk = hexlify(tmp_pk.format(compressed=True)).decode("utf-8") + appointment_signature = Cryptographer.sign(appointment.serialize(), tmp_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, tmp_compressed_pk + ) assert r.status_code == HTTP_BAD_REQUEST - assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json().get("error") + assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json.get("error") -def test_add_appointment_registered_no_free_slots(api, appointment): +def test_add_appointment_registered_no_free_slots(api, client, appointment): # Empty the user slots - api.gatekeeper.registered_users[client_pk_hex] = 0 + api.gatekeeper.registered_users[compressed_client_pk] = 0 # Properly formatted appointment appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) assert r.status_code == HTTP_BAD_REQUEST - assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json().get("error") + assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json.get("error") -def test_add_appointment_registered_not_enough_free_slots(api, appointment): +def test_add_appointment_registered_not_enough_free_slots(api, client, appointment): # Give some slots to the user - api.gatekeeper.registered_users[client_pk_hex] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = 1 # Properly formatted appointment appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) # Let's create a big blob - for _ in range(10): - appointment.encrypted_blob.data += appointment.encrypted_blob.data + appointment.encrypted_blob.data = TWO_SLOTS_BLOTS - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) assert r.status_code == HTTP_BAD_REQUEST - assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json().get("error") + assert "Error {}:".format(errors.APPOINTMENT_INVALID_SIGNATURE_OR_INSUFFICIENT_SLOTS) in r.json.get("error") -def test_add_appointment_multiple_times_same_user(api, appointment, n=MULTIPLE_APPOINTMENTS): +def test_add_appointment_multiple_times_same_user(api, client, appointment, n=MULTIPLE_APPOINTMENTS): # Multiple appointments with the same locator should be valid and counted as updates appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) # Simulate registering enough slots - api.gatekeeper.registered_users[client_pk_hex] = n - # DISCUSS: #34-store-identical-appointments + api.gatekeeper.registered_users[compressed_client_pk] = n for _ in range(n): - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) assert r.status_code == HTTP_OK - assert r.json().get("available_slots") == n - 1 + assert r.json.get("available_slots") == n - 1 # Since all updates came from the same user, only the last one is stored assert len(api.watcher.locator_uuid_map[appointment.locator]) == 1 -def test_add_appointment_multiple_times_different_users(api, appointment, n=MULTIPLE_APPOINTMENTS): +def test_add_appointment_multiple_times_different_users(api, client, appointment, n=MULTIPLE_APPOINTMENTS): # Create user keys and appointment signatures user_keys = [generate_keypair() for _ in range(n)] signatures = [Cryptographer.sign(appointment.serialize(), key[0]) for key in user_keys] + compressed_pks = [hexlify(pk.format(compressed=True)).decode("utf-8") for sk, pk in user_keys] # Add one slot per public key for pair in user_keys: api.gatekeeper.registered_users[hexlify(pair[1].format(compressed=True)).decode("utf-8")] = 2 # Send the appointments - for signature in signatures: - r = add_appointment({"appointment": appointment.to_dict(), "signature": signature}) + for compressed_pk, signature in zip(compressed_pks, signatures): + r = add_appointment(client, {"appointment": appointment.to_dict(), "signature": signature}, compressed_pk) assert r.status_code == HTTP_OK - assert r.json().get("available_slots") == 1 + assert r.json.get("available_slots") == 1 # Check that all the appointments have been added and that there are no duplicates assert len(set(api.watcher.locator_uuid_map[appointment.locator])) == n -def test_add_appointment_update_same_size(api, appointment): - # Update an appointment by one of the same size and check that no additional slots are filled - api.gatekeeper.registered_users[client_pk_hex] = 1 - - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) - assert r.status_code == HTTP_OK and r.json().get("available_slots") == 0 - - # The user has no additional slots, but it should be able to update - # Let's just reverse the encrypted blob for example - appointment.encrypted_blob.data = appointment.encrypted_blob.data[::-1] - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) - assert r.status_code == HTTP_OK and r.json().get("available_slots") == 0 - - -def test_add_appointment_update_bigger(api, appointment): - # Update an appointment by one bigger, and check additional slots are filled - api.gatekeeper.registered_users[client_pk_hex] = 2 - - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) - assert r.status_code == HTTP_OK and r.json().get("available_slots") == 1 - - # The user has one slot, so it should be able to update as long as it only takes 1 additional slot - appointment.encrypted_blob.data = "A" * ENCRYPTED_BLOB_MAX_SIZE_HEX + "AA" - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) - assert r.status_code == HTTP_OK and r.json().get("available_slots") == 0 - - # Check that it'll fail if no enough slots are available - # Double the size from before - appointment.encrypted_blob.data = "AA" * ENCRYPTED_BLOB_MAX_SIZE_HEX + "AA" - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) +def test_get_appointment_no_json(api, client, appointment): + r = client.post(add_appointment_endpoint, data="random_message") assert r.status_code == HTTP_BAD_REQUEST -def test_add_appointment_update_smaller(api, appointment): - # Update an appointment by one bigger, and check slots are freed - api.gatekeeper.registered_users[client_pk_hex] = 2 - - # This should take 2 slots - appointment.encrypted_blob.data = "A" * ENCRYPTED_BLOB_MAX_SIZE_HEX + "AA" - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) - assert r.status_code == HTTP_OK and r.json().get("available_slots") == 0 - - # Let's update with one just small enough - appointment.encrypted_blob.data = "A" * (ENCRYPTED_BLOB_MAX_SIZE_HEX - 2) - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) - assert r.status_code == HTTP_OK and r.json().get("available_slots") == 1 - - -def test_get_appointment_no_json(api, appointment): - r = requests.post(url=add_appointment_endpoint, data="random_message", timeout=5) +def test_get_appointment_json_no_inner_dict(api, client, appointment): + r = client.post(add_appointment_endpoint, json="random_message") assert r.status_code == HTTP_BAD_REQUEST -def test_get_appointment_json_no_inner_dict(api, appointment): - r = requests.post(url=add_appointment_endpoint, json="random_message", timeout=5) - assert r.status_code == HTTP_BAD_REQUEST - - -def test_request_random_appointment_registered_user(user_sk=client_sk): +def test_request_random_appointment_registered_user(client, user_sk=client_sk): locator = get_random_value_hex(LOCATOR_LEN_BYTES) message = "get appointment {}".format(locator) signature = Cryptographer.sign(message.encode("utf-8"), user_sk) data = {"locator": locator, "signature": signature} - r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) + r = client.post(get_appointment_endpoint, json=data) # We should get a 404 not found since we are using a made up locator - received_appointment = r.json() + received_appointment = r.json assert r.status_code == HTTP_NOT_FOUND assert received_appointment.get("status") == "not_found" -def test_request_appointment_not_registered_user(): +def test_request_appointment_not_registered_user(client): # Not registered users have no associated appointments, so this should fail tmp_sk, tmp_pk = generate_keypair() # The tower is designed so a not found appointment and a request from a non-registered user return the same error to # prevent proving. - test_request_random_appointment_registered_user(tmp_sk) + test_request_random_appointment_registered_user(client, tmp_sk) -def test_request_appointment_in_watcher(api, appointment): +def test_request_appointment_in_watcher(api, client, appointment): # Give slots to the user - api.gatekeeper.registered_users[client_pk_hex] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = 1 # Add an appointment appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) assert r.status_code == HTTP_OK message = "get appointment {}".format(appointment.locator) @@ -360,29 +324,30 @@ def test_request_appointment_in_watcher(api, appointment): data = {"locator": appointment.locator, "signature": signature} # Next we can request it - r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) + r = client.post(get_appointment_endpoint, json=data) assert r.status_code == HTTP_OK - r_json = json.loads(r.content) # Check that the appointment is on the watcher - assert r_json.get("status") == "being_watched" + assert r.json.get("status") == "being_watched" # Check the the sent appointment matches the received one - assert r_json.get("locator") == appointment.locator - assert appointment.to_dict() == r_json.get("appointment") + assert r.json.get("locator") == appointment.locator + assert appointment.to_dict() == r.json.get("appointment") -def test_request_appointment_in_responder(api, appointment): +def test_request_appointment_in_responder(api, client, appointment): # Give slots to the user - api.gatekeeper.registered_users[client_pk_hex] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = 1 # Let's do something similar to what we did with the watcher but now we'll send the dispute tx to the network dispute_tx = locator_dispute_tx_map.pop(appointment.locator) bitcoin_cli(bitcoind_connect_params).sendrawtransaction(dispute_tx) - # Add an appointment + # Add an appointment (avoid calling add_appointment to not add this one to the sent appointments list) appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = client.post( + add_appointment_endpoint, json={"appointment": appointment.to_dict(), "signature": appointment_signature} + ) assert r.status_code == HTTP_OK # Generate a block to trigger the watcher @@ -394,64 +359,134 @@ def test_request_appointment_in_responder(api, appointment): data = {"locator": appointment.locator, "signature": signature} # Next we can request it - r = requests.post(url=get_appointment_endpoint, json=data, timeout=5) + r = client.post(get_appointment_endpoint, json=data) assert r.status_code == HTTP_OK - r_json = json.loads(r.content) # Check that the appointment is on the watcher - assert r_json.get("status") == "dispute_responded" + assert r.json.get("status") == "dispute_responded" # Check the the sent appointment matches the received one - assert appointment.locator == r_json.get("locator") + assert appointment.locator == r.json.get("locator") assert appointment.encrypted_blob.data == Cryptographer.encrypt( - Blob(r_json.get("appointment").get("penalty_rawtx")), r_json.get("appointment").get("dispute_txid") + Blob(r.json.get("appointment").get("penalty_rawtx")), r.json.get("appointment").get("dispute_txid") ) # Delete appointment so it does not mess up with future tests - appointments.pop() - uuids = api.watcher.responder.tx_tracker_map.pop(r_json.get("appointment").get("penalty_txid")) + uuids = api.watcher.responder.tx_tracker_map.pop(r.json.get("appointment").get("penalty_txid")) api.watcher.responder.db_manager.delete_responder_tracker(uuids[0]) -def test_get_all_appointments_watcher(): - r = requests.get(url=get_all_appointment_endpoint) +def test_get_all_appointments_watcher(client): + r = client.get(get_all_appointment_endpoint) assert r.status_code == HTTP_OK - received_appointments = json.loads(r.content) + received_appointments = r.json # Make sure there all the locators re in the watcher watcher_locators = [v["locator"] for k, v in received_appointments["watcher_appointments"].items()] - local_locators = [appointment["locator"] for appointment in appointments] + local_locators = [appointment["locator"] for uuid, appointment in appointments.items()] assert set(watcher_locators) == set(local_locators) assert len(received_appointments["responder_trackers"]) == 0 -def test_get_all_appointments_responder(): +def test_get_all_appointments_responder(api, client): # Trigger all disputes - locators = [appointment["locator"] for appointment in appointments] + local_locators = [appointment.get("locator") for uuids, appointment in appointments.items()] for locator, dispute_tx in locator_dispute_tx_map.items(): - if locator in locators: + if locator in local_locators: bitcoin_cli(bitcoind_connect_params).sendrawtransaction(dispute_tx) # Confirm transactions generate_blocks(6) # Get all appointments - r = requests.get(url=get_all_appointment_endpoint) - received_appointments = json.loads(r.content) + r = client.get(get_all_appointment_endpoint) + received_appointments = r.json # Make sure there is not pending locator in the watcher responder_trackers = [v["locator"] for k, v in received_appointments["responder_trackers"].items()] - local_locators = [appointment["locator"] for appointment in appointments] assert set(responder_trackers) == set(local_locators) assert len(received_appointments["watcher_appointments"]) == 0 -def test_add_too_many_appointment(api): +# UPDATE TEST MUST BE AFTER get_all_appointments TESTS: +# This tests send data to the Watcher and Responder that may not be passed along, so it's easier to have it here and +# not keep track of what's being sent +def test_add_appointment_update_same_size(api, client, appointment): + # Update an appointment by one of the same size and check that no additional slots are filled + api.gatekeeper.registered_users[compressed_client_pk] = 1 + + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + # # Since we will replace the appointment, we won't added to appointments + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) + assert r.status_code == HTTP_OK and r.json.get("available_slots") == 0 + + # The user has no additional slots, but it should be able to update + # Let's just reverse the encrypted blob for example + appointment.encrypted_blob.data = appointment.encrypted_blob.data[::-1] + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) + assert r.status_code == HTTP_OK and r.json.get("available_slots") == 0 + + +def test_add_appointment_update_bigger(api, client, appointment): + # Update an appointment by one bigger, and check additional slots are filled + api.gatekeeper.registered_users[compressed_client_pk] = 2 + + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) + assert r.status_code == HTTP_OK and r.json.get("available_slots") == 1 + + # The user has one slot, so it should be able to update as long as it only takes 1 additional slot + appointment.encrypted_blob.data = TWO_SLOTS_BLOTS + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) + assert r.status_code == HTTP_OK and r.json.get("available_slots") == 0 + + # Check that it'll fail if no enough slots are available + # Double the size from before + appointment.encrypted_blob.data = TWO_SLOTS_BLOTS + TWO_SLOTS_BLOTS + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) + assert r.status_code == HTTP_BAD_REQUEST + + +def test_add_appointment_update_smaller(api, client, appointment): + # Update an appointment by one bigger, and check slots are freed + api.gatekeeper.registered_users[compressed_client_pk] = 2 + + # This should take 2 slots + appointment.encrypted_blob.data = TWO_SLOTS_BLOTS + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) + assert r.status_code == HTTP_OK and r.json.get("available_slots") == 0 + + # Let's update with one just small enough + appointment.encrypted_blob.data = "A" * (ENCRYPTED_BLOB_MAX_SIZE_HEX - 2) + appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) + assert r.status_code == HTTP_OK and r.json.get("available_slots") == 1 + + +def test_add_too_many_appointment(api, client): # Give slots to the user - api.gatekeeper.registered_users[client_pk_hex] = 200 + api.gatekeeper.registered_users[compressed_client_pk] = 200 free_appointment_slots = MAX_APPOINTMENTS - len(api.watcher.appointments) @@ -460,7 +495,9 @@ def test_add_too_many_appointment(api): locator_dispute_tx_map[appointment.locator] = dispute_tx appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment({"appointment": appointment.to_dict(), "signature": appointment_signature}) + r = add_appointment( + client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk + ) if i < free_appointment_slots: assert r.status_code == HTTP_OK From f8cc0996969d03dae659d0ae7c4011e92449aaf3 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 22:20:01 +0200 Subject: [PATCH 38/62] Adds register command to cli, modifies get_appointment to work with POST, refactors cli to improve modularity and cleaning the code - Adds register command - Modifies get_appointment command to work with POST + adds signature - Avoids passing config dict to functions if not necessary (passing single parameters instead) - Moves key loading to main since most of the commands will need the keys now - Refactors post_appointment and process_post_appointment_request to work with any post request, so it can be reused for both register and get_appointment - Functions have been rearanged in a more logical order - Error handling has been improved --- cli/teos_cli.py | 491 ++++++++++++++++++++++++------------------------ 1 file changed, 246 insertions(+), 245 deletions(-) diff --git a/cli/teos_cli.py b/cli/teos_cli.py index a4609d3..1a20a28 100644 --- a/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -22,12 +22,151 @@ from common.appointment import Appointment from common.config_loader import ConfigLoader from common.cryptographer import Cryptographer from common.tools import setup_logging, setup_data_folder -from common.tools import check_sha256_hex_format, check_locator_format, compute_locator +from common.tools import check_sha256_hex_format, check_locator_format, compute_locator, check_compressed_pk_format logger = Logger(actor="Client", log_name_prefix=LOG_PREFIX) common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX) +def register(compressed_pk, teos_url): + if not check_compressed_pk_format(compressed_pk): + logger.error("The cli public key is not valid") + return False + + # Send request to the server. + register_endpoint = "{}/register".format(teos_url) + data = {"public_key": compressed_pk} + + logger.info("Registering in the Eye of Satoshi") + server_response = post_request(data, register_endpoint) + if server_response: + response_json = process_post_response(server_response) + return response_json + + +def add_appointment(appointment_data, cli_sk, teos_pk, teos_url, appointments_folder_path): + """ + Manages the add_appointment command, from argument parsing, trough sending the appointment to the tower, until + saving the appointment receipt. + + The life cycle of the function is as follows: + - Check that the given commitment_txid is correct (proper format and not missing) + - Check that the transaction is correct (not missing) + - Create the appointment locator and encrypted blob from the commitment_txid and the penalty_tx + - Sign the appointment + - Send the appointment to the tower + - Wait for the response + - Check the tower's response and signature + - Store the receipt (appointment + signature) on disk + + If any of the above-mentioned steps fails, the method returns false, otherwise it returns true. + + Args: + appointment_data (:obj:`dict`): a dictionary containing the appointment data. + cli_sk (:obj:`PrivateKey`): the client's private key. + teos_pk (:obj:`PublicKey`): the tower's public key. + teos_url (:obj:`str`): the teos base url. + appointments_folder_path (:obj:`str`): the path to the appointments folder. + + + Returns: + :obj:`bool`: True if the appointment is accepted by the tower and the receipt is properly stored, false if any + error occurs during the process. + """ + + if appointment_data is None: + logger.error("The provided appointment JSON is empty") + return False + + if not check_sha256_hex_format(appointment_data.get("tx_id")): + logger.error("The provided txid is not valid") + return False + + tx_id = appointment_data.get("tx_id") + tx = appointment_data.get("tx") + + if None not in [tx_id, tx]: + appointment_data["locator"] = compute_locator(tx_id) + appointment_data["encrypted_blob"] = Cryptographer.encrypt(Blob(tx), tx_id) + + else: + logger.error("Appointment data is missing some fields") + return False + + appointment = Appointment.from_dict(appointment_data) + signature = Cryptographer.sign(appointment.serialize(), cli_sk) + + if not (appointment and signature): + return False + + data = {"appointment": appointment.to_dict(), "signature": signature} + + # Send appointment to the server. + add_appointment_endpoint = "{}/add_appointment".format(teos_url) + logger.info("Sending appointment to the Eye of Satoshi") + server_response = post_request(data, add_appointment_endpoint) + if server_response is None: + return False + + response_json = process_post_response(server_response) + + if response_json is None: + return False + + signature = response_json.get("signature") + # Check that the server signed the appointment as it should. + if signature is None: + logger.error("The response does not contain the signature of the appointment") + return False + + rpk = Cryptographer.recover_pk(appointment.serialize(), signature) + if not Cryptographer.verify_rpk(teos_pk, rpk): + logger.error("The returned appointment's signature is invalid") + return False + + logger.info("Appointment accepted and signed by the Eye of Satoshi") + logger.info("Remaining slots: {}".format(response_json.get("available_slots"))) + + # All good, store appointment and signature + return save_appointment_receipt(appointment.to_dict(), signature, appointments_folder_path) + + +def get_appointment(locator, cli_sk, teos_pk, teos_url): + """ + Gets information about an appointment from the tower. + + Args: + locator (:obj:`str`): the appointment locator used to identify it. + cli_sk (:obj:`PrivateKey`): the client's private key. + teos_pk (:obj:`PublicKey`): the tower's public key. + teos_url (:obj:`str`): the teos base url. + + Returns: + :obj:`dict` or :obj:`None`: a dictionary containing thew appointment data if the locator is valid and the tower + responds. ``None`` otherwise. + """ + + # FIXME: All responses from the tower should be signed. Not using teos_pk atm. + + valid_locator = check_locator_format(locator) + + if not valid_locator: + logger.error("The provided locator is not valid", locator=locator) + return None + + message = "get appointment {}".format(locator) + signature = Cryptographer.sign(message.encode(), cli_sk) + data = {"locator": locator, "signature": signature} + + # Send request to the server. + get_appointment_endpoint = "{}/get_appointment".format(teos_url) + logger.info("Sending appointment to the Eye of Satoshi") + server_response = post_request(data, get_appointment_endpoint) + response_json = process_post_response(server_response) + + return response_json + + def load_keys(teos_pk_path, cli_sk_path, cli_pk_path): """ Loads all the keys required so sign, send, and verify the appointment. @@ -71,115 +210,77 @@ def load_keys(teos_pk_path, cli_sk_path, cli_pk_path): try: cli_pk_der = Cryptographer.load_key_file(cli_pk_path) - PublicKey(cli_pk_der) + compressed_cli_pk = Cryptographer.get_compressed_pk(PublicKey(cli_pk_der)) except ValueError: logger.error("Client public key is invalid or cannot be parsed") return None - return teos_pk, cli_sk, cli_pk_der + return teos_pk, cli_sk, compressed_cli_pk -def add_appointment(args, teos_url, config): +def post_request(data, endpoint): """ - Manages the add_appointment command, from argument parsing, trough sending the appointment to the tower, until - saving the appointment receipt. - - The life cycle of the function is as follows: - - Load the add_appointment arguments - - Check that the given commitment_txid is correct (proper format and not missing) - - Check that the transaction is correct (not missing) - - Create the appointment locator and encrypted blob from the commitment_txid and the penalty_tx - - Load the client private key and sign the appointment - - Send the appointment to the tower - - Wait for the response - - Check the tower's response and signature - - Store the receipt (appointment + signature) on disk - - If any of the above-mentioned steps fails, the method returns false, otherwise it returns true. + Sends a post request to the tower. Args: - args (:obj:`list`): a list of arguments to pass to ``parse_add_appointment_args``. Must contain a json encoded - appointment, or the file option and the path to a file containing a json encoded appointment. - teos_url (:obj:`str`): the teos base url. - config (:obj:`dict`): a config dictionary following the format of :func:`create_config_dict `. + data (:obj:`dict`): a dictionary containing the data to be posted. + endpoint (:obj:`str`): the endpoint to send the post request. Returns: - :obj:`bool`: True if the appointment is accepted by the tower and the receipt is properly stored, false if any - error occurs during the process. + :obj:`dict` or ``None``: a json-encoded dictionary with the server response if the data can be posted. + None otherwise. """ - teos_pk, cli_sk, cli_pk_der = load_keys( - config.get("TEOS_PUBLIC_KEY"), config.get("CLI_PRIVATE_KEY"), config.get("CLI_PUBLIC_KEY") - ) + try: + return requests.post(url=endpoint, json=data, timeout=5) + + except ConnectTimeout: + logger.error("Can't connect to the Eye of Satoshi's API. Connection timeout") + + except ConnectionError: + logger.error("Can't connect to the Eye of Satoshi's API. Server cannot be reached") + + except (InvalidSchema, MissingSchema, InvalidURL): + logger.error("Invalid URL. No schema, or invalid schema, found ({})".format(endpoint)) + + except requests.exceptions.Timeout: + logger.error("The request timed out") + + return None + + +def process_post_response(response): + """ + Processes the server response to an post request. + + Args: + response (:obj:`requests.models.Response`): a ``Response`` object obtained from the sent request. + + Returns: + :obj:`dict` or :obj:`None`: a dictionary containing the tower's response data if it can be properly parsed and + the response type is ``HTTP_OK``. ``None`` otherwise. + """ + + if not response: + return None try: - hex_pk_der = binascii.hexlify(cli_pk_der) + response_json = response.json() - except binascii.Error as e: - logger.error("Could not successfully encode public key as hex", error=str(e)) - return False + except (json.JSONDecodeError, AttributeError): + logger.error( + "The server returned a non-JSON response", status_code=response.status_code, reason=response.reason + ) + return None - if teos_pk is None: - return False + if response.status_code != constants.HTTP_OK: + logger.error( + "The server returned an error", status_code=response.status_code, reason=response.reason, data=response_json + ) + return None - # Get appointment data from user. - appointment_data = parse_add_appointment_args(args) - - if appointment_data is None: - logger.error("The provided appointment JSON is empty") - return False - - valid_txid = check_sha256_hex_format(appointment_data.get("tx_id")) - - if not valid_txid: - logger.error("The provided txid is not valid") - return False - - tx_id = appointment_data.get("tx_id") - tx = appointment_data.get("tx") - - if None not in [tx_id, tx]: - appointment_data["locator"] = compute_locator(tx_id) - appointment_data["encrypted_blob"] = Cryptographer.encrypt(Blob(tx), tx_id) - - else: - logger.error("Appointment data is missing some fields") - return False - - appointment = Appointment.from_dict(appointment_data) - signature = Cryptographer.sign(appointment.serialize(), cli_sk) - - if not (appointment and signature): - return False - - data = {"appointment": appointment.to_dict(), "signature": signature, "public_key": hex_pk_der.decode("utf-8")} - - # Send appointment to the server. - server_response = post_appointment(data, teos_url) - if server_response is None: - return False - - response_json = process_post_appointment_response(server_response) - - if response_json is None: - return False - - signature = response_json.get("signature") - # Check that the server signed the appointment as it should. - if signature is None: - logger.error("The response does not contain the signature of the appointment") - return False - - rpk = Cryptographer.recover_pk(appointment.serialize(), signature) - if not Cryptographer.verify_rpk(teos_pk, rpk): - logger.error("The returned appointment's signature is invalid") - return False - - logger.info("Appointment accepted and signed by the Eye of Satoshi") - - # All good, store appointment and signature - return save_appointment_receipt(appointment.to_dict(), signature, config) + return response_json def parse_add_appointment_args(args): @@ -230,88 +331,14 @@ def parse_add_appointment_args(args): return appointment_data -def post_appointment(data, teos_url): - """ - Sends appointment data to add_appointment endpoint to be processed by the tower. - - Args: - data (:obj:`dict`): a dictionary containing three fields: an appointment, the client-side signature, and the - der-encoded client public key. - teos_url (:obj:`str`): the teos base url. - - Returns: - :obj:`dict` or ``None``: a json-encoded dictionary with the server response if the data can be posted. - None otherwise. - """ - - add_appointment_endpoint = "{}/add_appointment".format(teos_url) - - logger.info("Sending appointment to the Eye of Satoshi") - - try: - return requests.post(url=add_appointment_endpoint, json=data, timeout=5) - - except ConnectTimeout: - logger.error("Can't connect to the Eye of Satoshi's API. Connection timeout") - return None - - except ConnectionError: - logger.error("Can't connect to the Eye of Satoshi's API. Server cannot be reached") - return None - - except (InvalidSchema, MissingSchema, InvalidURL): - logger.error("Invalid URL. No schema, or invalid schema, found ({})".format(add_appointment_endpoint)) - - except requests.exceptions.Timeout: - logger.error("The request timed out") - - -def process_post_appointment_response(response): - """ - Processes the server response to an add_appointment request. - - Args: - response (:obj:`requests.models.Response`): a ``Response`` object obtained from the sent request. - - Returns: - :obj:`dict` or :obj:`None`: a dictionary containing the tower's response data if it can be properly parsed and - the response type is ``HTTP_OK``. ``None`` otherwise. - """ - - try: - response_json = response.json() - - except json.JSONDecodeError: - logger.error( - "The server returned a non-JSON response", status_code=response.status_code, reason=response.reason - ) - return None - - if response.status_code != constants.HTTP_OK: - if "error" not in response_json: - logger.error( - "The server returned an error status code but no error description", status_code=response.status_code - ) - else: - error = response_json["error"] - logger.error( - "The server returned an error status code with an error description", - status_code=response.status_code, - description=error, - ) - return None - - return response_json - - -def save_appointment_receipt(appointment, signature, config): +def save_appointment_receipt(appointment, signature, appointments_folder_path): """ Saves an appointment receipt to disk. A receipt consists in an appointment and a signature from the tower. Args: appointment (:obj:`Appointment `): the appointment to be saved on disk. signature (:obj:`str`): the signature of the appointment performed by the tower. - config (:obj:`dict`): a config dictionary following the format of :func:`create_config_dict `. + appointments_folder_path (:obj:`str`): the path to the appointments folder. Returns: :obj:`bool`: True if the appointment if properly saved, false otherwise. @@ -321,13 +348,13 @@ def save_appointment_receipt(appointment, signature, config): """ # Create the appointments directory if it doesn't already exist - os.makedirs(config.get("APPOINTMENTS_FOLDER_NAME"), exist_ok=True) + os.makedirs(appointments_folder_path, exist_ok=True) timestamp = int(time.time()) locator = appointment["locator"] uuid = uuid4().hex # prevent filename collisions - filename = "{}/appointment-{}-{}-{}.json".format(config.get("APPOINTMENTS_FOLDER_NAME"), timestamp, locator, uuid) + filename = "{}/appointment-{}-{}-{}.json".format(appointments_folder_path, timestamp, locator, uuid) data = {"appointment": appointment, "signature": signature} try: @@ -341,45 +368,6 @@ def save_appointment_receipt(appointment, signature, config): return False -def get_appointment(locator, teos_url): - """ - Gets information about an appointment from the tower. - - Args: - locator (:obj:`str`): the appointment locator used to identify it. - teos_url (:obj:`str`): the teos base url. - - Returns: - :obj:`dict` or :obj:`None`: a dictionary containing thew appointment data if the locator is valid and the tower - responds. ``None`` otherwise. - """ - - get_appointment_endpoint = "{}/get_appointment".format(teos_url) - valid_locator = check_locator_format(locator) - - if not valid_locator: - logger.error("The provided locator is not valid", locator=locator) - return None - - try: - r = requests.get(url="{}?locator={}".format(get_appointment_endpoint, locator), timeout=5) - return r.json() - - except ConnectTimeout: - logger.error("Can't connect to the Eye of Satoshi's API. Connection timeout") - return None - - except ConnectionError: - logger.error("Can't connect to the Eye of Satoshi's API. Server cannot be reached") - return None - - except requests.exceptions.InvalidSchema: - logger.error("No transport protocol found. Have you missed http(s):// in the server url?") - - except requests.exceptions.Timeout: - logger.error("The request timed out") - - def main(args, command_line_conf): # Loads config and sets up the data folder and log file config_loader = ConfigLoader(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF, command_line_conf) @@ -394,57 +382,70 @@ def main(args, command_line_conf): if not teos_url.startswith("http"): teos_url = "http://" + teos_url - try: - if args: - command = args.pop(0) + keys = load_keys(config.get("TEOS_PUBLIC_KEY"), config.get("CLI_PRIVATE_KEY"), config.get("CLI_PUBLIC_KEY")) + if keys is not None: + teos_pk, cli_sk, compress_cli_pk = keys - if command in commands: - if command == "add_appointment": - add_appointment(args, teos_url, config) + try: + if args: + command = args.pop(0) - elif command == "get_appointment": - if not args: - logger.error("No arguments were given") + if command in commands: + if command == "register": + register_data = register(compress_cli_pk, teos_url) + if register_data: + print(register_data) - else: - arg_opt = args.pop(0) + if command == "add_appointment": + # Get appointment data from user. + appointment_data = parse_add_appointment_args(args) + add_appointment( + appointment_data, cli_sk, teos_pk, teos_url, config.get("APPOINTMENTS_FOLDER_NAME") + ) - if arg_opt in ["-h", "--help"]: - sys.exit(help_get_appointment()) - - appointment_data = get_appointment(arg_opt, teos_url) - if appointment_data: - print(appointment_data) - - elif command == "help": - if args: - command = args.pop(0) - - if command == "add_appointment": - sys.exit(help_add_appointment()) - - elif command == "get_appointment": - sys.exit(help_get_appointment()) + elif command == "get_appointment": + if not args: + logger.error("No arguments were given") else: - logger.error("Unknown command. Use help to check the list of available commands") + arg_opt = args.pop(0) - else: - sys.exit(show_usage()) + if arg_opt in ["-h", "--help"]: + sys.exit(help_get_appointment()) + + appointment_data = get_appointment(arg_opt, cli_sk, teos_pk, teos_url) + if appointment_data: + print(appointment_data) + + elif command == "help": + if args: + command = args.pop(0) + + if command == "add_appointment": + sys.exit(help_add_appointment()) + + elif command == "get_appointment": + sys.exit(help_get_appointment()) + + else: + logger.error("Unknown command. Use help to check the list of available commands") + + else: + sys.exit(show_usage()) + + else: + logger.error("Unknown command. Use help to check the list of available commands") else: - logger.error("Unknown command. Use help to check the list of available commands") + logger.error("No command provided. Use help to check the list of available commands") - else: - logger.error("No command provided. Use help to check the list of available commands") - - except json.JSONDecodeError: - logger.error("Non-JSON encoded appointment passed as parameter") + except json.JSONDecodeError: + logger.error("Non-JSON encoded appointment passed as parameter") if __name__ == "__main__": command_line_conf = {} - commands = ["add_appointment", "get_appointment", "help"] + commands = ["register", "add_appointment", "get_appointment", "help"] try: opts, args = getopt(argv[1:], "s:p:h", ["server", "port", "help"]) From 31a25f3f603bf90329d6349ee49ce2b691dc12d4 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Mon, 30 Mar 2020 22:24:20 +0200 Subject: [PATCH 39/62] Adds unit tests for register, and adaps existing tests to match the changes in the commands --- test/cli/unit/test_teos_cli.py | 237 +++++++++++++++++---------------- 1 file changed, 122 insertions(+), 115 deletions(-) diff --git a/test/cli/unit/test_teos_cli.py b/test/cli/unit/test_teos_cli.py index ec34714..7ac0d32 100644 --- a/test/cli/unit/test_teos_cli.py +++ b/test/cli/unit/test_teos_cli.py @@ -20,15 +20,17 @@ common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=teos config = get_config() # dummy keys for the tests -dummy_sk = PrivateKey() -dummy_pk = dummy_sk.public_key -another_sk = PrivateKey() +dummy_cli_sk = PrivateKey.from_int(1) +dummy_cli_compressed_pk = dummy_cli_sk.public_key.format(compressed=True) +dummy_teos_sk = PrivateKey.from_int(2) +dummy_teos_pk = dummy_teos_sk.public_key +another_sk = PrivateKey.from_int(3) teos_url = "http://{}:{}".format(config.get("TEOS_SERVER"), config.get("TEOS_PORT")) add_appointment_endpoint = "{}/add_appointment".format(teos_url) get_appointment_endpoint = "{}/get_appointment".format(teos_url) -dummy_appointment_request = { +dummy_appointment_data = { "tx": get_random_value_hex(192), "tx_id": get_random_value_hex(32), "start_time": 1500, @@ -37,29 +39,93 @@ dummy_appointment_request = { } # This is the format appointment turns into once it hits "add_appointment" -dummy_appointment_full = { - "locator": compute_locator(dummy_appointment_request.get("tx_id")), - "start_time": dummy_appointment_request.get("start_time"), - "end_time": dummy_appointment_request.get("end_time"), - "to_self_delay": dummy_appointment_request.get("to_self_delay"), +dummy_appointment_dict = { + "locator": compute_locator(dummy_appointment_data.get("tx_id")), + "start_time": dummy_appointment_data.get("start_time"), + "end_time": dummy_appointment_data.get("end_time"), + "to_self_delay": dummy_appointment_data.get("to_self_delay"), "encrypted_blob": Cryptographer.encrypt( - Blob(dummy_appointment_request.get("tx")), dummy_appointment_request.get("tx_id") + Blob(dummy_appointment_data.get("tx")), dummy_appointment_data.get("tx_id") ), } -dummy_appointment = Appointment.from_dict(dummy_appointment_full) +dummy_appointment = Appointment.from_dict(dummy_appointment_dict) -def load_dummy_keys(*args): - return dummy_pk, dummy_sk, dummy_pk.format(compressed=True) +def get_signature(message, sk): + return Cryptographer.sign(message, sk) -def get_dummy_signature(*args): - return Cryptographer.sign(dummy_appointment.serialize(), dummy_sk) +def test_register(): + pass -def get_bad_signature(*args): - return Cryptographer.sign(dummy_appointment.serialize(), another_sk) +# TODO: 90-add-more-add-appointment-tests +@responses.activate +def test_add_appointment(): + # Simulate a request to add_appointment for dummy_appointment, make sure that the right endpoint is requested + # and the return value is True + response = { + "locator": dummy_appointment.locator, + "signature": get_signature(dummy_appointment.serialize(), dummy_teos_sk), + "available_slots": 100, + } + responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) + result = teos_cli.add_appointment( + dummy_appointment_data, dummy_cli_sk, dummy_teos_pk, teos_url, config.get("APPOINTMENTS_FOLDER_NAME") + ) + + assert len(responses.calls) == 1 + assert responses.calls[0].request.url == add_appointment_endpoint + assert result + + +@responses.activate +def test_add_appointment_with_invalid_signature(monkeypatch): + # Simulate a request to add_appointment for dummy_appointment, but sign with a different key, + # make sure that the right endpoint is requested, but the return value is False + + response = { + "locator": dummy_appointment.to_dict()["locator"], + "signature": get_signature(dummy_appointment.serialize(), another_sk), # Sign with a bad key + "available_slots": 100, + } + + responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) + result = teos_cli.add_appointment( + dummy_appointment_data, dummy_cli_sk, dummy_teos_pk, teos_url, config.get("APPOINTMENTS_FOLDER_NAME") + ) + + assert result is False + + shutil.rmtree(config.get("APPOINTMENTS_FOLDER_NAME")) + + +@responses.activate +def test_get_appointment(): + # Response of get_appointment endpoint is an appointment with status added to it. + response = { + "locator": dummy_appointment_dict.get("locator"), + "status": "being_watch", + "appointment": dummy_appointment_dict, + } + + responses.add(responses.POST, get_appointment_endpoint, json=response, status=200) + result = teos_cli.get_appointment(dummy_appointment_dict.get("locator"), dummy_cli_sk, dummy_teos_pk, teos_url) + + assert len(responses.calls) == 1 + assert responses.calls[0].request.url == get_appointment_endpoint + assert result.get("locator") == response.get("locator") + + +@responses.activate +def test_get_appointment_err(): + locator = get_random_value_hex(16) + + # Test that get_appointment handles a connection error appropriately. + responses.add(responses.POST, get_appointment_endpoint, body=ConnectionError()) + + assert not teos_cli.get_appointment(locator, dummy_cli_sk, dummy_teos_pk, teos_url) def test_load_keys(): @@ -68,9 +134,9 @@ def test_load_keys(): public_key_file_path = "pk_test_file" empty_file_path = "empty_file" with open(private_key_file_path, "wb") as f: - f.write(dummy_sk.to_der()) + f.write(dummy_cli_sk.to_der()) with open(public_key_file_path, "wb") as f: - f.write(dummy_pk.format(compressed=True)) + f.write(dummy_cli_compressed_pk) with open(empty_file_path, "wb") as f: pass @@ -97,42 +163,44 @@ def test_load_keys(): os.remove(empty_file_path) -# TODO: 90-add-more-add-appointment-tests +# WIP: HERE @responses.activate -def test_add_appointment(monkeypatch): - - # Simulate a request to add_appointment for dummy_appointment, make sure that the right endpoint is requested - # and the return value is True - monkeypatch.setattr(teos_cli, "load_keys", load_dummy_keys) - - response = {"locator": dummy_appointment.locator, "signature": get_dummy_signature()} - responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) - result = teos_cli.add_appointment([json.dumps(dummy_appointment_request)], teos_url, config) - - assert len(responses.calls) == 1 - assert responses.calls[0].request.url == add_appointment_endpoint - assert result - - -@responses.activate -def test_add_appointment_with_invalid_signature(monkeypatch): - # Simulate a request to add_appointment for dummy_appointment, but sign with a different key, - # make sure that the right endpoint is requested, but the return value is False - - # Make sure the test uses the bad dummy signature - monkeypatch.setattr(teos_cli, "load_keys", load_dummy_keys) - +def test_post_request(): response = { "locator": dummy_appointment.to_dict()["locator"], - "signature": get_bad_signature(), # Sign with a bad key + "signature": get_signature(dummy_appointment.serialize(), dummy_teos_sk), } responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) - result = teos_cli.add_appointment([json.dumps(dummy_appointment_request)], teos_url, config) + response = teos_cli.post_request(json.dumps(dummy_appointment_data), add_appointment_endpoint) - shutil.rmtree(config.get("APPOINTMENTS_FOLDER_NAME")) + assert len(responses.calls) == 1 + assert responses.calls[0].request.url == add_appointment_endpoint + assert response - assert result is False + +@responses.activate +def test_process_post_response(): + # Let's first crete a response + response = { + "locator": dummy_appointment.to_dict()["locator"], + "signature": get_signature(dummy_appointment.serialize(), dummy_teos_sk), + } + + # A 200 OK with a correct json response should return the json of the response + responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) + r = teos_cli.post_request(json.dumps(dummy_appointment_data), add_appointment_endpoint) + assert teos_cli.process_post_response(r) == r.json() + + # If we modify the response code for a rejection (lets say 404) we should get None + responses.replace(responses.POST, add_appointment_endpoint, json=response, status=404) + r = teos_cli.post_request(json.dumps(dummy_appointment_data), add_appointment_endpoint) + assert teos_cli.process_post_response(r) is None + + # The same should happen if the response is not in json + responses.replace(responses.POST, add_appointment_endpoint, status=404) + r = teos_cli.post_request(json.dumps(dummy_appointment_data), add_appointment_endpoint) + assert teos_cli.process_post_response(r) is None def test_parse_add_appointment_args(): @@ -146,7 +214,7 @@ def test_parse_add_appointment_args(): # If file exists and has data in it, function should work. with open("appt_test_file", "w") as f: - json.dump(dummy_appointment_request, f) + json.dump(dummy_appointment_data, f) appt_data = teos_cli.parse_add_appointment_args(["-f", "appt_test_file"]) assert appt_data @@ -154,56 +222,21 @@ def test_parse_add_appointment_args(): os.remove("appt_test_file") # If appointment json is passed in, function should work. - appt_data = teos_cli.parse_add_appointment_args([json.dumps(dummy_appointment_request)]) + appt_data = teos_cli.parse_add_appointment_args([json.dumps(dummy_appointment_data)]) assert appt_data -@responses.activate -def test_post_appointment(): - response = { - "locator": dummy_appointment.to_dict()["locator"], - "signature": Cryptographer.sign(dummy_appointment.serialize(), dummy_pk), - } - - responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) - response = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_url) - - assert len(responses.calls) == 1 - assert responses.calls[0].request.url == add_appointment_endpoint - assert response - - -@responses.activate -def test_process_post_appointment_response(): - # Let's first crete a response - response = { - "locator": dummy_appointment.to_dict()["locator"], - "signature": Cryptographer.sign(dummy_appointment.serialize(), dummy_pk), - } - - # A 200 OK with a correct json response should return the json of the response - responses.add(responses.POST, add_appointment_endpoint, json=response, status=200) - r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_url) - assert teos_cli.process_post_appointment_response(r) == r.json() - - # If we modify the response code tor a rejection (lets say 404) we should get None - responses.replace(responses.POST, add_appointment_endpoint, json=response, status=404) - r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_url) - assert teos_cli.process_post_appointment_response(r) is None - - # The same should happen if the response is not in json - responses.replace(responses.POST, add_appointment_endpoint, status=404) - r = teos_cli.post_appointment(json.dumps(dummy_appointment_request), teos_url) - assert teos_cli.process_post_appointment_response(r) is None - - def test_save_appointment_receipt(monkeypatch): appointments_folder = "test_appointments_receipts" config["APPOINTMENTS_FOLDER_NAME"] = appointments_folder # The functions creates a new directory if it does not exist assert not os.path.exists(appointments_folder) - teos_cli.save_appointment_receipt(dummy_appointment.to_dict(), get_dummy_signature(), config) + teos_cli.save_appointment_receipt( + dummy_appointment.to_dict(), + get_signature(dummy_appointment.serialize(), dummy_teos_sk), + config.get("APPOINTMENTS_FOLDER_NAME"), + ) assert os.path.exists(appointments_folder) # Check that the receipt has been saved by checking the file names @@ -211,29 +244,3 @@ def test_save_appointment_receipt(monkeypatch): assert any([dummy_appointment.locator in f for f in files]) shutil.rmtree(appointments_folder) - - -@responses.activate -def test_get_appointment(): - # Response of get_appointment endpoint is an appointment with status added to it. - dummy_appointment_full["status"] = "being_watched" - response = dummy_appointment_full - - request_url = "{}?locator={}".format(get_appointment_endpoint, response.get("locator")) - responses.add(responses.GET, request_url, json=response, status=200) - result = teos_cli.get_appointment(response.get("locator"), teos_url) - - assert len(responses.calls) == 1 - assert responses.calls[0].request.url == request_url - assert result.get("locator") == response.get("locator") - - -@responses.activate -def test_get_appointment_err(): - locator = get_random_value_hex(16) - - # Test that get_appointment handles a connection error appropriately. - request_url = "{}?locator={}".format(get_appointment_endpoint, locator) - responses.add(responses.GET, request_url, body=ConnectionError()) - - assert not teos_cli.get_appointment(locator, teos_url) From 03c8ad8c878677966a36ca8de5dd0d8b651580f6 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 31 Mar 2020 11:07:01 +0200 Subject: [PATCH 40/62] Improves API testing. Mocks data moving from Watcher/Responder instead of running them in the background. Closes #77. --- test/teos/unit/test_api.py | 328 ++++++++++++++++++++----------------- 1 file changed, 176 insertions(+), 152 deletions(-) diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 423081c..f417ee6 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -1,28 +1,25 @@ import pytest +from shutil import rmtree from binascii import hexlify from teos.api import API from teos import HOST, PORT import teos.errors as errors from teos.watcher import Watcher -from teos.tools import bitcoin_cli from teos.inspector import Inspector -from teos.responder import Responder +from teos.db_manager import DBManager from teos.gatekeeper import Gatekeeper from teos.chain_monitor import ChainMonitor +from teos.responder import Responder, TransactionTracker from test.teos.unit.conftest import ( - generate_block, - generate_blocks, get_random_value_hex, generate_dummy_appointment, generate_keypair, get_config, - bitcoind_connect_params, bitcoind_feed_params, ) -from common.blob import Blob from common.cryptographer import Cryptographer, hash_160 from common.constants import ( HTTP_OK, @@ -56,20 +53,24 @@ client_sk, client_pk = generate_keypair() compressed_client_pk = hexlify(client_pk.format(compressed=True)).decode("utf-8") +@pytest.fixture() +def get_all_db_manager(): + manager = DBManager("get_all_tmp_db") + # Add last know block for the Responder in the db + + yield manager + + manager.db.close() + rmtree("get_all_tmp_db") + + @pytest.fixture(scope="module", autouse=True) def api(db_manager, carrier, block_processor, run_bitcoind): - sk, pk = generate_keypair() responder = Responder(db_manager, carrier, block_processor) watcher = Watcher(db_manager, block_processor, responder, sk.to_der(), MAX_APPOINTMENTS, config.get("EXPIRY_DELTA")) - chain_monitor = ChainMonitor( - watcher.block_queue, watcher.responder.block_queue, block_processor, bitcoind_feed_params - ) - watcher.awake() - chain_monitor.monitor_chain() - gatekeeper = Gatekeeper(config.get("DEFAULT_SLOTS")) api = API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, gatekeeper) @@ -275,145 +276,6 @@ def test_add_appointment_multiple_times_different_users(api, client, appointment assert len(set(api.watcher.locator_uuid_map[appointment.locator])) == n -def test_get_appointment_no_json(api, client, appointment): - r = client.post(add_appointment_endpoint, data="random_message") - assert r.status_code == HTTP_BAD_REQUEST - - -def test_get_appointment_json_no_inner_dict(api, client, appointment): - r = client.post(add_appointment_endpoint, json="random_message") - assert r.status_code == HTTP_BAD_REQUEST - - -def test_request_random_appointment_registered_user(client, user_sk=client_sk): - locator = get_random_value_hex(LOCATOR_LEN_BYTES) - message = "get appointment {}".format(locator) - signature = Cryptographer.sign(message.encode("utf-8"), user_sk) - - data = {"locator": locator, "signature": signature} - r = client.post(get_appointment_endpoint, json=data) - - # We should get a 404 not found since we are using a made up locator - received_appointment = r.json - assert r.status_code == HTTP_NOT_FOUND - assert received_appointment.get("status") == "not_found" - - -def test_request_appointment_not_registered_user(client): - # Not registered users have no associated appointments, so this should fail - tmp_sk, tmp_pk = generate_keypair() - - # The tower is designed so a not found appointment and a request from a non-registered user return the same error to - # prevent proving. - test_request_random_appointment_registered_user(client, tmp_sk) - - -def test_request_appointment_in_watcher(api, client, appointment): - # Give slots to the user - api.gatekeeper.registered_users[compressed_client_pk] = 1 - - # Add an appointment - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = add_appointment( - client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk - ) - assert r.status_code == HTTP_OK - - message = "get appointment {}".format(appointment.locator) - signature = Cryptographer.sign(message.encode("utf-8"), client_sk) - data = {"locator": appointment.locator, "signature": signature} - - # Next we can request it - r = client.post(get_appointment_endpoint, json=data) - assert r.status_code == HTTP_OK - - # Check that the appointment is on the watcher - assert r.json.get("status") == "being_watched" - - # Check the the sent appointment matches the received one - assert r.json.get("locator") == appointment.locator - assert appointment.to_dict() == r.json.get("appointment") - - -def test_request_appointment_in_responder(api, client, appointment): - # Give slots to the user - api.gatekeeper.registered_users[compressed_client_pk] = 1 - - # Let's do something similar to what we did with the watcher but now we'll send the dispute tx to the network - dispute_tx = locator_dispute_tx_map.pop(appointment.locator) - bitcoin_cli(bitcoind_connect_params).sendrawtransaction(dispute_tx) - - # Add an appointment (avoid calling add_appointment to not add this one to the sent appointments list) - appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) - r = client.post( - add_appointment_endpoint, json={"appointment": appointment.to_dict(), "signature": appointment_signature} - ) - assert r.status_code == HTTP_OK - - # Generate a block to trigger the watcher - generate_block() - - # Request back the data - message = "get appointment {}".format(appointment.locator) - signature = Cryptographer.sign(message.encode("utf-8"), client_sk) - data = {"locator": appointment.locator, "signature": signature} - - # Next we can request it - r = client.post(get_appointment_endpoint, json=data) - assert r.status_code == HTTP_OK - - # Check that the appointment is on the watcher - assert r.json.get("status") == "dispute_responded" - - # Check the the sent appointment matches the received one - assert appointment.locator == r.json.get("locator") - assert appointment.encrypted_blob.data == Cryptographer.encrypt( - Blob(r.json.get("appointment").get("penalty_rawtx")), r.json.get("appointment").get("dispute_txid") - ) - - # Delete appointment so it does not mess up with future tests - uuids = api.watcher.responder.tx_tracker_map.pop(r.json.get("appointment").get("penalty_txid")) - api.watcher.responder.db_manager.delete_responder_tracker(uuids[0]) - - -def test_get_all_appointments_watcher(client): - r = client.get(get_all_appointment_endpoint) - assert r.status_code == HTTP_OK - - received_appointments = r.json - - # Make sure there all the locators re in the watcher - watcher_locators = [v["locator"] for k, v in received_appointments["watcher_appointments"].items()] - local_locators = [appointment["locator"] for uuid, appointment in appointments.items()] - - assert set(watcher_locators) == set(local_locators) - assert len(received_appointments["responder_trackers"]) == 0 - - -def test_get_all_appointments_responder(api, client): - # Trigger all disputes - local_locators = [appointment.get("locator") for uuids, appointment in appointments.items()] - for locator, dispute_tx in locator_dispute_tx_map.items(): - if locator in local_locators: - bitcoin_cli(bitcoind_connect_params).sendrawtransaction(dispute_tx) - - # Confirm transactions - generate_blocks(6) - - # Get all appointments - r = client.get(get_all_appointment_endpoint) - received_appointments = r.json - - # Make sure there is not pending locator in the watcher - responder_trackers = [v["locator"] for k, v in received_appointments["responder_trackers"].items()] - - assert set(responder_trackers) == set(local_locators) - assert len(received_appointments["watcher_appointments"]) == 0 - - -# UPDATE TEST MUST BE AFTER get_all_appointments TESTS: -# This tests send data to the Watcher and Responder that may not be passed along, so it's easier to have it here and -# not keep track of what's being sent def test_add_appointment_update_same_size(api, client, appointment): # Update an appointment by one of the same size and check that no additional slots are filled api.gatekeeper.registered_users[compressed_client_pk] = 1 @@ -503,3 +365,165 @@ def test_add_too_many_appointment(api, client): assert r.status_code == HTTP_OK else: assert r.status_code == HTTP_SERVICE_UNAVAILABLE + + +def test_get_appointment_no_json(api, client, appointment): + r = client.post(add_appointment_endpoint, data="random_message") + assert r.status_code == HTTP_BAD_REQUEST + + +def test_get_appointment_json_no_inner_dict(api, client, appointment): + r = client.post(add_appointment_endpoint, json="random_message") + assert r.status_code == HTTP_BAD_REQUEST + + +def test_request_random_appointment_registered_user(client, user_sk=client_sk): + locator = get_random_value_hex(LOCATOR_LEN_BYTES) + message = "get appointment {}".format(locator) + signature = Cryptographer.sign(message.encode("utf-8"), user_sk) + + data = {"locator": locator, "signature": signature} + r = client.post(get_appointment_endpoint, json=data) + + # We should get a 404 not found since we are using a made up locator + received_appointment = r.json + assert r.status_code == HTTP_NOT_FOUND + assert received_appointment.get("status") == "not_found" + + +def test_request_appointment_not_registered_user(client): + # Not registered users have no associated appointments, so this should fail + tmp_sk, tmp_pk = generate_keypair() + + # The tower is designed so a not found appointment and a request from a non-registered user return the same error to + # prevent proving. + test_request_random_appointment_registered_user(client, tmp_sk) + + +def test_request_appointment_in_watcher(api, client, appointment): + # Mock the appointment in the Watcher + uuid = hash_160("{}{}".format(appointment.locator, compressed_client_pk)) + api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + + # Next we can request it + message = "get appointment {}".format(appointment.locator) + signature = Cryptographer.sign(message.encode("utf-8"), client_sk) + data = {"locator": appointment.locator, "signature": signature} + r = client.post(get_appointment_endpoint, json=data) + assert r.status_code == HTTP_OK + + # Check that the appointment is on the watcher + assert r.json.get("status") == "being_watched" + + # Check the the sent appointment matches the received one + assert r.json.get("locator") == appointment.locator + assert appointment.to_dict() == r.json.get("appointment") + + +def test_request_appointment_in_responder(api, client, appointment): + # Mock the appointment in the Responder + tracker_data = { + "locator": appointment.locator, + "dispute_txid": get_random_value_hex(32), + "penalty_txid": get_random_value_hex(32), + "penalty_rawtx": get_random_value_hex(250), + "appointment_end": appointment.end_time, + } + tx_tracker = TransactionTracker.from_dict(tracker_data) + + uuid = hash_160("{}{}".format(appointment.locator, compressed_client_pk)) + api.watcher.db_manager.create_triggered_appointment_flag(uuid) + api.watcher.responder.db_manager.store_responder_tracker(uuid, tx_tracker.to_json()) + + # Request back the data + message = "get appointment {}".format(appointment.locator) + signature = Cryptographer.sign(message.encode("utf-8"), client_sk) + data = {"locator": appointment.locator, "signature": signature} + + # Next we can request it + r = client.post(get_appointment_endpoint, json=data) + assert r.status_code == HTTP_OK + + # Check that the appointment is on the watcher + assert r.json.get("status") == "dispute_responded" + + # Check the the sent appointment matches the received one + assert tx_tracker.locator == r.json.get("locator") + assert tx_tracker.dispute_txid == r.json.get("appointment").get("dispute_txid") + assert tx_tracker.penalty_txid == r.json.get("appointment").get("penalty_txid") + assert tx_tracker.penalty_rawtx == r.json.get("appointment").get("penalty_rawtx") + assert tx_tracker.appointment_end == r.json.get("appointment").get("appointment_end") + + +def test_get_all_appointments_watcher(api, client, get_all_db_manager, appointment): + # Let's reset the dbs so we can test this clean + api.watcher.db_manager = get_all_db_manager + api.watcher.responder.db_manager = get_all_db_manager + + # Check that they are wiped clean + r = client.get(get_all_appointment_endpoint) + assert r.status_code == HTTP_OK + assert len(r.json.get("watcher_appointments")) == 0 and len(r.json.get("responder_trackers")) == 0 + + # Add some appointments to the Watcher db + non_triggered_appointments = {} + for _ in range(10): + uuid = get_random_value_hex(16) + appointment.locator = get_random_value_hex(16) + non_triggered_appointments[uuid] = appointment.to_dict() + api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + + triggered_appointments = {} + for _ in range(10): + uuid = get_random_value_hex(16) + appointment.locator = get_random_value_hex(16) + triggered_appointments[uuid] = appointment.to_dict() + api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + api.watcher.db_manager.create_triggered_appointment_flag(uuid) + + # We should only get check the non-triggered appointments + r = client.get(get_all_appointment_endpoint) + assert r.status_code == HTTP_OK + + watcher_locators = [v["locator"] for k, v in r.json["watcher_appointments"].items()] + local_locators = [appointment["locator"] for uuid, appointment in non_triggered_appointments.items()] + + assert set(watcher_locators) == set(local_locators) + assert len(r.json["responder_trackers"]) == 0 + + +def test_get_all_appointments_responder(api, client, get_all_db_manager): + # Let's reset the dbs so we can test this clean + api.watcher.db_manager = get_all_db_manager + api.watcher.responder.db_manager = get_all_db_manager + + # Check that they are wiped clean + r = client.get(get_all_appointment_endpoint) + assert r.status_code == HTTP_OK + assert len(r.json.get("watcher_appointments")) == 0 and len(r.json.get("responder_trackers")) == 0 + + # Add some trackers to the Responder db + tx_trackers = {} + for _ in range(10): + uuid = get_random_value_hex(16) + tracker_data = { + "locator": get_random_value_hex(16), + "dispute_txid": get_random_value_hex(32), + "penalty_txid": get_random_value_hex(32), + "penalty_rawtx": get_random_value_hex(250), + "appointment_end": 20, + } + tracker = TransactionTracker.from_dict(tracker_data) + tx_trackers[uuid] = tracker.to_dict() + api.watcher.responder.db_manager.store_responder_tracker(uuid, tracker.to_json()) + api.watcher.db_manager.create_triggered_appointment_flag(uuid) + + # Get all appointments + r = client.get(get_all_appointment_endpoint) + + # Make sure there is not pending locator in the watcher + responder_trackers = [v["locator"] for k, v in r.json["responder_trackers"].items()] + local_locators = [tracker["locator"] for uuid, tracker in tx_trackers.items()] + + assert set(responder_trackers) == set(local_locators) + assert len(r.json["watcher_appointments"]) == 0 From b56123055da0af49ae73785d0533d7ba4d0b21ab Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 31 Mar 2020 11:57:25 +0200 Subject: [PATCH 41/62] Adds flake8 and fixes style issues --- CONTRIBUTING.md | 9 + cli/teos_cli.py | 1 - common/appointment.py | 8 +- common/cryptographer.py | 4 +- requirements-dev.txt | 2 + teos/__init__.py | 1 - teos/api.py | 2 +- teos/builder.py | 5 +- teos/carrier.py | 22 +- teos/cleaner.py | 5 +- teos/db_manager.py | 3 +- teos/help.py | 3 +- teos/responder.py | 2 +- teos/rpc_errors.py | 58 +++-- teos/watcher.py | 6 +- test/common/unit/test_tools.py | 8 +- test/teos/e2e/test_basic_e2e.py | 302 ------------------------- test/teos/unit/test_api.py | 9 +- test/teos/unit/test_block_processor.py | 4 +- test/teos/unit/test_chain_monitor.py | 6 +- test/teos/unit/test_inspector.py | 44 ++-- 21 files changed, 103 insertions(+), 401 deletions(-) delete mode 100644 test/teos/e2e/test_basic_e2e.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7490df1..2781ab7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,6 +8,15 @@ We use [black](https://github.com/psf/black) as our base code formatter with a l ```bash black --line-length=120 {source_file_or_directory} ``` + +In additon, we use [flake8](https://flake8.pycqa.org/en/latest/) to detect style issues with the code: + +```bash +flake8 --max-line-length=120 {source_file_or_directory} +``` + + Not all outputs from flake8 are mandatory. For instance, splitting **bullet points in docstrings (E501)** will cause issues when generating the docuementation, so we will leave that longer than the line lenght limit . Another example are **whitespaces before colons in inline fors (E203)**. `black` places them in that way, so we'll leave them like that. + On top of that, there are a few rules to also have in mind. ### Code Spacing diff --git a/cli/teos_cli.py b/cli/teos_cli.py index 1a20a28..994185a 100644 --- a/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -3,7 +3,6 @@ import sys import time import json import requests -import binascii from sys import argv from uuid import uuid4 from coincurve import PublicKey diff --git a/common/appointment.py b/common/appointment.py index 2e98649..41c7e57 100644 --- a/common/appointment.py +++ b/common/appointment.py @@ -10,12 +10,12 @@ class Appointment: The :class:`Appointment` contains the information regarding an appointment between a client and the Watchtower. Args: - locator (:mod:`str`): A 16-byte hex-encoded value used by the tower to detect channel breaches. It serves as a trigger - for the tower to decrypt and broadcast the penalty transaction. + locator (:mod:`str`): A 16-byte hex-encoded value used by the tower to detect channel breaches. It serves as a + trigger for the tower to decrypt and broadcast the penalty transaction. start_time (:mod:`int`): The block height where the tower is hired to start watching for breaches. end_time (:mod:`int`): The block height where the tower will stop watching for breaches. - to_self_delay (:mod:`int`): The ``to_self_delay`` encoded in the ``csv`` of the ``htlc`` that this appointment is - covering. + to_self_delay (:mod:`int`): The ``to_self_delay`` encoded in the ``csv`` of the ``htlc`` that this appointment + is covering. encrypted_blob (:obj:`EncryptedBlob `): An ``EncryptedBlob`` object containing an encrypted penalty transaction. The tower will decrypt it and broadcast the penalty transaction upon seeing a breach on the blockchain. diff --git a/common/cryptographer.py b/common/cryptographer.py index fbc3cf8..ebe15ac 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -159,8 +159,8 @@ class Cryptographer: ``SHA256(secret)`` is used as ``key``, and ``0 (12-byte)`` as ``iv``. Args: - encrypted_blob(:mod:`EncryptedBlob `): an ``EncryptedBlob`` potentially - containing a penalty transaction. + encrypted_blob(:mod:`EncryptedBlob `): an ``EncryptedBlob`` + potentially containing a penalty transaction. secret (:mod:`str`): a value to used to derive the decryption key. Should be the dispute txid. Returns: diff --git a/requirements-dev.txt b/requirements-dev.txt index 8a51da8..9d5c6ed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,6 @@ pytest black +flake8 responses bitcoind_mock===0.0.4 + diff --git a/teos/__init__.py b/teos/__init__.py index 780e3cc..ef94053 100644 --- a/teos/__init__.py +++ b/teos/__init__.py @@ -1,5 +1,4 @@ import os -from teos.utils.auth_proxy import AuthServiceProxy HOST = "localhost" PORT = 9814 diff --git a/teos/api.py b/teos/api.py index 17cd45b..ec65329 100644 --- a/teos/api.py +++ b/teos/api.py @@ -1,6 +1,6 @@ import os import logging -from math import ceil, floor +from math import ceil from flask import Flask, request, abort, jsonify import teos.errors as errors diff --git a/teos/builder.py b/teos/builder.py index 5a4dc42..379bac8 100644 --- a/teos/builder.py +++ b/teos/builder.py @@ -94,8 +94,9 @@ class Builder: @staticmethod def update_states(watcher, missed_blocks_watcher, missed_blocks_responder): """ - Updates the states of both the :mod:`Watcher ` and the :mod:`Responder `. - If both have pending blocks to process they need to be updates at the same time, block by block. + Updates the states of both the :mod:`Watcher ` and the + :mod:`Responder `. If both have pending blocks to process they need to be updates at + the same time, block by block. If only one instance has to be updated, ``populate_block_queue`` should be used. diff --git a/teos/carrier.py b/teos/carrier.py index 0537c63..587afa4 100644 --- a/teos/carrier.py +++ b/teos/carrier.py @@ -1,7 +1,7 @@ from teos import LOG_PREFIX -from teos.rpc_errors import * from common.logger import Logger from teos.tools import bitcoin_cli +import teos.rpc_errors as rpc_errors from teos.utils.auth_proxy import JSONRPCException from teos.errors import UNKNOWN_JSON_RPC_EXCEPTION, RPC_TX_REORGED_AFTER_BROADCAST @@ -81,17 +81,17 @@ class Carrier: except JSONRPCException as e: errno = e.error.get("code") # Since we're pushing a raw transaction to the network we can face several rejections - if errno == RPC_VERIFY_REJECTED: + if errno == rpc_errors.RPC_VERIFY_REJECTED: # DISCUSS: 37-transaction-rejection - receipt = Receipt(delivered=False, reason=RPC_VERIFY_REJECTED) + receipt = Receipt(delivered=False, reason=rpc_errors.RPC_VERIFY_REJECTED) logger.error("Transaction couldn't be broadcast", error=e.error) - elif errno == RPC_VERIFY_ERROR: + elif errno == rpc_errors.RPC_VERIFY_ERROR: # DISCUSS: 37-transaction-rejection - receipt = Receipt(delivered=False, reason=RPC_VERIFY_ERROR) + receipt = Receipt(delivered=False, reason=rpc_errors.RPC_VERIFY_ERROR) logger.error("Transaction couldn't be broadcast", error=e.error) - elif errno == RPC_VERIFY_ALREADY_IN_CHAIN: + elif errno == rpc_errors.RPC_VERIFY_ALREADY_IN_CHAIN: logger.info("Transaction is already in the blockchain. Getting confirmation count", txid=txid) # If the transaction is already in the chain, we get the number of confirmations and watch the tracker @@ -100,7 +100,9 @@ class Carrier: if tx_info is not None: confirmations = int(tx_info.get("confirmations")) - receipt = Receipt(delivered=True, confirmations=confirmations, reason=RPC_VERIFY_ALREADY_IN_CHAIN) + receipt = Receipt( + delivered=True, confirmations=confirmations, reason=rpc_errors.RPC_VERIFY_ALREADY_IN_CHAIN + ) else: # There's a really unlikely edge case where a transaction can be reorged between receiving the @@ -108,12 +110,12 @@ class Carrier: # mempool, which again is really unlikely. receipt = Receipt(delivered=False, reason=RPC_TX_REORGED_AFTER_BROADCAST) - elif errno == RPC_DESERIALIZATION_ERROR: + elif errno == rpc_errors.RPC_DESERIALIZATION_ERROR: # Adding this here just for completeness. We should never end up here. The Carrier only sends txs # handed by the Responder, who receives them from the Watcher, who checks that the tx can be properly # deserialized logger.info("Transaction cannot be deserialized".format(txid)) - receipt = Receipt(delivered=False, reason=RPC_DESERIALIZATION_ERROR) + receipt = Receipt(delivered=False, reason=rpc_errors.RPC_DESERIALIZATION_ERROR) else: # If something else happens (unlikely but possible) log it so we can treat it in future releases @@ -145,7 +147,7 @@ class Carrier: # While it's quite unlikely, the transaction that was already in the blockchain could have been # reorged while we were querying bitcoind to get the confirmation count. In such a case we just # restart the tracker - if e.error.get("code") == RPC_INVALID_ADDRESS_OR_KEY: + if e.error.get("code") == rpc_errors.RPC_INVALID_ADDRESS_OR_KEY: logger.info("Transaction not found in mempool nor blockchain", txid=txid) else: diff --git a/teos/cleaner.py b/teos/cleaner.py index 539b603..25bd988 100644 --- a/teos/cleaner.py +++ b/teos/cleaner.py @@ -123,8 +123,9 @@ class Cleaner: """ Deletes a completed appointment from memory (:obj:`Watcher `) and disk. - Currently, an appointment is only completed if it cannot make it to the (:obj:`Responder `), - otherwise, it will be flagged as triggered and removed once the tracker is completed. + Currently, an appointment is only completed if it cannot make it to the + (:obj:`Responder `), otherwise, it will be flagged as triggered and removed once the + tracker is completed. Args: completed_appointments (:obj:`list`): a list of appointments to be deleted. diff --git a/teos/db_manager.py b/teos/db_manager.py index 92a92da..8b743cd 100644 --- a/teos/db_manager.py +++ b/teos/db_manager.py @@ -172,6 +172,7 @@ class DBManager: def load_watcher_appointments(self, include_triggered=False): """ Loads all the appointments from the database (all entries with the ``WATCHER_PREFIX`` prefix). + Args: include_triggered (:obj:`bool`): Whether to include the appointments flagged as triggered or not. ``False`` by default. @@ -289,7 +290,7 @@ class DBManager: current_locator_map = self.load_locator_map(locator) - if set(locator_map).issubset(current_locator_map) and len(locator_map) is not 0: + if set(locator_map).issubset(current_locator_map) and len(locator_map) != 0: key = (LOCATOR_MAP_PREFIX + locator).encode("utf-8") self.db.put(key, json.dumps(locator_map).encode("utf-8")) diff --git a/teos/help.py b/teos/help.py index a973b0b..e003732 100644 --- a/teos/help.py +++ b/teos/help.py @@ -3,7 +3,8 @@ def show_usage(): "USAGE: " "\n\tpython teosd.py [global options]" "\n\nGLOBAL OPTIONS:" - "\n\t--btcnetwork \t\tNetwork bitcoind is connected to. Either mainnet, testnet or regtest. Defaults to 'mainnet' (modifiable in conf file)." + "\n\t--btcnetwork \t\tNetwork bitcoind is connected to. Either mainnet, testnet or regtest. Defaults to " + "'mainnet' (modifiable in conf file)." "\n\t--btcrpcuser \t\tbitcoind rpcuser. Defaults to 'user' (modifiable in conf file)." "\n\t--btcrpcpassword \tbitcoind rpcpassword. Defaults to 'passwd' (modifiable in conf file)." "\n\t--btcrpcconnect \tbitcoind rpcconnect. Defaults to 'localhost' (modifiable in conf file)." diff --git a/teos/responder.py b/teos/responder.py index 526850f..4ecdaa6 100644 --- a/teos/responder.py +++ b/teos/responder.py @@ -303,7 +303,7 @@ class Responder: # Clear the receipts issued in this block self.carrier.issued_receipts = {} - if len(self.trackers) is 0: + if len(self.trackers) != 0: logger.info("No more pending trackers") # Register the last processed block for the responder diff --git a/teos/rpc_errors.py b/teos/rpc_errors.py index 39151df..aeb2d66 100644 --- a/teos/rpc_errors.py +++ b/teos/rpc_errors.py @@ -3,16 +3,16 @@ # General application defined errors RPC_MISC_ERROR = -1 # std::exception thrown in command handling RPC_TYPE_ERROR = -3 # Unexpected type was passed as parameter -RPC_INVALID_ADDRESS_OR_KEY = -5 # Invalid address or key -RPC_OUT_OF_MEMORY = -7 # Ran out of memory during operation -RPC_INVALID_PARAMETER = -8 # Invalid missing or duplicate parameter -RPC_DATABASE_ERROR = -20 # Database error -RPC_DESERIALIZATION_ERROR = -22 # Error parsing or validating structure in raw format -RPC_VERIFY_ERROR = -25 # General error during transaction or block submission -RPC_VERIFY_REJECTED = -26 # Transaction or block was rejected by network rules -RPC_VERIFY_ALREADY_IN_CHAIN = -27 # Transaction already in chain -RPC_IN_WARMUP = -28 # Client still warming up -RPC_METHOD_DEPRECATED = -32 # RPC method is deprecated +RPC_INVALID_ADDRESS_OR_KEY = -5 # Invalid address or key +RPC_OUT_OF_MEMORY = -7 # Ran out of memory during operation +RPC_INVALID_PARAMETER = -8 # Invalid missing or duplicate parameter +RPC_DATABASE_ERROR = -20 # Database error +RPC_DESERIALIZATION_ERROR = -22 # Error parsing or validating structure in raw format +RPC_VERIFY_ERROR = -25 # General error during transaction or block submission +RPC_VERIFY_REJECTED = -26 # Transaction or block was rejected by network rules +RPC_VERIFY_ALREADY_IN_CHAIN = -27 # Transaction already in chain +RPC_IN_WARMUP = -28 # Client still warming up +RPC_METHOD_DEPRECATED = -32 # RPC method is deprecated # Aliases for backward compatibility RPC_TRANSACTION_ERROR = RPC_VERIFY_ERROR @@ -20,25 +20,23 @@ RPC_TRANSACTION_REJECTED = RPC_VERIFY_REJECTED RPC_TRANSACTION_ALREADY_IN_CHAIN = RPC_VERIFY_ALREADY_IN_CHAIN # P2P client errors -RPC_CLIENT_NOT_CONNECTED = -9 # Bitcoin is not connected -RPC_CLIENT_IN_INITIAL_DOWNLOAD = -10 # Still downloading initial blocks -RPC_CLIENT_NODE_ALREADY_ADDED = -23 # Node is already added -RPC_CLIENT_NODE_NOT_ADDED = -24 # Node has not been added before -RPC_CLIENT_NODE_NOT_CONNECTED = -29 # Node to disconnect not found in connected nodes -RPC_CLIENT_INVALID_IP_OR_SUBNET = -30 # Invalid IP/Subnet -RPC_CLIENT_P2P_DISABLED = -31 # No valid connection manager instance found +RPC_CLIENT_NOT_CONNECTED = -9 # Bitcoin is not connected +RPC_CLIENT_IN_INITIAL_DOWNLOAD = -10 # Still downloading initial blocks +RPC_CLIENT_NODE_ALREADY_ADDED = -23 # Node is already added +RPC_CLIENT_NODE_NOT_ADDED = -24 # Node has not been added before +RPC_CLIENT_NODE_NOT_CONNECTED = -29 # Node to disconnect not found in connected nodes +RPC_CLIENT_INVALID_IP_OR_SUBNET = -30 # Invalid IP/Subnet +RPC_CLIENT_P2P_DISABLED = -31 # No valid connection manager instance found # Wallet errors -RPC_WALLET_ERROR = -4 # Unspecified problem with wallet (key not found etc.) -RPC_WALLET_INSUFFICIENT_FUNDS = -6 # Not enough funds in wallet or account -RPC_WALLET_INVALID_LABEL_NAME = -11 # Invalid label name -RPC_WALLET_KEYPOOL_RAN_OUT = -12 # Keypool ran out call keypoolrefill first -RPC_WALLET_UNLOCK_NEEDED = -13 # Enter the wallet passphrase with walletpassphrase first -RPC_WALLET_PASSPHRASE_INCORRECT = -14 # The wallet passphrase entered was incorrect -RPC_WALLET_WRONG_ENC_STATE = ( - -15 -) # Command given in wrong wallet encryption state (encrypting an encrypted wallet etc.) -RPC_WALLET_ENCRYPTION_FAILED = -16 # Failed to encrypt the wallet -RPC_WALLET_ALREADY_UNLOCKED = -17 # Wallet is already unlocked -RPC_WALLET_NOT_FOUND = -18 # Invalid wallet specified -RPC_WALLET_NOT_SPECIFIED = -19 # No wallet specified (error when there are multiple wallets loaded) +RPC_WALLET_ERROR = -4 # Unspecified problem with wallet (key not found etc.) +RPC_WALLET_INSUFFICIENT_FUNDS = -6 # Not enough funds in wallet or account +RPC_WALLET_INVALID_LABEL_NAME = -11 # Invalid label name +RPC_WALLET_KEYPOOL_RAN_OUT = -12 # Keypool ran out call keypoolrefill first +RPC_WALLET_UNLOCK_NEEDED = -13 # Enter the wallet passphrase with walletpassphrase first +RPC_WALLET_PASSPHRASE_INCORRECT = -14 # The wallet passphrase entered was incorrect +RPC_WALLET_WRONG_ENC_STATE = -15 # Command given in wrong wallet encryption state (encrypting an encrypted wallet etc.) +RPC_WALLET_ENCRYPTION_FAILED = -16 # Failed to encrypt the wallet +RPC_WALLET_ALREADY_UNLOCKED = -17 # Wallet is already unlocked +RPC_WALLET_NOT_FOUND = -18 # Invalid wallet specified +RPC_WALLET_NOT_SPECIFIED = -19 # No wallet specified (error when there are multiple wallets loaded) diff --git a/teos/watcher.py b/teos/watcher.py index 24fec8b..055c3c4 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -21,8 +21,8 @@ class Watcher: The :class:`Watcher` keeps track of the accepted appointments in ``appointments`` and, for new received block, checks if any breach has happened by comparing the txids with the appointment locators. If a breach is seen, the - :obj:`EncryptedBlob ` of the corresponding appointment is decrypted and the data - is passed to the :obj:`Responder `. + :obj:`EncryptedBlob ` of the corresponding appointment is decrypted and the + data is passed to the :obj:`Responder `. If an appointment reaches its end with no breach, the data is simply deleted. @@ -225,7 +225,7 @@ class Watcher: appointments_to_delete, self.appointments, self.locator_uuid_map, self.db_manager ) - if len(self.appointments) is 0: + if len(self.appointments) != 0: logger.info("No more pending appointments") # Register the last processed block for the watcher diff --git a/test/common/unit/test_tools.py b/test/common/unit/test_tools.py index 0410670..3e862b1 100644 --- a/test/common/unit/test_tools.py +++ b/test/common/unit/test_tools.py @@ -102,12 +102,12 @@ def test_setup_logging(): f_log_suffix = "_file_log" c_log_suffix = "_console_log" - assert len(logging.getLogger(prefix + f_log_suffix).handlers) is 0 - assert len(logging.getLogger(prefix + c_log_suffix).handlers) is 0 + assert len(logging.getLogger(prefix + f_log_suffix).handlers) == 0 + assert len(logging.getLogger(prefix + c_log_suffix).handlers) == 0 setup_logging(log_file, prefix) - assert len(logging.getLogger(prefix + f_log_suffix).handlers) is 1 - assert len(logging.getLogger(prefix + c_log_suffix).handlers) is 1 + assert len(logging.getLogger(prefix + f_log_suffix).handlers) == 1 + assert len(logging.getLogger(prefix + c_log_suffix).handlers) == 1 os.remove(log_file) diff --git a/test/teos/e2e/test_basic_e2e.py b/test/teos/e2e/test_basic_e2e.py deleted file mode 100644 index 1b19258..0000000 --- a/test/teos/e2e/test_basic_e2e.py +++ /dev/null @@ -1,302 +0,0 @@ -import json -import binascii -from time import sleep -from riemann.tx import Tx - -from cli import teos_cli, DATA_DIR, DEFAULT_CONF, CONF_FILE_NAME - -import common.cryptographer -from common.blob import Blob -from common.logger import Logger -from common.tools import compute_locator -from common.appointment import Appointment -from common.cryptographer import Cryptographer -from teos.utils.auth_proxy import JSONRPCException -from test.teos.e2e.conftest import ( - END_TIME_DELTA, - build_appointment_data, - get_random_value_hex, - create_penalty_tx, - run_teosd, - get_config, -) - -cli_config = get_config(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF) -common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix="") - -teos_base_endpoint = "http://{}:{}".format(cli_config.get("TEOS_SERVER"), cli_config.get("TEOS_PORT")) -teos_add_appointment_endpoint = "{}/add_appointment".format(teos_base_endpoint) -teos_get_appointment_endpoint = "{}/get_appointment".format(teos_base_endpoint) - -# Run teosd -teosd_process = run_teosd() - - -def broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, addr): - # Broadcast the commitment transaction and mine a block - bitcoin_cli.sendrawtransaction(commitment_tx) - bitcoin_cli.generatetoaddress(1, addr) - - -def get_appointment_info(locator): - # Check that the justice has been triggered (the appointment has moved from Watcher to Responder) - sleep(1) # Let's add a bit of delay so the state can be updated - return teos_cli.get_appointment(locator, teos_base_endpoint) - - -def test_appointment_life_cycle(bitcoin_cli, create_txs): - commitment_tx, penalty_tx = create_txs - commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") - appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) - locator = compute_locator(commitment_tx_id) - - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True - - appointment_info = get_appointment_info(locator) - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "being_watched" - - new_addr = bitcoin_cli.getnewaddress() - broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) - - appointment_info = get_appointment_info(locator) - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "dispute_responded" - - # It can be also checked by ensuring that the penalty transaction made it to the network - penalty_tx_id = bitcoin_cli.decoderawtransaction(penalty_tx).get("txid") - - try: - bitcoin_cli.getrawtransaction(penalty_tx_id) - assert True - - except JSONRPCException: - # If the transaction if not found. - assert False - - # Now let's mine some blocks so the appointment reaches its end. - # Since we are running all the nodes remotely data may take more time than normal, and some confirmations may be - # missed, so we generate more than enough confirmations and add some delays. - for _ in range(int(1.5 * END_TIME_DELTA)): - sleep(1) - bitcoin_cli.generatetoaddress(1, new_addr) - - appointment_info = get_appointment_info(locator) - assert appointment_info[0].get("status") == "not_found" - - -def test_appointment_malformed_penalty(bitcoin_cli, create_txs): - # Lets start by creating two valid transaction - commitment_tx, penalty_tx = create_txs - - # Now we can modify the penalty so it is invalid when broadcast - mod_penalty_tx = Tx.from_hex(penalty_tx) - tx_in = mod_penalty_tx.tx_ins[0].copy(redeem_script=b"") - mod_penalty_tx = mod_penalty_tx.copy(tx_ins=[tx_in]) - - commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") - appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, mod_penalty_tx.hex()) - locator = compute_locator(commitment_tx_id) - - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True - - # Broadcast the commitment transaction and mine a block - new_addr = bitcoin_cli.getnewaddress() - broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) - - # The appointment should have been removed since the penalty_tx was malformed. - sleep(1) - appointment_info = get_appointment_info(locator) - - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "not_found" - - -def test_appointment_wrong_key(bitcoin_cli, create_txs): - # This tests an appointment encrypted with a key that has not been derived from the same source as the locator. - # Therefore the tower won't be able to decrypt the blob once the appointment is triggered. - commitment_tx, penalty_tx = create_txs - - # The appointment data is built using a random 32-byte value. - appointment_data = build_appointment_data(bitcoin_cli, get_random_value_hex(32), penalty_tx) - - # We can't use teos_cli.add_appointment here since it computes the locator internally, so let's do it manually. - # We will encrypt the blob using the random value and derive the locator from the commitment tx. - appointment_data["locator"] = compute_locator(bitcoin_cli.decoderawtransaction(commitment_tx).get("txid")) - appointment_data["encrypted_blob"] = Cryptographer.encrypt(Blob(penalty_tx), get_random_value_hex(32)) - appointment = Appointment.from_dict(appointment_data) - - teos_pk, cli_sk, cli_pk_der = teos_cli.load_keys( - cli_config.get("TEOS_PUBLIC_KEY"), cli_config.get("CLI_PRIVATE_KEY"), cli_config.get("CLI_PUBLIC_KEY") - ) - hex_pk_der = binascii.hexlify(cli_pk_der) - - signature = Cryptographer.sign(appointment.serialize(), cli_sk) - data = {"appointment": appointment.to_dict(), "signature": signature, "public_key": hex_pk_der.decode("utf-8")} - - # Send appointment to the server. - response = teos_cli.post_appointment(data, teos_base_endpoint) - response_json = teos_cli.process_post_appointment_response(response) - - # Check that the server has accepted the appointment - signature = response_json.get("signature") - assert signature is not None - rpk = Cryptographer.recover_pk(appointment.serialize(), signature) - assert Cryptographer.verify_rpk(teos_pk, rpk) is True - assert response_json.get("locator") == appointment.locator - - # Trigger the appointment - new_addr = bitcoin_cli.getnewaddress() - broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) - - # The appointment should have been removed since the decryption failed. - sleep(1) - appointment_info = get_appointment_info(appointment.locator) - - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "not_found" - - -def test_two_identical_appointments(bitcoin_cli, create_txs): - # Tests sending two identical appointments to the tower. - # At the moment there are no checks for identical appointments, so both will be accepted, decrypted and kept until - # the end. - # TODO: 34-exact-duplicate-appointment - # This tests sending an appointment with two valid transaction with the same locator. - commitment_tx, penalty_tx = create_txs - commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") - - appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) - locator = compute_locator(commitment_tx_id) - - # Send the appointment twice - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True - - # Broadcast the commitment transaction and mine a block - new_addr = bitcoin_cli.getnewaddress() - broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) - - # The first appointment should have made it to the Responder, and the second one should have been dropped for - # double-spending - sleep(1) - appointment_info = get_appointment_info(locator) - - assert appointment_info is not None - assert len(appointment_info) == 2 - - for info in appointment_info: - assert info.get("status") == "dispute_responded" - assert info.get("penalty_rawtx") == penalty_tx - - -def test_two_appointment_same_locator_different_penalty(bitcoin_cli, create_txs): - # This tests sending an appointment with two valid transaction with the same locator. - commitment_tx, penalty_tx1 = create_txs - commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") - - # We need to create a second penalty spending from the same commitment - decoded_commitment_tx = bitcoin_cli.decoderawtransaction(commitment_tx) - new_addr = bitcoin_cli.getnewaddress() - penalty_tx2 = create_penalty_tx(bitcoin_cli, decoded_commitment_tx, new_addr) - - appointment1_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx1) - appointment2_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx2) - locator = compute_locator(commitment_tx_id) - - assert teos_cli.add_appointment([json.dumps(appointment1_data)], teos_base_endpoint, cli_config) is True - assert teos_cli.add_appointment([json.dumps(appointment2_data)], teos_base_endpoint, cli_config) is True - - # Broadcast the commitment transaction and mine a block - new_addr = bitcoin_cli.getnewaddress() - broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) - - # The first appointment should have made it to the Responder, and the second one should have been dropped for - # double-spending - sleep(1) - appointment_info = get_appointment_info(locator) - - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "dispute_responded" - assert appointment_info[0].get("penalty_rawtx") == penalty_tx1 - - -def test_appointment_shutdown_teos_trigger_back_online(create_txs, bitcoin_cli): - global teosd_process - - teos_pid = teosd_process.pid - - commitment_tx, penalty_tx = create_txs - commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") - appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) - locator = compute_locator(commitment_tx_id) - - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True - - # Restart teos - teosd_process.terminate() - teosd_process = run_teosd() - - assert teos_pid != teosd_process.pid - - # Check that the appointment is still in the Watcher - appointment_info = get_appointment_info(locator) - - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "being_watched" - - # Trigger appointment after restart - new_addr = bitcoin_cli.getnewaddress() - broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) - - # The appointment should have been moved to the Responder - sleep(1) - appointment_info = get_appointment_info(locator) - - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "dispute_responded" - - -def test_appointment_shutdown_teos_trigger_while_offline(create_txs, bitcoin_cli): - global teosd_process - - teos_pid = teosd_process.pid - - commitment_tx, penalty_tx = create_txs - commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") - appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) - locator = compute_locator(commitment_tx_id) - - assert teos_cli.add_appointment([json.dumps(appointment_data)], teos_base_endpoint, cli_config) is True - - # Check that the appointment is still in the Watcher - appointment_info = get_appointment_info(locator) - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "being_watched" - - # Shutdown and trigger - teosd_process.terminate() - new_addr = bitcoin_cli.getnewaddress() - broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) - - # Restart - teosd_process = run_teosd() - assert teos_pid != teosd_process.pid - - # The appointment should have been moved to the Responder - sleep(1) - appointment_info = get_appointment_info(locator) - - assert appointment_info is not None - assert len(appointment_info) == 1 - assert appointment_info[0].get("status") == "dispute_responded" - - teosd_process.terminate() diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index f417ee6..1013685 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -9,16 +9,9 @@ from teos.watcher import Watcher from teos.inspector import Inspector from teos.db_manager import DBManager from teos.gatekeeper import Gatekeeper -from teos.chain_monitor import ChainMonitor from teos.responder import Responder, TransactionTracker -from test.teos.unit.conftest import ( - get_random_value_hex, - generate_dummy_appointment, - generate_keypair, - get_config, - bitcoind_feed_params, -) +from test.teos.unit.conftest import get_random_value_hex, generate_dummy_appointment, generate_keypair, get_config from common.cryptographer import Cryptographer, hash_160 from common.constants import ( diff --git a/test/teos/unit/test_block_processor.py b/test/teos/unit/test_block_processor.py index abcb964..f61082c 100644 --- a/test/teos/unit/test_block_processor.py +++ b/test/teos/unit/test_block_processor.py @@ -1,6 +1,4 @@ -import pytest - -from test.teos.unit.conftest import get_random_value_hex, generate_block, generate_blocks, fork, bitcoind_connect_params +from test.teos.unit.conftest import get_random_value_hex, generate_block, generate_blocks, fork hex_tx = ( diff --git a/test/teos/unit/test_chain_monitor.py b/test/teos/unit/test_chain_monitor.py index c0d969b..3c2f24b 100644 --- a/test/teos/unit/test_chain_monitor.py +++ b/test/teos/unit/test_chain_monitor.py @@ -5,7 +5,7 @@ from threading import Thread, Event, Condition from teos.chain_monitor import ChainMonitor -from test.teos.unit.conftest import get_random_value_hex, generate_block, bitcoind_connect_params, bitcoind_feed_params +from test.teos.unit.conftest import get_random_value_hex, generate_block, bitcoind_feed_params def test_init(run_bitcoind, block_processor): @@ -64,8 +64,8 @@ def test_update_state(block_processor): def test_monitor_chain_polling(db_manager, block_processor): # Try polling with the Watcher - wq = Queue() - chain_monitor = ChainMonitor(Queue(), Queue(), block_processor, bitcoind_feed_params) + watcher_queue = Queue() + chain_monitor = ChainMonitor(watcher_queue, Queue(), block_processor, bitcoind_feed_params) chain_monitor.best_tip = block_processor.get_best_block_hash() chain_monitor.polling_delta = 0.1 diff --git a/test/teos/unit/test_inspector.py b/test/teos/unit/test_inspector.py index 6f02e4c..b3993f5 100644 --- a/test/teos/unit/test_inspector.py +++ b/test/teos/unit/test_inspector.py @@ -1,7 +1,7 @@ import pytest from binascii import unhexlify -from teos.errors import * +import teos.errors as errors from teos import LOG_PREFIX from teos.block_processor import BlockProcessor from teos.inspector import Inspector, InspectionFailed @@ -11,7 +11,7 @@ from common.logger import Logger from common.appointment import Appointment from common.constants import LOCATOR_LEN_BYTES, LOCATOR_LEN_HEX -from test.teos.unit.conftest import get_random_value_hex, generate_keypair, bitcoind_connect_params, get_config +from test.teos.unit.conftest import get_random_value_hex, bitcoind_connect_params, get_config common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX) @@ -53,7 +53,7 @@ def test_check_locator(): inspector.check_locator(locator) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_SIZE + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_SIZE raise e # Wrong size (too small) @@ -63,7 +63,7 @@ def test_check_locator(): inspector.check_locator(locator) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_SIZE + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_SIZE raise e # Empty @@ -73,7 +73,7 @@ def test_check_locator(): inspector.check_locator(locator) except InspectionFailed as e: - assert e.erno == APPOINTMENT_EMPTY_FIELD + assert e.erno == errors.APPOINTMENT_EMPTY_FIELD raise e # Wrong type (several types tested, it should do for anything that is not a string) @@ -85,7 +85,7 @@ def test_check_locator(): inspector.check_locator(locator) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_TYPE raise e # Wrong format (no hex) @@ -96,7 +96,7 @@ def test_check_locator(): inspector.check_locator(locator) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_FORMAT + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_FORMAT raise e @@ -116,7 +116,7 @@ def test_check_start_time(): inspector.check_start_time(start_time, current_time) except InspectionFailed as e: - assert e.erno == APPOINTMENT_FIELD_TOO_SMALL + assert e.erno == errors.APPOINTMENT_FIELD_TOO_SMALL raise e # Empty field @@ -126,7 +126,7 @@ def test_check_start_time(): inspector.check_start_time(start_time, current_time) except InspectionFailed as e: - assert e.erno == APPOINTMENT_EMPTY_FIELD + assert e.erno == errors.APPOINTMENT_EMPTY_FIELD raise e # Wrong data type @@ -137,7 +137,7 @@ def test_check_start_time(): inspector.check_start_time(start_time, current_time) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_TYPE raise e @@ -158,7 +158,7 @@ def test_check_end_time(): inspector.check_end_time(end_time, start_time, current_time) except InspectionFailed as e: - assert e.erno == APPOINTMENT_FIELD_TOO_SMALL + assert e.erno == errors.APPOINTMENT_FIELD_TOO_SMALL raise e # End time too small (either same height as current block or in the past) @@ -170,7 +170,7 @@ def test_check_end_time(): inspector.check_end_time(end_time, start_time, current_time) except InspectionFailed as e: - assert e.erno == APPOINTMENT_FIELD_TOO_SMALL + assert e.erno == errors.APPOINTMENT_FIELD_TOO_SMALL raise e # Empty field @@ -180,7 +180,7 @@ def test_check_end_time(): inspector.check_end_time(end_time, start_time, current_time) except InspectionFailed as e: - assert e.erno == APPOINTMENT_EMPTY_FIELD + assert e.erno == errors.APPOINTMENT_EMPTY_FIELD raise e # Wrong data type @@ -191,7 +191,7 @@ def test_check_end_time(): inspector.check_end_time(end_time, start_time, current_time) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_TYPE raise e @@ -209,7 +209,7 @@ def test_check_to_self_delay(): inspector.check_to_self_delay(to_self_delay) except InspectionFailed as e: - assert e.erno == APPOINTMENT_FIELD_TOO_SMALL + assert e.erno == errors.APPOINTMENT_FIELD_TOO_SMALL raise e # Empty field @@ -219,7 +219,7 @@ def test_check_to_self_delay(): inspector.check_to_self_delay(to_self_delay) except InspectionFailed as e: - assert e.erno == APPOINTMENT_EMPTY_FIELD + assert e.erno == errors.APPOINTMENT_EMPTY_FIELD raise e # Wrong data type @@ -230,7 +230,7 @@ def test_check_to_self_delay(): inspector.check_to_self_delay(to_self_delay) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_TYPE raise e @@ -251,7 +251,7 @@ def test_check_blob(): inspector.check_blob(encrypted_blob) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_TYPE + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_TYPE raise e # Empty field @@ -261,7 +261,7 @@ def test_check_blob(): inspector.check_blob(encrypted_blob) except InspectionFailed as e: - assert e.erno == APPOINTMENT_EMPTY_FIELD + assert e.erno == errors.APPOINTMENT_EMPTY_FIELD raise e # Wrong format (no hex) @@ -272,7 +272,7 @@ def test_check_blob(): inspector.check_blob(encrypted_blob) except InspectionFailed as e: - assert e.erno == APPOINTMENT_WRONG_FIELD_FORMAT + assert e.erno == errors.APPOINTMENT_WRONG_FIELD_FORMAT raise e @@ -313,7 +313,7 @@ def test_inspect_wrong(run_bitcoind): inspector.inspect(data) except InspectionFailed as e: print(data) - assert e.erno == APPOINTMENT_WRONG_FIELD + assert e.erno == errors.APPOINTMENT_WRONG_FIELD raise e # None data @@ -321,5 +321,5 @@ def test_inspect_wrong(run_bitcoind): try: inspector.inspect(None) except InspectionFailed as e: - assert e.erno == APPOINTMENT_EMPTY_FIELD + assert e.erno == errors.APPOINTMENT_EMPTY_FIELD raise e From 502f507b2170ae18b0435c182cd4afa1ed89c498 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 31 Mar 2020 15:00:30 +0200 Subject: [PATCH 42/62] Adds missing size field in Builder when loading data from the database --- teos/builder.py | 6 +++++- test/teos/unit/test_builder.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/teos/builder.py b/teos/builder.py index 379bac8..6d087fa 100644 --- a/teos/builder.py +++ b/teos/builder.py @@ -26,7 +26,11 @@ class Builder: locator_uuid_map = {} for uuid, data in appointments_data.items(): - appointments[uuid] = {"locator": data.get("locator"), "end_time": data.get("end_time")} + appointments[uuid] = { + "locator": data.get("locator"), + "end_time": data.get("end_time"), + "size": len(data.get("encrypted_blob")), + } if data.get("locator") in locator_uuid_map: locator_uuid_map[data.get("locator")].append(uuid) diff --git a/test/teos/unit/test_builder.py b/test/teos/unit/test_builder.py index 014d797..756cc5e 100644 --- a/test/teos/unit/test_builder.py +++ b/test/teos/unit/test_builder.py @@ -46,6 +46,7 @@ def test_build_appointments(): assert uuid in appointments_data.keys() assert appointments_data[uuid].get("locator") == appointment.get("locator") assert appointments_data[uuid].get("end_time") == appointment.get("end_time") + assert len(appointments_data[uuid].get("encrypted_blob")) == appointment.get("size") assert uuid in locator_uuid_map[appointment.get("locator")] From 3e32d8a85e41dcf2255f7905b467cfa7ef791783 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 31 Mar 2020 16:54:52 +0200 Subject: [PATCH 43/62] Fixes typo in variable and inconsistent return type --- cli/teos_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/teos_cli.py b/cli/teos_cli.py index 994185a..bdd4989 100644 --- a/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -30,7 +30,7 @@ common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_ def register(compressed_pk, teos_url): if not check_compressed_pk_format(compressed_pk): logger.error("The cli public key is not valid") - return False + return None # Send request to the server. register_endpoint = "{}/register".format(teos_url) @@ -383,7 +383,7 @@ def main(args, command_line_conf): keys = load_keys(config.get("TEOS_PUBLIC_KEY"), config.get("CLI_PRIVATE_KEY"), config.get("CLI_PUBLIC_KEY")) if keys is not None: - teos_pk, cli_sk, compress_cli_pk = keys + teos_pk, cli_sk, compressed_cli_pk = keys try: if args: From bf0ae36940e02aad1e40350abad0617555bac619 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 31 Mar 2020 16:55:40 +0200 Subject: [PATCH 44/62] Removes old commented code --- test/teos/e2e/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/teos/e2e/conftest.py b/test/teos/e2e/conftest.py index 1cf4c63..f9e7f0b 100644 --- a/test/teos/e2e/conftest.py +++ b/test/teos/e2e/conftest.py @@ -17,8 +17,6 @@ END_TIME_DELTA = 10 @pytest.fixture(scope="session") def bitcoin_cli(): config = get_config(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF) - print(config) - # btc_connect_params = {k: v["value"] for k, v in DEFAULT_CONF.items() if k.startswith("BTC")} return AuthServiceProxy( "http://%s:%s@%s:%d" From 0364dba5cafd6eadeb326c5a505807ad5d69e829 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 31 Mar 2020 16:56:52 +0200 Subject: [PATCH 45/62] Fixes e2e tests to match the user authentication apporach Last two tests are patched, user data must be stored between restarts --- test/teos/e2e/test_basic_e2e.py | 389 ++++++++++++++++++++++++++++++++ 1 file changed, 389 insertions(+) create mode 100644 test/teos/e2e/test_basic_e2e.py diff --git a/test/teos/e2e/test_basic_e2e.py b/test/teos/e2e/test_basic_e2e.py new file mode 100644 index 0000000..9bc63c2 --- /dev/null +++ b/test/teos/e2e/test_basic_e2e.py @@ -0,0 +1,389 @@ +from time import sleep +from riemann.tx import Tx +from binascii import hexlify +from coincurve import PrivateKey + +from cli import teos_cli, DATA_DIR, DEFAULT_CONF, CONF_FILE_NAME + +import common.cryptographer +from common.blob import Blob +from common.logger import Logger +from common.tools import compute_locator +from common.appointment import Appointment +from common.cryptographer import Cryptographer +from teos.utils.auth_proxy import JSONRPCException +from test.teos.e2e.conftest import ( + END_TIME_DELTA, + build_appointment_data, + get_random_value_hex, + create_penalty_tx, + run_teosd, + get_config, +) + +cli_config = get_config(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF) +common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix="") + +teos_base_endpoint = "http://{}:{}".format(cli_config.get("TEOS_SERVER"), cli_config.get("TEOS_PORT")) +teos_add_appointment_endpoint = "{}/add_appointment".format(teos_base_endpoint) +teos_get_appointment_endpoint = "{}/get_appointment".format(teos_base_endpoint) + +# Run teosd +teosd_process = run_teosd() + +teos_pk, cli_sk, compressed_cli_pk = teos_cli.load_keys( + cli_config.get("TEOS_PUBLIC_KEY"), cli_config.get("CLI_PRIVATE_KEY"), cli_config.get("CLI_PUBLIC_KEY") +) + + +def broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, addr): + # Broadcast the commitment transaction and mine a block + bitcoin_cli.sendrawtransaction(commitment_tx) + bitcoin_cli.generatetoaddress(1, addr) + + +def get_appointment_info(locator, sk=cli_sk): + sleep(1) # Let's add a bit of delay so the state can be updated + return teos_cli.get_appointment(locator, sk, teos_pk, teos_base_endpoint) + + +def add_appointment(appointment_data, sk=cli_sk): + return teos_cli.add_appointment( + appointment_data, sk, teos_pk, teos_base_endpoint, cli_config.get("APPOINTMENTS_FOLDER_NAME") + ) + + +def test_commands_non_registered(bitcoin_cli, create_txs): + # All commands should failed if the user is not registered + + # Add appointment + commitment_tx, penalty_tx = create_txs + commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") + appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) + + assert add_appointment(appointment_data) is False + + # Get appointment + assert get_appointment_info(appointment_data.get("locator")) is None + + +def test_commands_registered(bitcoin_cli, create_txs): + # Test registering and trying again + teos_cli.register(compressed_cli_pk, teos_base_endpoint) + + # Add appointment + commitment_tx, penalty_tx = create_txs + commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") + appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) + + assert add_appointment(appointment_data) is True + + # Get appointment + r = get_appointment_info(appointment_data.get("locator")) + assert r.get("locator") == appointment_data.get("locator") + assert r.get("appointment").get("locator") == appointment_data.get("locator") + assert r.get("appointment").get("encrypted_blob") == appointment_data.get("encrypted_blob") + assert r.get("appointment").get("start_time") == appointment_data.get("start_time") + assert r.get("appointment").get("end_time") == appointment_data.get("end_time") + + +def test_appointment_life_cycle(bitcoin_cli, create_txs): + # First of all we need to register + # FIXME: requires register command in the cli + commitment_tx, penalty_tx = create_txs + commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") + appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) + locator = compute_locator(commitment_tx_id) + + assert add_appointment(appointment_data) is True + + appointment_info = get_appointment_info(locator) + assert appointment_info is not None + assert appointment_info.get("status") == "being_watched" + + new_addr = bitcoin_cli.getnewaddress() + broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) + + appointment_info = get_appointment_info(locator) + assert appointment_info is not None + assert appointment_info.get("status") == "dispute_responded" + + # It can be also checked by ensuring that the penalty transaction made it to the network + penalty_tx_id = bitcoin_cli.decoderawtransaction(penalty_tx).get("txid") + + try: + bitcoin_cli.getrawtransaction(penalty_tx_id) + assert True + + except JSONRPCException: + # If the transaction is not found. + assert False + + # Now let's mine some blocks so the appointment reaches its end. + # Since we are running all the nodes remotely data may take more time than normal, and some confirmations may be + # missed, so we generate more than enough confirmations and add some delays. + for _ in range(int(1.5 * END_TIME_DELTA)): + sleep(1) + bitcoin_cli.generatetoaddress(1, new_addr) + + assert get_appointment_info(locator) is None + + +def test_appointment_malformed_penalty(bitcoin_cli, create_txs): + # Lets start by creating two valid transaction + commitment_tx, penalty_tx = create_txs + + # Now we can modify the penalty so it is invalid when broadcast + mod_penalty_tx = Tx.from_hex(penalty_tx) + tx_in = mod_penalty_tx.tx_ins[0].copy(redeem_script=b"") + mod_penalty_tx = mod_penalty_tx.copy(tx_ins=[tx_in]) + + commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") + appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, mod_penalty_tx.hex()) + locator = compute_locator(commitment_tx_id) + + assert add_appointment(appointment_data) is True + + # Broadcast the commitment transaction and mine a block + new_addr = bitcoin_cli.getnewaddress() + broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) + + # The appointment should have been removed since the penalty_tx was malformed. + sleep(1) + assert get_appointment_info(locator) is None + + +def test_appointment_wrong_key(bitcoin_cli, create_txs): + # This tests an appointment encrypted with a key that has not been derived from the same source as the locator. + # Therefore the tower won't be able to decrypt the blob once the appointment is triggered. + commitment_tx, penalty_tx = create_txs + + # The appointment data is built using a random 32-byte value. + appointment_data = build_appointment_data(bitcoin_cli, get_random_value_hex(32), penalty_tx) + + # We can't use teos_cli.add_appointment here since it computes the locator internally, so let's do it manually. + # We will encrypt the blob using the random value and derive the locator from the commitment tx. + appointment_data["locator"] = compute_locator(bitcoin_cli.decoderawtransaction(commitment_tx).get("txid")) + appointment_data["encrypted_blob"] = Cryptographer.encrypt(Blob(penalty_tx), get_random_value_hex(32)) + appointment = Appointment.from_dict(appointment_data) + + signature = Cryptographer.sign(appointment.serialize(), cli_sk) + data = {"appointment": appointment.to_dict(), "signature": signature} + + # Send appointment to the server. + response = teos_cli.post_request(data, teos_add_appointment_endpoint) + response_json = teos_cli.process_post_response(response) + + # Check that the server has accepted the appointment + signature = response_json.get("signature") + assert signature is not None + rpk = Cryptographer.recover_pk(appointment.serialize(), signature) + assert Cryptographer.verify_rpk(teos_pk, rpk) is True + assert response_json.get("locator") == appointment.locator + + # Trigger the appointment + new_addr = bitcoin_cli.getnewaddress() + broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) + + # The appointment should have been removed since the decryption failed. + sleep(1) + assert get_appointment_info(appointment.locator) is None + + +def test_two_identical_appointments(bitcoin_cli, create_txs): + # Tests sending two identical appointments to the tower. + # This tests sending an appointment with two valid transaction with the same locator. + # If they come from the same user, the last one will be kept + commitment_tx, penalty_tx = create_txs + commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") + + appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) + locator = compute_locator(commitment_tx_id) + + # Send the appointment twice + assert add_appointment(appointment_data) is True + assert add_appointment(appointment_data) is True + + # Broadcast the commitment transaction and mine a block + new_addr = bitcoin_cli.getnewaddress() + broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) + + # The last appointment should have made it to the Responder + sleep(1) + appointment_info = get_appointment_info(locator) + + assert appointment_info is not None + assert appointment_info.get("status") == "dispute_responded" + assert appointment_info.get("appointment").get("penalty_rawtx") == penalty_tx + + +# FIXME: This test won't work since we're still passing appointment replicas to the Responder. +# Uncomment when #88 is addressed +# def test_two_identical_appointments_different_users(bitcoin_cli, create_txs): +# # Tests sending two identical appointments from different users to the tower. +# # This tests sending an appointment with two valid transaction with the same locator. +# # If they come from different users, both will be kept, but one will be dropped fro double-spending when passing to +# # the responder +# commitment_tx, penalty_tx = create_txs +# commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") +# +# appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) +# locator = compute_locator(commitment_tx_id) +# +# # tmp keys from a different user +# tmp_sk = PrivateKey() +# tmp_compressed_pk = hexlify(tmp_sk.public_key.format(compressed=True)).decode("utf-8") +# teos_cli.register(tmp_compressed_pk, teos_base_endpoint) +# +# # Send the appointment twice +# assert add_appointment(appointment_data) is True +# assert add_appointment(appointment_data, sk=tmp_sk) is True +# +# # Check that we can get it from both users +# appointment_info = get_appointment_info(locator) +# assert appointment_info.get("status") == "being_watched" +# appointment_info = get_appointment_info(locator, sk=tmp_sk) +# assert appointment_info.get("status") == "being_watched" +# +# # Broadcast the commitment transaction and mine a block +# new_addr = bitcoin_cli.getnewaddress() +# broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) +# +# # The last appointment should have made it to the Responder +# sleep(1) +# appointment_info = get_appointment_info(locator) +# appointment_dup_info = get_appointment_info(locator, sk=tmp_sk) +# +# # One of the two request must be None, while the other must be valid +# assert (appointment_info is None and appointment_dup_info is not None) or ( +# appointment_dup_info is None and appointment_info is not None +# ) +# +# appointment_info = appointment_info if appointment_info is None else appointment_dup_info +# +# assert appointment_info.get("status") == "dispute_responded" +# assert appointment_info.get("appointment").get("penalty_rawtx") == penalty_tx + + +def test_two_appointment_same_locator_different_penalty_different_users(bitcoin_cli, create_txs): + # This tests sending an appointment with two valid transaction with the same locator. + commitment_tx, penalty_tx1 = create_txs + commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") + + # We need to create a second penalty spending from the same commitment + decoded_commitment_tx = bitcoin_cli.decoderawtransaction(commitment_tx) + new_addr = bitcoin_cli.getnewaddress() + penalty_tx2 = create_penalty_tx(bitcoin_cli, decoded_commitment_tx, new_addr) + + appointment1_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx1) + appointment2_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx2) + locator = compute_locator(commitment_tx_id) + + # tmp keys from a different user + tmp_sk = PrivateKey() + tmp_compressed_pk = hexlify(tmp_sk.public_key.format(compressed=True)).decode("utf-8") + teos_cli.register(tmp_compressed_pk, teos_base_endpoint) + + assert add_appointment(appointment1_data) is True + assert add_appointment(appointment2_data, sk=tmp_sk) is True + + # Broadcast the commitment transaction and mine a block + new_addr = bitcoin_cli.getnewaddress() + broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) + + # One of the transactions must have made it to the Responder while the other must have been dropped for + # double-spending + sleep(1) + appointment_info = get_appointment_info(locator) + appointment2_info = get_appointment_info(locator, sk=tmp_sk) + + # One of the two request must be None, while the other must be valid + assert (appointment_info is None and appointment2_info is not None) or ( + appointment2_info is None and appointment_info is not None + ) + + if appointment_info is None: + appointment_info = appointment2_info + appointment1_data = appointment2_data + + assert appointment_info.get("status") == "dispute_responded" + assert appointment_info.get("locator") == appointment1_data.get("locator") + + +def test_appointment_shutdown_teos_trigger_back_online(create_txs, bitcoin_cli): + global teosd_process + + teos_pid = teosd_process.pid + + commitment_tx, penalty_tx = create_txs + commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") + appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) + locator = compute_locator(commitment_tx_id) + + assert add_appointment(appointment_data) is True + + # Restart teos + teosd_process.terminate() + teosd_process = run_teosd() + + assert teos_pid != teosd_process.pid + + # FIXME: We have to cheat here since users are not kept between restarts atm + sleep(1) + teos_cli.register(compressed_cli_pk, teos_base_endpoint) + + # Check that the appointment is still in the Watcher + appointment_info = get_appointment_info(locator) + + assert appointment_info is not None + assert appointment_info.get("status") == "being_watched" + + # Trigger appointment after restart + new_addr = bitcoin_cli.getnewaddress() + broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) + + # The appointment should have been moved to the Responder + sleep(1) + appointment_info = get_appointment_info(locator) + + assert appointment_info is not None + assert appointment_info.get("status") == "dispute_responded" + + +def test_appointment_shutdown_teos_trigger_while_offline(create_txs, bitcoin_cli): + global teosd_process + + teos_pid = teosd_process.pid + + commitment_tx, penalty_tx = create_txs + commitment_tx_id = bitcoin_cli.decoderawtransaction(commitment_tx).get("txid") + appointment_data = build_appointment_data(bitcoin_cli, commitment_tx_id, penalty_tx) + locator = compute_locator(commitment_tx_id) + + assert add_appointment(appointment_data) is True + + # Check that the appointment is still in the Watcher + appointment_info = get_appointment_info(locator) + assert appointment_info is not None + assert appointment_info.get("status") == "being_watched" + + # Shutdown and trigger + teosd_process.terminate() + new_addr = bitcoin_cli.getnewaddress() + broadcast_transaction_and_mine_block(bitcoin_cli, commitment_tx, new_addr) + + # Restart + teosd_process = run_teosd() + assert teos_pid != teosd_process.pid + + # FIXME: We have to cheat here since users are not kept between restarts atm + sleep(1) + teos_cli.register(compressed_cli_pk, teos_base_endpoint) + + # The appointment should have been moved to the Responder + appointment_info = get_appointment_info(locator) + + assert appointment_info is not None + assert appointment_info.get("status") == "dispute_responded" + + teosd_process.terminate() From 18e9a0fe5f47e01696492303cf4b6057a2ff07d5 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Tue, 31 Mar 2020 17:51:19 +0200 Subject: [PATCH 46/62] Removes 2kib restriction from readmes --- cli/README.md | 1 - cli/TEOS-API.md | 1 - 2 files changed, 2 deletions(-) diff --git a/cli/README.md b/cli/README.md index 2552da4..61b9082 100644 --- a/cli/README.md +++ b/cli/README.md @@ -54,7 +54,6 @@ The alpha release does not have authentication, payments nor rate limiting, ther - `start_time` should be within the next 6 blocks `[current_time+1, current_time+6]`. - `end_time` cannot be bigger than (roughly) a month. That is `4320` blocks on top of `start_time`. -- `encrypted_blob`s are limited to `2 kib`. #### Usage diff --git a/cli/TEOS-API.md b/cli/TEOS-API.md index 0d5c3cf..d0323b5 100644 --- a/cli/TEOS-API.md +++ b/cli/TEOS-API.md @@ -33,7 +33,6 @@ The alpha release does not have authentication, payments nor rate limiting, ther - `start_time` should be within the next 6 blocks `[current_time+1, current_time+6]`. - `end_time` cannot be bigger than (roughtly) a month. That is `4320` blocks on top of `start_time`. -- `encrypted_blob`s are limited to `2 kib`. #### Appointment example From 2d67fe8638a47e1257fbd2b4addcb0c55e98f00f Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 11:14:20 +0200 Subject: [PATCH 47/62] rpc_passwd -> rpc_password. The former can be confusing since the bitcoind parameter is rpcpassword. Avoiding that. --- README.md | 4 ++-- teos/__init__.py | 2 +- teos/template.conf | 2 +- teos/teosd.py | 2 +- teos/tools.py | 2 +- test/teos/e2e/conftest.py | 2 +- test/teos/e2e/teos.conf | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0947250..945047a 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ The configuration file options to change the network where `teos` will run are t ``` [bitcoind] btc_rpc_user = "user" -btc_rpc_passwd = "passwd" +btc_rpc_password = "passwd" btc_rpc_connect = "localhost" btc_rpc_port = 8332 btc_network = "mainnet" @@ -69,7 +69,7 @@ For regtest, it should look like: ``` [bitcoind] btc_rpc_user = "user" -btc_rpc_passwd = "passwd" +btc_rpc_password = "passwd" btc_rpc_connect = "localhost" btc_rpc_port = 18443 btc_network = "regtest" diff --git a/teos/__init__.py b/teos/__init__.py index ef94053..acd5f83 100644 --- a/teos/__init__.py +++ b/teos/__init__.py @@ -9,7 +9,7 @@ LOG_PREFIX = "teos" # Default conf fields DEFAULT_CONF = { "BTC_RPC_USER": {"value": "user", "type": str}, - "BTC_RPC_PASSWD": {"value": "passwd", "type": str}, + "BTC_RPC_PASSWORD": {"value": "passwd", "type": str}, "BTC_RPC_CONNECT": {"value": "127.0.0.1", "type": str}, "BTC_RPC_PORT": {"value": 8332, "type": int}, "BTC_NETWORK": {"value": "mainnet", "type": str}, diff --git a/teos/template.conf b/teos/template.conf index 46c61e8..1989de6 100644 --- a/teos/template.conf +++ b/teos/template.conf @@ -1,6 +1,6 @@ [bitcoind] btc_rpc_user = user -btc_rpc_passwd = passwd +btc_rpc_password = passwd btc_rpc_connect = localhost btc_rpc_port = 8332 btc_network = mainnet diff --git a/teos/teosd.py b/teos/teosd.py index c33e42b..72646fe 100644 --- a/teos/teosd.py +++ b/teos/teosd.py @@ -173,7 +173,7 @@ if __name__ == "__main__": if opt in ["--btcrpcuser"]: command_line_conf["BTC_RPC_USER"] = arg if opt in ["--btcrpcpassword"]: - command_line_conf["BTC_RPC_PASSWD"] = arg + command_line_conf["BTC_RPC_PASSWORD"] = arg if opt in ["--btcrpcconnect"]: command_line_conf["BTC_RPC_CONNECT"] = arg if opt in ["--btcrpcport"]: diff --git a/teos/tools.py b/teos/tools.py index 25e6f20..dded387 100644 --- a/teos/tools.py +++ b/teos/tools.py @@ -26,7 +26,7 @@ def bitcoin_cli(btc_connect_params): "http://%s:%s@%s:%d" % ( btc_connect_params.get("BTC_RPC_USER"), - btc_connect_params.get("BTC_RPC_PASSWD"), + btc_connect_params.get("BTC_RPC_PASSWORD"), btc_connect_params.get("BTC_RPC_CONNECT"), btc_connect_params.get("BTC_RPC_PORT"), ) diff --git a/test/teos/e2e/conftest.py b/test/teos/e2e/conftest.py index f9e7f0b..eb892c0 100644 --- a/test/teos/e2e/conftest.py +++ b/test/teos/e2e/conftest.py @@ -22,7 +22,7 @@ def bitcoin_cli(): "http://%s:%s@%s:%d" % ( config.get("BTC_RPC_USER"), - config.get("BTC_RPC_PASSWD"), + config.get("BTC_RPC_PASSWORD"), config.get("BTC_RPC_CONNECT"), config.get("BTC_RPC_PORT"), ) diff --git a/test/teos/e2e/teos.conf b/test/teos/e2e/teos.conf index 33e4294..6b6fd33 100644 --- a/test/teos/e2e/teos.conf +++ b/test/teos/e2e/teos.conf @@ -1,6 +1,6 @@ [bitcoind] btc_rpc_user = user -btc_rpc_passwd = passwd +btc_rpc_password = passwd btc_rpc_connect = localhost btc_rpc_port = 18445 btc_network = regtest From 0e392d1f2aa3dc60ade825a794e9de28d8d88f7d Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 17:04:33 +0200 Subject: [PATCH 48/62] Cryptographer.sigrec_decode assumed signatures where properly encoded. - Fixes a bug in sigrec_decode where the decoding function assumend that the first by was formatted as 31 + SigRec. Non properly encoded signatures made the function to crash due to an overflow (int_to_bytes(x) for negative x) --- common/cryptographer.py | 18 +++++++++++++----- test/common/unit/test_cryptographer.py | 9 +++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/common/cryptographer.py b/common/cryptographer.py index ebe15ac..e7d4fac 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -43,6 +43,7 @@ def hash_160(message): return h160 +# NOTCOVERED def sigrec_encode(rsig_rid): """ Encodes a pk-recoverable signature to be used in LN. ```rsig_rid`` can be obtained trough @@ -63,6 +64,7 @@ def sigrec_encode(rsig_rid): return sigrec +# NOTCOVERED def sigrec_decode(sigrec): """ Decodes a pk-recoverable signature in the format used by LN to be input to ``PublicKey.from_signature_and_message``. @@ -72,12 +74,18 @@ def sigrec_decode(sigrec): Returns: :obj:`bytes`: the decoded signature. + + Raises: + :obj:`ValueError`: if the SigRec is not properly encoded (first byte is not 31 + recovery id) """ - rid, rsig = int_to_bytes(sigrec[0] - 31), sigrec[1:] - rsig_rid = rsig + rid + int_rid, rsig = sigrec[0] - 31, sigrec[1:] + if int_rid < 0: + raise ValueError("Wrong SigRec") + else: + rid = int_to_bytes(int_rid) - return rsig_rid + return rsig + rid # FIXME: Common has not log file, so it needs to log in the same log as the caller. This is a temporary fix. @@ -299,9 +307,9 @@ class Cryptographer: return None sigrec = pyzbase32.decode_bytes(zb32_sig) - rsig_recid = sigrec_decode(sigrec) try: + rsig_recid = sigrec_decode(sigrec) pk = PublicKey.from_signature_and_message(rsig_recid, LN_MESSAGE_PREFIX + message, hasher=sha256d) return pk @@ -313,7 +321,7 @@ class Cryptographer: except Exception as e: if "failed to recover ECDSA public key" in str(e): - logger.error("Cannot recover public key from signature".format(type(rsig_recid))) + logger.error("Cannot recover public key from signature") else: logger.error("Unknown exception", error=str(e)) diff --git a/test/common/unit/test_cryptographer.py b/test/common/unit/test_cryptographer.py index ddbb707..bb60125 100644 --- a/test/common/unit/test_cryptographer.py +++ b/test/common/unit/test_cryptographer.py @@ -208,6 +208,15 @@ def test_recover_pk(): assert isinstance(rpk, PublicKey) +def test_recover_pk_invalid_sigrec(): + message = "Hey, it's me" + signature = "ddbfb019e4d56155b4175066c2b615ab765d317ae7996d188b4a5fae4cc394adf98fef46034d0553149392219ca6d37dca9abdfa6366a8e54b28f19d3e5efa8a14b556205dc7f33a" + + # The given signature, when zbase32 decoded, has a fist byte with value lower than 31. + # The first byte of the signature should be 31 + SigRec, so this should fail + assert Cryptographer.recover_pk(message, signature) is None + + def test_recover_pk_ground_truth(): # Use a message a signature generated by c-lightning and see if we recover the proper key message = b"Test message" From 7031b552f7a03853187f7e9c62f612e2367c6f1b Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 17:13:20 +0200 Subject: [PATCH 49/62] Splits DBManager in parent and child classes so a UserDBManager can be implemented - DBManager preserves the basic methods, while the appointment related ones have been moved to AppointmentDBM. - AppointmentDBM handles json loads and dumps for appointments and trackers, so the corresponding methods can be removed from their classes. - Updates docs. --- common/appointment.py | 13 -- teos/__init__.py | 2 +- teos/appointments_dbm.py | 422 +++++++++++++++++++++++++++++++++++++ teos/cleaner.py | 24 +-- teos/db_manager.py | 437 ++++----------------------------------- teos/responder.py | 24 +-- teos/teosd.py | 4 +- teos/watcher.py | 8 +- 8 files changed, 490 insertions(+), 444 deletions(-) create mode 100644 teos/appointments_dbm.py diff --git a/common/appointment.py b/common/appointment.py index 41c7e57..164fb67 100644 --- a/common/appointment.py +++ b/common/appointment.py @@ -81,19 +81,6 @@ class Appointment: return appointment - def to_json(self): - """ - Exports an appointment as a deterministic json encoded string. - - This method ensures that multiple invocations with the same data yield the same value. This is the format used - to store appointments in the database. - - Returns: - :obj:`str`: A json-encoded str representing the appointment. - """ - - return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) - def serialize(self): """ Serializes an appointment to be signed. diff --git a/teos/__init__.py b/teos/__init__.py index acd5f83..0eba6bd 100644 --- a/teos/__init__.py +++ b/teos/__init__.py @@ -22,5 +22,5 @@ DEFAULT_CONF = { "MIN_TO_SELF_DELAY": {"value": 20, "type": int}, "LOG_FILE": {"value": "teos.log", "type": str, "path": True}, "TEOS_SECRET_KEY": {"value": "teos_sk.der", "type": str, "path": True}, - "DB_PATH": {"value": "appointments", "type": str, "path": True}, + "APPOINTMENTS_DB_PATH": {"value": "appointments", "type": str, "path": True}, } diff --git a/teos/appointments_dbm.py b/teos/appointments_dbm.py new file mode 100644 index 0000000..52cb946 --- /dev/null +++ b/teos/appointments_dbm.py @@ -0,0 +1,422 @@ +import json +import plyvel + +from teos.db_manager import DBManager + +from teos import LOG_PREFIX + +from common.logger import Logger + +logger = Logger(actor="AppointmentsDBM", log_name_prefix=LOG_PREFIX) + +WATCHER_PREFIX = "w" +WATCHER_LAST_BLOCK_KEY = "bw" +RESPONDER_PREFIX = "r" +RESPONDER_LAST_BLOCK_KEY = "br" +LOCATOR_MAP_PREFIX = "m" +TRIGGERED_APPOINTMENTS_PREFIX = "ta" + + +class AppointmentsDBM(DBManager): + """ + The :class:`AppointmentsDBM` is the class in charge of interacting with the appointments database (``LevelDB``). + Keys and values are stored as bytes in the database but processed as strings by the manager. + + The database is split in six prefixes: + + - ``WATCHER_PREFIX``, defined as ``b'w``, is used to store :obj:`Watcher ` appointments. + - ``RESPONDER_PREFIX``, defines as ``b'r``, is used to store :obj:`Responder ` trackers. + - ``WATCHER_LAST_BLOCK_KEY``, defined as ``b'bw``, is used to store the last block hash known by the :obj:`Watcher `. + - ``RESPONDER_LAST_BLOCK_KEY``, defined as ``b'br``, is used to store the last block hash known by the :obj:`Responder `. + - ``LOCATOR_MAP_PREFIX``, defined as ``b'm``, is used to store the ``locator:uuid`` maps. + - ``TRIGGERED_APPOINTMENTS_PREFIX``, defined as ``b'ta``, is used to stored triggered appointments (appointments that have been handed to the :obj:`Responder `.) + + Args: + db_path (:obj:`str`): the path (relative or absolute) to the system folder containing the database. A fresh + database will be create if the specified path does not contain one. + + Raises: + ValueError: If the provided ``db_path`` is not a string. + plyvel.Error: If the db is currently unavailable (being used by another process). + """ + + def __init__(self, db_path): + if not isinstance(db_path, str): + raise ValueError("db_path must be a valid path/name") + + try: + super().__init__(db_path) + + except plyvel.Error as e: + if "LOCK: Resource temporarily unavailable" in str(e): + logger.info("The db is already being used by another process (LOCK)") + + raise e + + def load_appointments_db(self, prefix): + """ + Loads all data from the appointments database given a prefix. Two prefixes are defined: ``WATCHER_PREFIX`` and + ``RESPONDER_PREFIX``. + + Args: + prefix (:obj:`str`): the prefix of the data to load. + + Returns: + :obj:`dict`: A dictionary containing the requested data (appointments or trackers) indexed by ``uuid``. + + Returns an empty dictionary if no data is found. + """ + + data = {} + + for k, v in self.db.iterator(prefix=prefix.encode("utf-8")): + # Get uuid and appointment_data from the db + uuid = k[len(prefix) :].decode("utf-8") + data[uuid] = json.loads(v) + + return data + + def get_last_known_block(self, key): + """ + Loads the last known block given a key (either ``WATCHER_LAST_BLOCK_KEY`` or ``RESPONDER_LAST_BLOCK_KEY``). + + Returns: + :obj:`str` or :obj:`None`: A 16-byte hex-encoded str representing the last known block hash. + + Returns ``None`` if the entry is not found. + """ + + last_block = self.db.get(key.encode("utf-8")) + + if last_block: + last_block = last_block.decode("utf-8") + + return last_block + + def load_watcher_appointment(self, key): + """ + Loads an appointment from the database using ``WATCHER_PREFIX`` as prefix to the given ``key``. + + Returns: + :obj:`dict`: A dictionary containing the appointment data if they ``key`` is found. + + Returns ``None`` otherwise. + """ + + data = self.load_entry(key, prefix=WATCHER_PREFIX) + + try: + data = json.loads(data) + except (TypeError, json.decoder.JSONDecodeError): + data = None + + return data + + def load_responder_tracker(self, key): + """ + Loads a tracker from the database using ``RESPONDER_PREFIX`` as a prefix to the given ``key``. + + Returns: + :obj:`dict`: A dictionary containing the tracker data if they ``key`` is found. + + Returns ``None`` otherwise. + """ + + data = self.load_entry(key, prefix=RESPONDER_PREFIX) + + try: + data = json.loads(data) + except (TypeError, json.decoder.JSONDecodeError): + data = None + + return data + + def load_watcher_appointments(self, include_triggered=False): + """ + Loads all the appointments from the database (all entries with the ``WATCHER_PREFIX`` prefix). + + Args: + include_triggered (:obj:`bool`): Whether to include the appointments flagged as triggered or not. ``False`` + by default. + + Returns: + :obj:`dict`: A dictionary with all the appointments stored in the database. An empty dictionary is there + are none. + """ + + appointments = self.load_appointments_db(prefix=WATCHER_PREFIX) + triggered_appointments = self.load_all_triggered_flags() + + if not include_triggered: + not_triggered = list(set(appointments.keys()).difference(triggered_appointments)) + appointments = {uuid: appointments[uuid] for uuid in not_triggered} + + return appointments + + def load_responder_trackers(self): + """ + Loads all the trackers from the database (all entries with the ``RESPONDER_PREFIX`` prefix). + + Returns: + :obj:`dict`: A dictionary with all the trackers stored in the database. An empty dictionary is there are + none. + """ + + return self.load_appointments_db(prefix=RESPONDER_PREFIX) + + def store_watcher_appointment(self, uuid, appointment): + """ + Stores an appointment in the database using the ``WATCHER_PREFIX`` prefix. + + Args: + uuid (:obj:`str`): the identifier of the appointment to be stored. + appointment (:obj: `dict`): an appointment encoded as dictionary. + """ + + self.create_entry(uuid, json.dumps(appointment), prefix=WATCHER_PREFIX) + logger.info("Adding appointment to Watchers's db", uuid=uuid) + + def store_responder_tracker(self, uuid, tracker): + """ + Stores a tracker in the database using the ``RESPONDER_PREFIX`` prefix. + + Args: + uuid (:obj:`str`): the identifier of the appointment to be stored. + tracker (:obj: `dict`): a tracker encoded as dictionary. + """ + + self.create_entry(uuid, json.dumps(tracker), prefix=RESPONDER_PREFIX) + logger.info("Adding appointment to Responder's db", uuid=uuid) + + def load_locator_map(self, locator): + """ + Loads the ``locator:uuid`` map of a given ``locator`` from the database. + + Args: + locator (:obj:`str`): a 16-byte hex-encoded string representing the appointment locator. + + Returns: + :obj:`dict` or :obj:`None`: The requested ``locator:uuid`` map if found. + + Returns ``None`` otherwise. + """ + + key = (LOCATOR_MAP_PREFIX + locator).encode("utf-8") + locator_map = self.db.get(key) + + if locator_map is not None: + locator_map = json.loads(locator_map.decode("utf-8")) + + else: + logger.info("Locator not found in the db", locator=locator) + + return locator_map + + def create_append_locator_map(self, locator, uuid): + """ + Creates (or appends to if already exists) a ``locator:uuid`` map. + + If the map already exists, the new ``uuid`` is appended to the existing ones (if it is not already there). + + Args: + locator (:obj:`str`): a 16-byte hex-encoded string used as the key of the map. + uuid (:obj:`str`): a 16-byte hex-encoded unique id to create (or add to) the map. + """ + + locator_map = self.load_locator_map(locator) + + if locator_map is not None: + if uuid not in locator_map: + locator_map.append(uuid) + logger.info("Updating locator map", locator=locator, uuid=uuid) + + else: + logger.info("UUID already in the map", locator=locator, uuid=uuid) + + else: + locator_map = [uuid] + logger.info("Creating new locator map", locator=locator, uuid=uuid) + + key = (LOCATOR_MAP_PREFIX + locator).encode("utf-8") + self.db.put(key, json.dumps(locator_map).encode("utf-8")) + + def update_locator_map(self, locator, locator_map): + """ + Updates a ``locator:uuid`` map in the database by deleting one of it's uuid. It will only work as long as + the given ``locator_map`` is a subset of the current one and it's not empty. + + Args: + locator (:obj:`str`): a 16-byte hex-encoded string used as the key of the map. + locator_map (:obj:`list`): a list of uuids to replace the current one on the db. + """ + + current_locator_map = self.load_locator_map(locator) + + if set(locator_map).issubset(current_locator_map) and len(locator_map) != 0: + key = (LOCATOR_MAP_PREFIX + locator).encode("utf-8") + self.db.put(key, json.dumps(locator_map).encode("utf-8")) + + else: + logger.error("Trying to update a locator_map with completely different, or empty, data") + + def delete_locator_map(self, locator): + """ + Deletes a ``locator:uuid`` map. + + Args: + locator (:obj:`str`): a 16-byte hex-encoded string identifying the map to delete. + """ + + self.delete_entry(locator, prefix=LOCATOR_MAP_PREFIX) + logger.info("Deleting locator map from db", uuid=locator) + + def delete_watcher_appointment(self, uuid): + """ + Deletes an appointment from the database. + + Args: + uuid (:obj:`str`): a 16-byte hex-encoded string identifying the appointment to be deleted. + """ + + self.delete_entry(uuid, prefix=WATCHER_PREFIX) + logger.info("Deleting appointment from Watcher's db", uuid=uuid) + + def batch_delete_watcher_appointments(self, uuids): + """ + Deletes an appointment from the database. + + Args: + uuids (:obj:`list`): a list of 16-byte hex-encoded strings identifying the appointments to be deleted. + """ + + with self.db.write_batch() as b: + for uuid in uuids: + b.delete((WATCHER_PREFIX + uuid).encode("utf-8")) + logger.info("Deleting appointment from Watcher's db", uuid=uuid) + + def delete_responder_tracker(self, uuid): + """ + Deletes a tracker from the database. + + Args: + uuid (:obj:`str`): a 16-byte hex-encoded string identifying the tracker to be deleted. + """ + + self.delete_entry(uuid, prefix=RESPONDER_PREFIX) + logger.info("Deleting appointment from Responder's db", uuid=uuid) + + def batch_delete_responder_trackers(self, uuids): + """ + Deletes an appointment from the database. + + Args: + uuids (:obj:`list`): a list of 16-byte hex-encoded strings identifying the trackers to be deleted. + """ + + with self.db.write_batch() as b: + for uuid in uuids: + b.delete((RESPONDER_PREFIX + uuid).encode("utf-8")) + logger.info("Deleting appointment from Responder's db", uuid=uuid) + + def load_last_block_hash_watcher(self): + """ + Loads the last known block hash of the :obj:`Watcher ` from the database. + + Returns: + :obj:`str` or :obj:`None`: A 32-byte hex-encoded string representing the last known block hash if found. + + Returns ``None`` otherwise. + """ + return self.get_last_known_block(WATCHER_LAST_BLOCK_KEY) + + def load_last_block_hash_responder(self): + """ + Loads the last known block hash of the :obj:`Responder ` from the database. + + Returns: + :obj:`str` or :obj:`None`: A 32-byte hex-encoded string representing the last known block hash if found. + + Returns ``None`` otherwise. + """ + return self.get_last_known_block(RESPONDER_LAST_BLOCK_KEY) + + def store_last_block_hash_watcher(self, block_hash): + """ + Stores a block hash as the last known block of the :obj:`Watcher `. + + Args: + block_hash (:obj:`str`): the block hash to be stored (32-byte hex-encoded) + """ + + self.create_entry(WATCHER_LAST_BLOCK_KEY, block_hash) + + def store_last_block_hash_responder(self, block_hash): + """ + Stores a block hash as the last known block of the :obj:`Responder `. + + Args: + block_hash (:obj:`str`): the block hash to be stored (32-byte hex-encoded) + """ + + self.create_entry(RESPONDER_LAST_BLOCK_KEY, block_hash) + + def create_triggered_appointment_flag(self, uuid): + """ + Creates a flag that signals that an appointment has been triggered. + + Args: + uuid (:obj:`str`): the identifier of the flag to be created. + """ + + self.db.put((TRIGGERED_APPOINTMENTS_PREFIX + uuid).encode("utf-8"), "".encode("utf-8")) + logger.info("Flagging appointment as triggered", uuid=uuid) + + def batch_create_triggered_appointment_flag(self, uuids): + """ + Creates a flag that signals that an appointment has been triggered for every appointment in the given list + + Args: + uuids (:obj:`list`): a list of identifier for the appointments to flag. + """ + + with self.db.write_batch() as b: + for uuid in uuids: + b.put((TRIGGERED_APPOINTMENTS_PREFIX + uuid).encode("utf-8"), b"") + logger.info("Flagging appointment as triggered", uuid=uuid) + + def load_all_triggered_flags(self): + """ + Loads all the appointment triggered flags from the database. + + Returns: + :obj:`list`: a list of all the uuids of the triggered appointments. + """ + + return [ + k.decode()[len(TRIGGERED_APPOINTMENTS_PREFIX) :] + for k, v in self.db.iterator(prefix=TRIGGERED_APPOINTMENTS_PREFIX.encode("utf-8")) + ] + + def delete_triggered_appointment_flag(self, uuid): + """ + Deletes a flag that signals that an appointment has been triggered. + + Args: + uuid (:obj:`str`): the identifier of the flag to be removed. + """ + + self.delete_entry(uuid, prefix=TRIGGERED_APPOINTMENTS_PREFIX) + logger.info("Removing triggered flag from appointment appointment", uuid=uuid) + + def batch_delete_triggered_appointment_flag(self, uuids): + """ + Deletes a list of flag signaling that some appointment have been triggered. + + Args: + uuids (:obj:`list`): the identifier of the flag to be removed. + """ + + with self.db.write_batch() as b: + for uuid in uuids: + b.delete((TRIGGERED_APPOINTMENTS_PREFIX + uuid).encode("utf-8")) + logger.info("Removing triggered flag from appointment appointment", uuid=uuid) diff --git a/teos/cleaner.py b/teos/cleaner.py index 25bd988..e0aefdc 100644 --- a/teos/cleaner.py +++ b/teos/cleaner.py @@ -43,8 +43,8 @@ class Cleaner: Args: uuid (:obj:`str`): the identifier of the appointment to be deleted. - db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the - database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. """ db_manager.delete_watcher_appointment(uuid) @@ -61,8 +61,8 @@ class Cleaner: Args: uuids (:obj:`list`): a list of identifiers to be removed from the map. locator (:obj:`str`): the identifier of the map to be either updated or deleted. - db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the - database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. """ locator_map = db_manager.load_locator_map(locator) @@ -95,8 +95,8 @@ class Cleaner: appointments. locator_uuid_map (:obj:`dict`): a ``locator:uuid`` map for the :obj:`Watcher ` appointments. - db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the - database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. """ locator_maps_to_update = {} @@ -133,8 +133,8 @@ class Cleaner: appointments. locator_uuid_map (:obj:`dict`): a ``locator:uuid`` map for the :obj:`Watcher ` appointments. - db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the - database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. """ locator_maps_to_update = {} @@ -170,8 +170,8 @@ class Cleaner: appointments. locator_uuid_map (:obj:`dict`): a ``locator:uuid`` map for the :obj:`Watcher ` appointments. - db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the - database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. """ for uuid in triggered_appointments: @@ -191,8 +191,8 @@ class Cleaner: ` trackers. completed_trackers (:obj:`dict`): a dict of completed trackers to be deleted (uuid:confirmations). height (:obj:`int`): the block height at which the trackers were completed. - db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the - database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. """ locator_maps_to_update = {} diff --git a/teos/db_manager.py b/teos/db_manager.py index 8b743cd..d911a4b 100644 --- a/teos/db_manager.py +++ b/teos/db_manager.py @@ -1,34 +1,11 @@ -import json import plyvel -from teos import LOG_PREFIX - -from common.logger import Logger - -logger = Logger(actor="DBManager", log_name_prefix=LOG_PREFIX) - -WATCHER_PREFIX = "w" -WATCHER_LAST_BLOCK_KEY = "bw" -RESPONDER_PREFIX = "r" -RESPONDER_LAST_BLOCK_KEY = "br" -LOCATOR_MAP_PREFIX = "m" -TRIGGERED_APPOINTMENTS_PREFIX = "ta" - class DBManager: """ - The :class:`DBManager` is the class in charge of interacting with the appointments database (``LevelDB``). + The :class:`DBManager` is the class in charge of interacting with a database (``LevelDB``). Keys and values are stored as bytes in the database but processed as strings by the manager. - The database is split in six prefixes: - - - ``WATCHER_PREFIX``, defined as ``b'w``, is used to store :obj:`Watcher ` appointments. - - ``RESPONDER_PREFIX``, defines as ``b'r``, is used to store :obj:`Responder ` trackers. - - ``WATCHER_LAST_BLOCK_KEY``, defined as ``b'bw``, is used to store the last block hash known by the :obj:`Watcher `. - - ``RESPONDER_LAST_BLOCK_KEY``, defined as ``b'br``, is used to store the last block hash known by the :obj:`Responder `. - - ``LOCATOR_MAP_PREFIX``, defined as ``b'm``, is used to store the ``locator:uuid`` maps. - - ``TRIGGERED_APPOINTMENTS_PREFIX``, defined as ``b'ta``, is used to stored triggered appointments (appointments that have been handed to the :obj:`Responder `.) - Args: db_path (:obj:`str`): the path (relative or absolute) to the system folder containing the database. A fresh database will be create if the specified path does not contain one. @@ -42,57 +19,7 @@ class DBManager: if not isinstance(db_path, str): raise ValueError("db_path must be a valid path/name") - try: - self.db = plyvel.DB(db_path) - - except plyvel.Error as e: - if "create_if_missing is false" in str(e): - logger.info("No db found. Creating a fresh one") - self.db = plyvel.DB(db_path, create_if_missing=True) - - elif "LOCK: Resource temporarily unavailable" in str(e): - logger.info("The db is already being used by another process (LOCK)") - raise e - - def load_appointments_db(self, prefix): - """ - Loads all data from the appointments database given a prefix. Two prefixes are defined: ``WATCHER_PREFIX`` and - ``RESPONDER_PREFIX``. - - Args: - prefix (:obj:`str`): the prefix of the data to load. - - Returns: - :obj:`dict`: A dictionary containing the requested data (appointments or trackers) indexed by ``uuid``. - - Returns an empty dictionary if no data is found. - """ - - data = {} - - for k, v in self.db.iterator(prefix=prefix.encode("utf-8")): - # Get uuid and appointment_data from the db - uuid = k[len(prefix) :].decode("utf-8") - data[uuid] = json.loads(v) - - return data - - def get_last_known_block(self, key): - """ - Loads the last known block given a key (either ``WATCHER_LAST_BLOCK_KEY`` or ``RESPONDER_LAST_BLOCK_KEY``). - - Returns: - :obj:`str` or :obj:`None`: A 16-byte hex-encoded str representing the last known block hash. - - Returns ``None`` if the entry is not found. - """ - - last_block = self.db.get(key.encode("utf-8")) - - if last_block: - last_block = last_block.decode("utf-8") - - return last_block + self.db = plyvel.DB(db_path, create_if_missing=True) def create_entry(self, key, value, prefix=None): """ @@ -102,8 +29,20 @@ class DBManager: key (:obj:`str`): the key of the new entry, used to identify it. value (:obj:`str`): the data stored under the given ``key``. prefix (:obj:`str`): an optional prefix added to the ``key``. + + Raises: + (:obj:`TypeError`) if key, value or prefix are not strings. """ + if not isinstance(key, str): + raise TypeError("Key must be str") + + if not isinstance(value, str): + raise TypeError("Value must be str") + + if not isinstance(prefix, str) and prefix is not None: + raise TypeError("Prefix (if set) must be str") + if isinstance(prefix, str): key = prefix + key @@ -112,349 +51,55 @@ class DBManager: self.db.put(key, value) - def load_entry(self, key): + def load_entry(self, key, prefix=None): """ - Loads an entry from the database given a ``key``. + Loads an entry from the database given a ``key`` (and optionally a ``prefix``). Args: key (:obj:`str`): the key that identifies the entry to be loaded. + prefix (:obj:`str`): an optional prefix added to the ``key``. Returns: - :obj:`dict` or :obj:`None`: A dictionary containing the requested data (an appointment or a tracker). + :obj:`bytes` or :obj:`None`: A byte-array containing the requested data. Returns ``None`` if the entry is not found. + + Raises: + (:obj:`TypeError`) if key or prefix are not strings. """ - data = self.db.get(key.encode("utf-8")) - data = json.loads(data) if data is not None else data - return data + if not isinstance(key, str): + raise TypeError("Key must be str") + + if not isinstance(prefix, str) and prefix is not None: + raise TypeError("Prefix (if set) must be str") + + if isinstance(prefix, str): + key = prefix + key + + return self.db.get(key.encode("utf-8")) def delete_entry(self, key, prefix=None): """ - Deletes an entry from the database given an ``key`` (and optionally a ``prefix``) + Deletes an entry from the database given an ``key`` (and optionally a ``prefix``). Args: key (:obj:`str`): the key that identifies the data to be deleted. prefix (:obj:`str`): an optional prefix to be prepended to the ``key``. + + Raises: + (:obj:`TypeError`) if key or prefix are not strings. """ + if not isinstance(key, str): + raise TypeError("Key must be str") + + if not isinstance(prefix, str) and prefix is not None: + raise TypeError("Prefix (if set) must be str") + if isinstance(prefix, str): key = prefix + key key = key.encode("utf-8") self.db.delete(key) - - def load_watcher_appointment(self, key): - """ - Loads an appointment from the database using ``WATCHER_PREFIX`` as prefix to the given ``key``. - - Returns: - :obj:`dict`: A dictionary containing the appointment data if they ``key`` is found. - - Returns ``None`` otherwise. - """ - - return self.load_entry(WATCHER_PREFIX + key) - - def load_responder_tracker(self, key): - """ - Loads a tracker from the database using ``RESPONDER_PREFIX`` as a prefix to the given ``key``. - - Returns: - :obj:`dict`: A dictionary containing the tracker data if they ``key`` is found. - - Returns ``None`` otherwise. - """ - - return self.load_entry(RESPONDER_PREFIX + key) - - def load_watcher_appointments(self, include_triggered=False): - """ - Loads all the appointments from the database (all entries with the ``WATCHER_PREFIX`` prefix). - - Args: - include_triggered (:obj:`bool`): Whether to include the appointments flagged as triggered or not. ``False`` - by default. - - Returns: - :obj:`dict`: A dictionary with all the appointments stored in the database. An empty dictionary is there - are none. - """ - - appointments = self.load_appointments_db(prefix=WATCHER_PREFIX) - triggered_appointments = self.load_all_triggered_flags() - - if not include_triggered: - not_triggered = list(set(appointments.keys()).difference(triggered_appointments)) - appointments = {uuid: appointments[uuid] for uuid in not_triggered} - - return appointments - - def load_responder_trackers(self): - """ - Loads all the trackers from the database (all entries with the ``RESPONDER_PREFIX`` prefix). - - Returns: - :obj:`dict`: A dictionary with all the trackers stored in the database. An empty dictionary is there are - none. - """ - - return self.load_appointments_db(prefix=RESPONDER_PREFIX) - - def store_watcher_appointment(self, uuid, appointment): - """ - Stores an appointment in the database using the ``WATCHER_PREFIX`` prefix. - - Args: - uuid (:obj:`str`): the identifier of the appointment to be stored. - appointment (:obj: `str`): the json encoded appointment to be stored as data. - """ - - self.create_entry(uuid, appointment, prefix=WATCHER_PREFIX) - logger.info("Adding appointment to Watchers's db", uuid=uuid) - - def store_responder_tracker(self, uuid, tracker): - """ - Stores a tracker in the database using the ``RESPONDER_PREFIX`` prefix. - - Args: - uuid (:obj:`str`): the identifier of the appointment to be stored. - tracker (:obj: `str`): the json encoded tracker to be stored as data. - """ - - self.create_entry(uuid, tracker, prefix=RESPONDER_PREFIX) - logger.info("Adding appointment to Responder's db", uuid=uuid) - - def load_locator_map(self, locator): - """ - Loads the ``locator:uuid`` map of a given ``locator`` from the database. - - Args: - locator (:obj:`str`): a 16-byte hex-encoded string representing the appointment locator. - - Returns: - :obj:`dict` or :obj:`None`: The requested ``locator:uuid`` map if found. - - Returns ``None`` otherwise. - """ - - key = (LOCATOR_MAP_PREFIX + locator).encode("utf-8") - locator_map = self.db.get(key) - - if locator_map is not None: - locator_map = json.loads(locator_map.decode("utf-8")) - - else: - logger.info("Locator not found in the db", locator=locator) - - return locator_map - - def create_append_locator_map(self, locator, uuid): - """ - Creates (or appends to if already exists) a ``locator:uuid`` map. - - If the map already exists, the new ``uuid`` is appended to the existing ones (if it is not already there). - - Args: - locator (:obj:`str`): a 16-byte hex-encoded string used as the key of the map. - uuid (:obj:`str`): a 16-byte hex-encoded unique id to create (or add to) the map. - """ - - locator_map = self.load_locator_map(locator) - - if locator_map is not None: - if uuid not in locator_map: - locator_map.append(uuid) - logger.info("Updating locator map", locator=locator, uuid=uuid) - - else: - logger.info("UUID already in the map", locator=locator, uuid=uuid) - - else: - locator_map = [uuid] - logger.info("Creating new locator map", locator=locator, uuid=uuid) - - key = (LOCATOR_MAP_PREFIX + locator).encode("utf-8") - self.db.put(key, json.dumps(locator_map).encode("utf-8")) - - def update_locator_map(self, locator, locator_map): - """ - Updates a ``locator:uuid`` map in the database by deleting one of it's uuid. It will only work as long as - the given ``locator_map`` is a subset of the current one and it's not empty. - - Args: - locator (:obj:`str`): a 16-byte hex-encoded string used as the key of the map. - locator_map (:obj:`list`): a list of uuids to replace the current one on the db. - """ - - current_locator_map = self.load_locator_map(locator) - - if set(locator_map).issubset(current_locator_map) and len(locator_map) != 0: - key = (LOCATOR_MAP_PREFIX + locator).encode("utf-8") - self.db.put(key, json.dumps(locator_map).encode("utf-8")) - - else: - logger.error("Trying to update a locator_map with completely different, or empty, data") - - def delete_locator_map(self, locator): - """ - Deletes a ``locator:uuid`` map. - - Args: - locator (:obj:`str`): a 16-byte hex-encoded string identifying the map to delete. - """ - - self.delete_entry(locator, prefix=LOCATOR_MAP_PREFIX) - logger.info("Deleting locator map from db", uuid=locator) - - def delete_watcher_appointment(self, uuid): - """ - Deletes an appointment from the database. - - Args: - uuid (:obj:`str`): a 16-byte hex-encoded string identifying the appointment to be deleted. - """ - - self.delete_entry(uuid, prefix=WATCHER_PREFIX) - logger.info("Deleting appointment from Watcher's db", uuid=uuid) - - def batch_delete_watcher_appointments(self, uuids): - """ - Deletes an appointment from the database. - - Args: - uuids (:obj:`list`): a list of 16-byte hex-encoded strings identifying the appointments to be deleted. - """ - - with self.db.write_batch() as b: - for uuid in uuids: - b.delete((WATCHER_PREFIX + uuid).encode("utf-8")) - logger.info("Deleting appointment from Watcher's db", uuid=uuid) - - def delete_responder_tracker(self, uuid): - """ - Deletes a tracker from the database. - - Args: - uuid (:obj:`str`): a 16-byte hex-encoded string identifying the tracker to be deleted. - """ - - self.delete_entry(uuid, prefix=RESPONDER_PREFIX) - logger.info("Deleting appointment from Responder's db", uuid=uuid) - - def batch_delete_responder_trackers(self, uuids): - """ - Deletes an appointment from the database. - - Args: - uuids (:obj:`list`): a list of 16-byte hex-encoded strings identifying the trackers to be deleted. - """ - - with self.db.write_batch() as b: - for uuid in uuids: - b.delete((RESPONDER_PREFIX + uuid).encode("utf-8")) - logger.info("Deleting appointment from Responder's db", uuid=uuid) - - def load_last_block_hash_watcher(self): - """ - Loads the last known block hash of the :obj:`Watcher ` from the database. - - Returns: - :obj:`str` or :obj:`None`: A 32-byte hex-encoded string representing the last known block hash if found. - - Returns ``None`` otherwise. - """ - return self.get_last_known_block(WATCHER_LAST_BLOCK_KEY) - - def load_last_block_hash_responder(self): - """ - Loads the last known block hash of the :obj:`Responder ` from the database. - - Returns: - :obj:`str` or :obj:`None`: A 32-byte hex-encoded string representing the last known block hash if found. - - Returns ``None`` otherwise. - """ - return self.get_last_known_block(RESPONDER_LAST_BLOCK_KEY) - - def store_last_block_hash_watcher(self, block_hash): - """ - Stores a block hash as the last known block of the :obj:`Watcher `. - - Args: - block_hash (:obj:`str`): the block hash to be stored (32-byte hex-encoded) - """ - - self.create_entry(WATCHER_LAST_BLOCK_KEY, block_hash) - - def store_last_block_hash_responder(self, block_hash): - """ - Stores a block hash as the last known block of the :obj:`Responder `. - - Args: - block_hash (:obj:`str`): the block hash to be stored (32-byte hex-encoded) - """ - - self.create_entry(RESPONDER_LAST_BLOCK_KEY, block_hash) - - def create_triggered_appointment_flag(self, uuid): - """ - Creates a flag that signals that an appointment has been triggered. - - Args: - uuid (:obj:`str`): the identifier of the flag to be created. - """ - - self.db.put((TRIGGERED_APPOINTMENTS_PREFIX + uuid).encode("utf-8"), "".encode("utf-8")) - logger.info("Flagging appointment as triggered", uuid=uuid) - - def batch_create_triggered_appointment_flag(self, uuids): - """ - Creates a flag that signals that an appointment has been triggered for every appointment in the given list - - Args: - uuids (:obj:`list`): a list of identifier for the appointments to flag. - """ - - with self.db.write_batch() as b: - for uuid in uuids: - b.put((TRIGGERED_APPOINTMENTS_PREFIX + uuid).encode("utf-8"), b"") - logger.info("Flagging appointment as triggered", uuid=uuid) - - def load_all_triggered_flags(self): - """ - Loads all the appointment triggered flags from the database. - - Returns: - :obj:`list`: a list of all the uuids of the triggered appointments. - """ - - return [ - k.decode()[len(TRIGGERED_APPOINTMENTS_PREFIX) :] - for k, v in self.db.iterator(prefix=TRIGGERED_APPOINTMENTS_PREFIX.encode("utf-8")) - ] - - def delete_triggered_appointment_flag(self, uuid): - """ - Deletes a flag that signals that an appointment has been triggered. - - Args: - uuid (:obj:`str`): the identifier of the flag to be removed. - """ - - self.delete_entry(uuid, prefix=TRIGGERED_APPOINTMENTS_PREFIX) - logger.info("Removing triggered flag from appointment appointment", uuid=uuid) - - def batch_delete_triggered_appointment_flag(self, uuids): - """ - Deletes a list of flag signaling that some appointment have been triggered. - - Args: - uuids (:obj:`list`): the identifier of the flag to be removed. - """ - - with self.db.write_batch() as b: - for uuid in uuids: - b.delete((TRIGGERED_APPOINTMENTS_PREFIX + uuid).encode("utf-8")) - logger.info("Removing triggered flag from appointment appointment", uuid=uuid) diff --git a/teos/responder.py b/teos/responder.py index 4ecdaa6..9cbb21a 100644 --- a/teos/responder.py +++ b/teos/responder.py @@ -89,16 +89,6 @@ class TransactionTracker: return tx_tracker - def to_json(self): - """ - Exports a :obj:`TransactionTracker` as a json-encoded dictionary. - - Returns: - :obj:`str`: A json-encoded dictionary containing the :obj:`TransactionTracker` data. - """ - - return json.dumps(self.to_dict()) - class Responder: """ @@ -107,8 +97,8 @@ class Responder: the blockchain. Args: - db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the - database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. Attributes: trackers (:obj:`dict`): A dictionary containing the minimum information about the :obj:`TransactionTracker` @@ -121,11 +111,11 @@ class Responder: has missed. Used to trigger rebroadcast if needed. block_queue (:obj:`Queue`): A queue used by the :obj:`Responder` to receive block hashes from ``bitcoind``. It is populated by the :obj:`ChainMonitor `. - db_manager (:obj:`DBManager `): A ``DBManager`` instance to interact with the - database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. carrier (:obj:`Carrier `): a ``Carrier`` instance to send transactions to bitcoind. - block_processor (:obj:`DBManager `): a ``BlockProcessor`` instance to get - data from bitcoind. + block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance to + get data from bitcoind. last_known_block (:obj:`str`): the last block known by the ``Responder``. """ @@ -251,7 +241,7 @@ class Responder: if penalty_txid not in self.unconfirmed_txs and confirmations == 0: self.unconfirmed_txs.append(penalty_txid) - self.db_manager.store_responder_tracker(uuid, tracker.to_json()) + self.db_manager.store_responder_tracker(uuid, tracker.to_dict()) logger.info( "New tracker added", dispute_txid=dispute_txid, penalty_txid=penalty_txid, appointment_end=appointment_end diff --git a/teos/teosd.py b/teos/teosd.py index 72646fe..f23447a 100644 --- a/teos/teosd.py +++ b/teos/teosd.py @@ -16,7 +16,7 @@ from teos.builder import Builder from teos.carrier import Carrier from teos.inspector import Inspector from teos.responder import Responder -from teos.db_manager import DBManager +from teos.appointments_dbm import AppointmentsDBM from teos.gatekeeper import Gatekeeper from teos.chain_monitor import ChainMonitor from teos.block_processor import BlockProcessor @@ -50,7 +50,7 @@ def main(command_line_conf): setup_logging(config.get("LOG_FILE"), LOG_PREFIX) logger.info("Starting TEOS") - db_manager = DBManager(config.get("DB_PATH")) + db_manager = AppointmentsDBM(config.get("APPOINTMENTS_DB_PATH")) bitcoind_connect_params = {k: v for k, v in config.items() if k.startswith("BTC")} bitcoind_feed_params = {k: v for k, v in config.items() if k.startswith("FEED")} diff --git a/teos/watcher.py b/teos/watcher.py index 055c3c4..79c5cfe 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -30,7 +30,8 @@ class Watcher: :obj:`ChainMonitor `. Args: - db_manager (:obj:`DBManager `): a ``DBManager`` instance to interact with the database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance to get block from bitcoind. responder (:obj:`Responder `): a ``Responder`` instance. @@ -46,7 +47,8 @@ class Watcher: appointments with the same ``locator``. block_queue (:obj:`Queue`): A queue used by the :obj:`Watcher` to receive block hashes from ``bitcoind``. It is populated by the :obj:`ChainMonitor `. - db_manager (:obj:`DBManager `): A db manager instance to interact with the database. + db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance + to interact with the database. block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance to get block from bitcoind. responder (:obj:`Responder `): a ``Responder`` instance. @@ -144,7 +146,7 @@ class Watcher: else: self.locator_uuid_map[appointment.locator] = [uuid] - self.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + self.db_manager.store_watcher_appointment(uuid, appointment.to_dict()) self.db_manager.create_append_locator_map(appointment.locator, uuid) appointment_added = True From 4a3f4bc5229cac00a07baccfc06e936c410fab87 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 17:19:41 +0200 Subject: [PATCH 50/62] Addapts tests to match 7031b552f7a03853187f7e9c62f612e2367c6f1b. Improves DBManager tests. --- test/common/unit/test_appointment.py | 20 -- test/teos/unit/conftest.py | 4 +- test/teos/unit/test_api.py | 14 +- test/teos/unit/test_appointments_dbm.py | 380 +++++++++++++++++++++ test/teos/unit/test_cleaner.py | 8 +- test/teos/unit/test_db_manager.py | 419 +++--------------------- test/teos/unit/test_responder.py | 19 +- test/teos/unit/test_watcher.py | 10 +- 8 files changed, 449 insertions(+), 425 deletions(-) create mode 100644 test/teos/unit/test_appointments_dbm.py diff --git a/test/common/unit/test_appointment.py b/test/common/unit/test_appointment.py index 9018505..36488e7 100644 --- a/test/common/unit/test_appointment.py +++ b/test/common/unit/test_appointment.py @@ -71,26 +71,6 @@ def test_to_dict(appointment_data): ) -def test_to_json(appointment_data): - appointment = Appointment( - appointment_data["locator"], - appointment_data["start_time"], - appointment_data["end_time"], - appointment_data["to_self_delay"], - appointment_data["encrypted_blob"], - ) - - dict_appointment = json.loads(appointment.to_json()) - - assert ( - appointment_data["locator"] == dict_appointment["locator"] - and appointment_data["start_time"] == dict_appointment["start_time"] - and appointment_data["end_time"] == dict_appointment["end_time"] - and appointment_data["to_self_delay"] == dict_appointment["to_self_delay"] - and EncryptedBlob(appointment_data["encrypted_blob"]) == EncryptedBlob(dict_appointment["encrypted_blob"]) - ) - - def test_from_dict(appointment_data): # The appointment should be build if we don't miss any field appointment = Appointment.from_dict(appointment_data) diff --git a/test/teos/unit/conftest.py b/test/teos/unit/conftest.py index 52d95f7..6076b52 100644 --- a/test/teos/unit/conftest.py +++ b/test/teos/unit/conftest.py @@ -12,10 +12,10 @@ from bitcoind_mock.transaction import create_dummy_transaction from teos.carrier import Carrier from teos.tools import bitcoin_cli -from teos.db_manager import DBManager from teos import LOG_PREFIX, DEFAULT_CONF from teos.responder import TransactionTracker from teos.block_processor import BlockProcessor +from teos.appointments_dbm import AppointmentsDBM import common.cryptographer from common.blob import Blob @@ -53,7 +53,7 @@ def prng_seed(): @pytest.fixture(scope="module") def db_manager(): - manager = DBManager("test_db") + manager = AppointmentsDBM("test_db") # Add last know block for the Responder in the db yield manager diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 1013685..00b0844 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -7,8 +7,8 @@ from teos import HOST, PORT import teos.errors as errors from teos.watcher import Watcher from teos.inspector import Inspector -from teos.db_manager import DBManager from teos.gatekeeper import Gatekeeper +from teos.appointments_dbm import AppointmentsDBM from teos.responder import Responder, TransactionTracker from test.teos.unit.conftest import get_random_value_hex, generate_dummy_appointment, generate_keypair, get_config @@ -48,7 +48,7 @@ compressed_client_pk = hexlify(client_pk.format(compressed=True)).decode("utf-8" @pytest.fixture() def get_all_db_manager(): - manager = DBManager("get_all_tmp_db") + manager = AppointmentsDBM("get_all_tmp_db") # Add last know block for the Responder in the db yield manager @@ -396,7 +396,7 @@ def test_request_appointment_not_registered_user(client): def test_request_appointment_in_watcher(api, client, appointment): # Mock the appointment in the Watcher uuid = hash_160("{}{}".format(appointment.locator, compressed_client_pk)) - api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_dict()) # Next we can request it message = "get appointment {}".format(appointment.locator) @@ -426,7 +426,7 @@ def test_request_appointment_in_responder(api, client, appointment): uuid = hash_160("{}{}".format(appointment.locator, compressed_client_pk)) api.watcher.db_manager.create_triggered_appointment_flag(uuid) - api.watcher.responder.db_manager.store_responder_tracker(uuid, tx_tracker.to_json()) + api.watcher.responder.db_manager.store_responder_tracker(uuid, tx_tracker.to_dict()) # Request back the data message = "get appointment {}".format(appointment.locator) @@ -464,14 +464,14 @@ def test_get_all_appointments_watcher(api, client, get_all_db_manager, appointme uuid = get_random_value_hex(16) appointment.locator = get_random_value_hex(16) non_triggered_appointments[uuid] = appointment.to_dict() - api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_dict()) triggered_appointments = {} for _ in range(10): uuid = get_random_value_hex(16) appointment.locator = get_random_value_hex(16) triggered_appointments[uuid] = appointment.to_dict() - api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_dict()) api.watcher.db_manager.create_triggered_appointment_flag(uuid) # We should only get check the non-triggered appointments @@ -508,7 +508,7 @@ def test_get_all_appointments_responder(api, client, get_all_db_manager): } tracker = TransactionTracker.from_dict(tracker_data) tx_trackers[uuid] = tracker.to_dict() - api.watcher.responder.db_manager.store_responder_tracker(uuid, tracker.to_json()) + api.watcher.responder.db_manager.store_responder_tracker(uuid, tracker.to_dict()) api.watcher.db_manager.create_triggered_appointment_flag(uuid) # Get all appointments diff --git a/test/teos/unit/test_appointments_dbm.py b/test/teos/unit/test_appointments_dbm.py new file mode 100644 index 0000000..b6cd69f --- /dev/null +++ b/test/teos/unit/test_appointments_dbm.py @@ -0,0 +1,380 @@ +import os +import json +import pytest +import shutil +from uuid import uuid4 + +from teos.appointments_dbm import AppointmentsDBM +from teos.appointments_dbm import ( + WATCHER_LAST_BLOCK_KEY, + RESPONDER_LAST_BLOCK_KEY, + LOCATOR_MAP_PREFIX, + TRIGGERED_APPOINTMENTS_PREFIX, +) + +from common.constants import LOCATOR_LEN_BYTES + +from test.teos.unit.conftest import get_random_value_hex, generate_dummy_appointment + + +@pytest.fixture(scope="module") +def watcher_appointments(): + return {uuid4().hex: generate_dummy_appointment(real_height=False)[0] for _ in range(10)} + + +@pytest.fixture(scope="module") +def responder_trackers(): + return {get_random_value_hex(16): get_random_value_hex(32) for _ in range(10)} + + +def open_create_db(db_path): + + try: + db_manager = AppointmentsDBM(db_path) + + return db_manager + + except ValueError: + return False + + +def test_load_appointments_db(db_manager): + # Let's made up a prefix and try to load data from the database using it + prefix = "XX" + db_appointments = db_manager.load_appointments_db(prefix) + + assert len(db_appointments) == 0 + + # We can add a bunch of data to the db and try again (data is stored in json by the manager) + local_appointments = {} + for _ in range(10): + key = get_random_value_hex(16) + value = get_random_value_hex(32) + local_appointments[key] = value + + db_manager.db.put((prefix + key).encode("utf-8"), json.dumps({"value": value}).encode("utf-8")) + + db_appointments = db_manager.load_appointments_db(prefix) + + # Check that both keys and values are the same + assert db_appointments.keys() == local_appointments.keys() + + values = [appointment["value"] for appointment in db_appointments.values()] + assert set(values) == set(local_appointments.values()) and (len(values) == len(local_appointments)) + + +def test_get_last_known_block(): + db_path = "empty_db" + + # First we check if the db exists, and if so we delete it + if os.path.isdir(db_path): + shutil.rmtree(db_path) + + # Check that the db can be created if it does not exist + db_manager = open_create_db(db_path) + + # Trying to get any last block for either the watcher or the responder should return None for an empty db + + for key in [WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY]: + assert db_manager.get_last_known_block(key) is None + + # After saving some block in the db we should get that exact value + for key in [WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY]: + block_hash = get_random_value_hex(32) + db_manager.db.put(key.encode("utf-8"), block_hash.encode("utf-8")) + assert db_manager.get_last_known_block(key) == block_hash + + # Removing test db + shutil.rmtree(db_path) + + +def test_load_watcher_appointments_empty(db_manager): + assert len(db_manager.load_watcher_appointments()) == 0 + + +def test_load_responder_trackers_empty(db_manager): + assert len(db_manager.load_responder_trackers()) == 0 + + +def test_load_locator_map_empty(db_manager): + assert db_manager.load_locator_map(get_random_value_hex(LOCATOR_LEN_BYTES)) is None + + +def test_create_append_locator_map(db_manager): + uuid = uuid4().hex + locator = get_random_value_hex(LOCATOR_LEN_BYTES) + db_manager.create_append_locator_map(locator, uuid) + + # Check that the locator map has been properly stored + assert db_manager.load_locator_map(locator) == [uuid] + + # If we try to add the same uuid again the list shouldn't change + db_manager.create_append_locator_map(locator, uuid) + assert db_manager.load_locator_map(locator) == [uuid] + + # Add another uuid to the same locator and check that it also works + uuid2 = uuid4().hex + db_manager.create_append_locator_map(locator, uuid2) + + assert set(db_manager.load_locator_map(locator)) == set([uuid, uuid2]) + + +def test_update_locator_map(db_manager): + # Let's create a couple of appointments with the same locator + locator = get_random_value_hex(32) + uuid1 = uuid4().hex + uuid2 = uuid4().hex + db_manager.create_append_locator_map(locator, uuid1) + db_manager.create_append_locator_map(locator, uuid2) + + locator_map = db_manager.load_locator_map(locator) + assert uuid1 in locator_map + + locator_map.remove(uuid1) + db_manager.update_locator_map(locator, locator_map) + + locator_map_after = db_manager.load_locator_map(locator) + assert uuid1 not in locator_map_after and uuid2 in locator_map_after and len(locator_map_after) == 1 + + +def test_update_locator_map_wong_data(db_manager): + # Let's try to update the locator map with a different list of uuids + locator = get_random_value_hex(32) + db_manager.create_append_locator_map(locator, uuid4().hex) + db_manager.create_append_locator_map(locator, uuid4().hex) + + locator_map = db_manager.load_locator_map(locator) + wrong_map_update = [uuid4().hex] + db_manager.update_locator_map(locator, wrong_map_update) + locator_map_after = db_manager.load_locator_map(locator) + + assert locator_map_after == locator_map + + +def test_update_locator_map_empty(db_manager): + # We shouldn't be able to update a map with an empty list + locator = get_random_value_hex(32) + db_manager.create_append_locator_map(locator, uuid4().hex) + db_manager.create_append_locator_map(locator, uuid4().hex) + + locator_map = db_manager.load_locator_map(locator) + db_manager.update_locator_map(locator, []) + locator_map_after = db_manager.load_locator_map(locator) + + assert locator_map_after == locator_map + + +def test_delete_locator_map(db_manager): + locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX) + assert len(locator_maps) != 0 + + for locator, uuids in locator_maps.items(): + db_manager.delete_locator_map(locator) + + locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX) + assert len(locator_maps) == 0 + + +def test_store_load_watcher_appointment(db_manager, watcher_appointments): + for uuid, appointment in watcher_appointments.items(): + db_manager.store_watcher_appointment(uuid, appointment.to_dict()) + + db_watcher_appointments = db_manager.load_watcher_appointments() + + # Check that the two appointment collections are equal by checking: + # - Their size is equal + # - Each element in one collection exists in the other + + assert watcher_appointments.keys() == db_watcher_appointments.keys() + + for uuid, appointment in watcher_appointments.items(): + assert appointment.to_dict() == db_watcher_appointments[uuid] + + +def test_store_load_triggered_appointment(db_manager): + db_watcher_appointments = db_manager.load_watcher_appointments() + db_watcher_appointments_with_triggered = db_manager.load_watcher_appointments(include_triggered=True) + + assert db_watcher_appointments == db_watcher_appointments_with_triggered + + # Create an appointment flagged as triggered + triggered_appointment, _ = generate_dummy_appointment(real_height=False) + uuid = uuid4().hex + db_manager.store_watcher_appointment(uuid, triggered_appointment.to_dict()) + db_manager.create_triggered_appointment_flag(uuid) + + # The new appointment is grabbed only if we set include_triggered + assert db_watcher_appointments == db_manager.load_watcher_appointments() + assert uuid in db_manager.load_watcher_appointments(include_triggered=True) + + +def test_store_load_responder_trackers(db_manager, responder_trackers): + for key, value in responder_trackers.items(): + db_manager.store_responder_tracker(key, {"value": value}) + + db_responder_trackers = db_manager.load_responder_trackers() + + values = [tracker["value"] for tracker in db_responder_trackers.values()] + + assert responder_trackers.keys() == db_responder_trackers.keys() + assert set(responder_trackers.values()) == set(values) and len(responder_trackers) == len(values) + + +def test_delete_watcher_appointment(db_manager, watcher_appointments): + # Let's delete all we added + db_watcher_appointments = db_manager.load_watcher_appointments(include_triggered=True) + assert len(db_watcher_appointments) != 0 + + for key in watcher_appointments.keys(): + db_manager.delete_watcher_appointment(key) + + db_watcher_appointments = db_manager.load_watcher_appointments() + assert len(db_watcher_appointments) == 0 + + +def test_batch_delete_watcher_appointments(db_manager, watcher_appointments): + # Let's start by adding a bunch of appointments + for uuid, appointment in watcher_appointments.items(): + db_manager.store_watcher_appointment(uuid, appointment.to_dict()) + + first_half = list(watcher_appointments.keys())[: len(watcher_appointments) // 2] + second_half = list(watcher_appointments.keys())[len(watcher_appointments) // 2 :] + + # Let's now delete half of them in a batch update + db_manager.batch_delete_watcher_appointments(first_half) + + db_watcher_appointments = db_manager.load_watcher_appointments() + assert not set(db_watcher_appointments.keys()).issuperset(first_half) + assert set(db_watcher_appointments.keys()).issuperset(second_half) + + # Let's delete the rest + db_manager.batch_delete_watcher_appointments(second_half) + + # Now there should be no appointments left + db_watcher_appointments = db_manager.load_watcher_appointments() + assert not db_watcher_appointments + + +def test_delete_responder_tracker(db_manager, responder_trackers): + # Same for the responder + db_responder_trackers = db_manager.load_responder_trackers() + assert len(db_responder_trackers) != 0 + + for key in responder_trackers.keys(): + db_manager.delete_responder_tracker(key) + + db_responder_trackers = db_manager.load_responder_trackers() + assert len(db_responder_trackers) == 0 + + +def test_batch_delete_responder_trackers(db_manager, responder_trackers): + # Let's start by adding a bunch of appointments + for uuid, value in responder_trackers.items(): + db_manager.store_responder_tracker(uuid, {"value": value}) + + first_half = list(responder_trackers.keys())[: len(responder_trackers) // 2] + second_half = list(responder_trackers.keys())[len(responder_trackers) // 2 :] + + # Let's now delete half of them in a batch update + db_manager.batch_delete_responder_trackers(first_half) + + db_responder_trackers = db_manager.load_responder_trackers() + assert not set(db_responder_trackers.keys()).issuperset(first_half) + assert set(db_responder_trackers.keys()).issuperset(second_half) + + # Let's delete the rest + db_manager.batch_delete_responder_trackers(second_half) + + # Now there should be no trackers left + db_responder_trackers = db_manager.load_responder_trackers() + assert not db_responder_trackers + + +def test_store_load_last_block_hash_watcher(db_manager): + # Let's first create a made up block hash + local_last_block_hash = get_random_value_hex(32) + db_manager.store_last_block_hash_watcher(local_last_block_hash) + + db_last_block_hash = db_manager.load_last_block_hash_watcher() + + assert local_last_block_hash == db_last_block_hash + + +def test_store_load_last_block_hash_responder(db_manager): + # Same for the responder + local_last_block_hash = get_random_value_hex(32) + db_manager.store_last_block_hash_responder(local_last_block_hash) + + db_last_block_hash = db_manager.load_last_block_hash_responder() + + assert local_last_block_hash == db_last_block_hash + + +def test_create_triggered_appointment_flag(db_manager): + # Test that flags are added + key = get_random_value_hex(16) + db_manager.create_triggered_appointment_flag(key) + + assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is not None + + # Test to get a random one that we haven't added + key = get_random_value_hex(16) + assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is None + + +def test_batch_create_triggered_appointment_flag(db_manager): + # Test that flags are added in batch + keys = [get_random_value_hex(16) for _ in range(10)] + + # Checked that non of the flags is already in the db + db_flags = db_manager.load_all_triggered_flags() + assert not set(db_flags).issuperset(keys) + + # Make sure that they are now + db_manager.batch_create_triggered_appointment_flag(keys) + db_flags = db_manager.load_all_triggered_flags() + assert set(db_flags).issuperset(keys) + + +def test_load_all_triggered_flags(db_manager): + # There should be a some flags in the db from the previous tests. Let's load them + flags = db_manager.load_all_triggered_flags() + + # We can add another flag and see that there's two now + new_uuid = uuid4().hex + db_manager.create_triggered_appointment_flag(new_uuid) + flags.append(new_uuid) + + assert set(db_manager.load_all_triggered_flags()) == set(flags) + + +def test_delete_triggered_appointment_flag(db_manager): + # Test data is properly deleted. + keys = db_manager.load_all_triggered_flags() + + # Delete all entries + for k in keys: + db_manager.delete_triggered_appointment_flag(k) + + # Try to load them back + for k in keys: + assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + k).encode("utf-8")) is None + + +def test_batch_delete_triggered_appointment_flag(db_manager): + # Let's add some flags first + keys = [get_random_value_hex(16) for _ in range(10)] + db_manager.batch_create_triggered_appointment_flag(keys) + + # And now let's delete in batch + first_half = keys[: len(keys) // 2] + second_half = keys[len(keys) // 2 :] + + db_manager.batch_delete_triggered_appointment_flag(first_half) + db_falgs = db_manager.load_all_triggered_flags() + assert not set(db_falgs).issuperset(first_half) + assert set(db_falgs).issuperset(second_half) + + # Delete the rest + db_manager.batch_delete_triggered_appointment_flag(second_half) + assert not db_manager.load_all_triggered_flags() diff --git a/test/teos/unit/test_cleaner.py b/test/teos/unit/test_cleaner.py index 6e44f11..ad2e263 100644 --- a/test/teos/unit/test_cleaner.py +++ b/test/teos/unit/test_cleaner.py @@ -27,7 +27,7 @@ def set_up_appointments(db_manager, total_appointments): appointments[uuid] = {"locator": appointment.locator} locator_uuid_map[locator] = [uuid] - db_manager.store_watcher_appointment(uuid, appointment.to_json()) + db_manager.store_watcher_appointment(uuid, appointment.to_dict()) db_manager.create_append_locator_map(locator, uuid) # Each locator can have more than one uuid assigned to it. @@ -37,7 +37,7 @@ def set_up_appointments(db_manager, total_appointments): appointments[uuid] = {"locator": appointment.locator} locator_uuid_map[locator].append(uuid) - db_manager.store_watcher_appointment(uuid, appointment.to_json()) + db_manager.store_watcher_appointment(uuid, appointment.to_dict()) db_manager.create_append_locator_map(locator, uuid) return appointments, locator_uuid_map @@ -60,7 +60,7 @@ def set_up_trackers(db_manager, total_trackers): trackers[uuid] = {"locator": tracker.locator, "penalty_txid": tracker.penalty_txid} tx_tracker_map[penalty_txid] = [uuid] - db_manager.store_responder_tracker(uuid, tracker.to_json()) + db_manager.store_responder_tracker(uuid, tracker.to_dict()) db_manager.create_append_locator_map(tracker.locator, uuid) # Each penalty_txid can have more than one uuid assigned to it. @@ -70,7 +70,7 @@ def set_up_trackers(db_manager, total_trackers): trackers[uuid] = {"locator": tracker.locator, "penalty_txid": tracker.penalty_txid} tx_tracker_map[penalty_txid].append(uuid) - db_manager.store_responder_tracker(uuid, tracker.to_json()) + db_manager.store_responder_tracker(uuid, tracker.to_dict()) db_manager.create_append_locator_map(tracker.locator, uuid) return trackers, tx_tracker_map diff --git a/test/teos/unit/test_db_manager.py b/test/teos/unit/test_db_manager.py index 5b25d46..d4b275d 100644 --- a/test/teos/unit/test_db_manager.py +++ b/test/teos/unit/test_db_manager.py @@ -1,30 +1,10 @@ import os import json -import pytest import shutil -from uuid import uuid4 +import pytest from teos.db_manager import DBManager -from teos.db_manager import ( - WATCHER_LAST_BLOCK_KEY, - RESPONDER_LAST_BLOCK_KEY, - LOCATOR_MAP_PREFIX, - TRIGGERED_APPOINTMENTS_PREFIX, -) - -from common.constants import LOCATOR_LEN_BYTES - -from test.teos.unit.conftest import get_random_value_hex, generate_dummy_appointment - - -@pytest.fixture(scope="module") -def watcher_appointments(): - return {uuid4().hex: generate_dummy_appointment(real_height=False)[0] for _ in range(10)} - - -@pytest.fixture(scope="module") -def responder_trackers(): - return {get_random_value_hex(16): get_random_value_hex(32) for _ in range(10)} +from test.teos.unit.conftest import get_random_value_hex def open_create_db(db_path): @@ -62,67 +42,15 @@ def test_init(): shutil.rmtree(db_path) -def test_load_appointments_db(db_manager): - # Let's made up a prefix and try to load data from the database using it - prefix = "XX" - db_appointments = db_manager.load_appointments_db(prefix) - - assert len(db_appointments) == 0 - - # We can add a bunch of data to the db and try again (data is stored in json by the manager) - local_appointments = {} - for _ in range(10): - key = get_random_value_hex(16) - value = get_random_value_hex(32) - local_appointments[key] = value - - db_manager.db.put((prefix + key).encode("utf-8"), json.dumps({"value": value}).encode("utf-8")) - - db_appointments = db_manager.load_appointments_db(prefix) - - # Check that both keys and values are the same - assert db_appointments.keys() == local_appointments.keys() - - values = [appointment["value"] for appointment in db_appointments.values()] - assert set(values) == set(local_appointments.values()) and (len(values) == len(local_appointments)) - - -def test_get_last_known_block(): - db_path = "empty_db" - - # First we check if the db exists, and if so we delete it - if os.path.isdir(db_path): - shutil.rmtree(db_path) - - # Check that the db can be created if it does not exist - db_manager = open_create_db(db_path) - - # Trying to get any last block for either the watcher or the responder should return None for an empty db - - for key in [WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY]: - assert db_manager.get_last_known_block(key) is None - - # After saving some block in the db we should get that exact value - for key in [WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY]: - block_hash = get_random_value_hex(32) - db_manager.db.put(key.encode("utf-8"), block_hash.encode("utf-8")) - assert db_manager.get_last_known_block(key) == block_hash - - # Removing test db - shutil.rmtree(db_path) - - def test_create_entry(db_manager): key = get_random_value_hex(16) value = get_random_value_hex(32) - # Adding a value with no prefix (create entry encodes values in utf-8 internally) + # Adding a value with no prefix should work db_manager.create_entry(key, value) - - # We should be able to get it straightaway from the key assert db_manager.db.get(key.encode("utf-8")).decode("utf-8") == value - # If we prefix the key we should be able to get it if we add the prefix, but not otherwise + # Prefixing the key would require the prefix to load key = get_random_value_hex(16) prefix = "w" db_manager.create_entry(key, value, prefix=prefix) @@ -130,22 +58,51 @@ def test_create_entry(db_manager): assert db_manager.db.get((prefix + key).encode("utf-8")).decode("utf-8") == value assert db_manager.db.get(key.encode("utf-8")) is None - # Same if we try to use any other prefix - another_prefix = "r" - assert db_manager.db.get((another_prefix + key).encode("utf-8")) is None + # Keys, prefixes, and values of wrong format should fail + with pytest.raises(TypeError): + db_manager.create_entry(key=None) + + with pytest.raises(TypeError): + db_manager.create_entry(key=key, value=None) + + with pytest.raises(TypeError): + db_manager.create_entry(key=key, value=value, prefix=1) + + +def test_load_entry(db_manager): + key = get_random_value_hex(16) + value = get_random_value_hex(32) + + # Loading an existing key should work + db_manager.db.put(key.encode("utf-8"), value.encode("utf-8")) + assert db_manager.load_entry(key) == value.encode("utf-8") + + # Adding an existing prefix should work + assert db_manager.load_entry(key[2:], prefix=key[:2]) == value.encode("utf-8") + + # Adding a non-existing prefix should return None + assert db_manager.load_entry(key, prefix=get_random_value_hex(2)) is None + + # Loading a non-existing entry should return None + assert db_manager.load_entry(get_random_value_hex(16)) is None + + # Trying to load a non str key or prefix should fail + with pytest.raises(TypeError): + db_manager.load_entry(None) + + with pytest.raises(TypeError): + db_manager.load_entry(get_random_value_hex(16), prefix=1) def test_delete_entry(db_manager): - # Let's first get the key all the things we've wrote so far in the db + # Let's get the key all the things we've wrote so far in the db and empty the db. data = [k.decode("utf-8") for k, v in db_manager.db.iterator()] - - # Let's empty the db now for key in data: db_manager.delete_entry(key) assert len([k for k, v in db_manager.db.iterator()]) == 0 - # Let's check that the same works if a prefix is provided. + # The same works if a prefix is provided. prefix = "r" key = get_random_value_hex(16) value = get_random_value_hex(32) @@ -158,294 +115,12 @@ def test_delete_entry(db_manager): db_manager.delete_entry(key, prefix) assert db_manager.db.get((prefix + key).encode("utf-8")) is None + # Deleting a non-existing key should be fine + db_manager.delete_entry(key, prefix) -def test_load_watcher_appointments_empty(db_manager): - assert len(db_manager.load_watcher_appointments()) == 0 + # Trying to delete a non str key or prefix should fail + with pytest.raises(TypeError): + db_manager.delete_entry(None) - -def test_load_responder_trackers_empty(db_manager): - assert len(db_manager.load_responder_trackers()) == 0 - - -def test_load_locator_map_empty(db_manager): - assert db_manager.load_locator_map(get_random_value_hex(LOCATOR_LEN_BYTES)) is None - - -def test_create_append_locator_map(db_manager): - uuid = uuid4().hex - locator = get_random_value_hex(LOCATOR_LEN_BYTES) - db_manager.create_append_locator_map(locator, uuid) - - # Check that the locator map has been properly stored - assert db_manager.load_locator_map(locator) == [uuid] - - # If we try to add the same uuid again the list shouldn't change - db_manager.create_append_locator_map(locator, uuid) - assert db_manager.load_locator_map(locator) == [uuid] - - # Add another uuid to the same locator and check that it also works - uuid2 = uuid4().hex - db_manager.create_append_locator_map(locator, uuid2) - - assert set(db_manager.load_locator_map(locator)) == set([uuid, uuid2]) - - -def test_update_locator_map(db_manager): - # Let's create a couple of appointments with the same locator - locator = get_random_value_hex(32) - uuid1 = uuid4().hex - uuid2 = uuid4().hex - db_manager.create_append_locator_map(locator, uuid1) - db_manager.create_append_locator_map(locator, uuid2) - - locator_map = db_manager.load_locator_map(locator) - assert uuid1 in locator_map - - locator_map.remove(uuid1) - db_manager.update_locator_map(locator, locator_map) - - locator_map_after = db_manager.load_locator_map(locator) - assert uuid1 not in locator_map_after and uuid2 in locator_map_after and len(locator_map_after) == 1 - - -def test_update_locator_map_wong_data(db_manager): - # Let's try to update the locator map with a different list of uuids - locator = get_random_value_hex(32) - db_manager.create_append_locator_map(locator, uuid4().hex) - db_manager.create_append_locator_map(locator, uuid4().hex) - - locator_map = db_manager.load_locator_map(locator) - wrong_map_update = [uuid4().hex] - db_manager.update_locator_map(locator, wrong_map_update) - locator_map_after = db_manager.load_locator_map(locator) - - assert locator_map_after == locator_map - - -def test_update_locator_map_empty(db_manager): - # We shouldn't be able to update a map with an empty list - locator = get_random_value_hex(32) - db_manager.create_append_locator_map(locator, uuid4().hex) - db_manager.create_append_locator_map(locator, uuid4().hex) - - locator_map = db_manager.load_locator_map(locator) - db_manager.update_locator_map(locator, []) - locator_map_after = db_manager.load_locator_map(locator) - - assert locator_map_after == locator_map - - -def test_delete_locator_map(db_manager): - locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX) - assert len(locator_maps) != 0 - - for locator, uuids in locator_maps.items(): - db_manager.delete_locator_map(locator) - - locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX) - assert len(locator_maps) == 0 - - -def test_store_load_watcher_appointment(db_manager, watcher_appointments): - for uuid, appointment in watcher_appointments.items(): - db_manager.store_watcher_appointment(uuid, appointment.to_json()) - - db_watcher_appointments = db_manager.load_watcher_appointments() - - # Check that the two appointment collections are equal by checking: - # - Their size is equal - # - Each element in one collection exists in the other - - assert watcher_appointments.keys() == db_watcher_appointments.keys() - - for uuid, appointment in watcher_appointments.items(): - assert json.dumps(db_watcher_appointments[uuid], sort_keys=True, separators=(",", ":")) == appointment.to_json() - - -def test_store_load_triggered_appointment(db_manager): - db_watcher_appointments = db_manager.load_watcher_appointments() - db_watcher_appointments_with_triggered = db_manager.load_watcher_appointments(include_triggered=True) - - assert db_watcher_appointments == db_watcher_appointments_with_triggered - - # Create an appointment flagged as triggered - triggered_appointment, _ = generate_dummy_appointment(real_height=False) - uuid = uuid4().hex - db_manager.store_watcher_appointment(uuid, triggered_appointment.to_json()) - db_manager.create_triggered_appointment_flag(uuid) - - # The new appointment is grabbed only if we set include_triggered - assert db_watcher_appointments == db_manager.load_watcher_appointments() - assert uuid in db_manager.load_watcher_appointments(include_triggered=True) - - -def test_store_load_responder_trackers(db_manager, responder_trackers): - for key, value in responder_trackers.items(): - db_manager.store_responder_tracker(key, json.dumps({"value": value})) - - db_responder_trackers = db_manager.load_responder_trackers() - - values = [tracker["value"] for tracker in db_responder_trackers.values()] - - assert responder_trackers.keys() == db_responder_trackers.keys() - assert set(responder_trackers.values()) == set(values) and len(responder_trackers) == len(values) - - -def test_delete_watcher_appointment(db_manager, watcher_appointments): - # Let's delete all we added - db_watcher_appointments = db_manager.load_watcher_appointments(include_triggered=True) - assert len(db_watcher_appointments) != 0 - - for key in watcher_appointments.keys(): - db_manager.delete_watcher_appointment(key) - - db_watcher_appointments = db_manager.load_watcher_appointments() - assert len(db_watcher_appointments) == 0 - - -def test_batch_delete_watcher_appointments(db_manager, watcher_appointments): - # Let's start by adding a bunch of appointments - for uuid, appointment in watcher_appointments.items(): - db_manager.store_watcher_appointment(uuid, appointment.to_json()) - - first_half = list(watcher_appointments.keys())[: len(watcher_appointments) // 2] - second_half = list(watcher_appointments.keys())[len(watcher_appointments) // 2 :] - - # Let's now delete half of them in a batch update - db_manager.batch_delete_watcher_appointments(first_half) - - db_watcher_appointments = db_manager.load_watcher_appointments() - assert not set(db_watcher_appointments.keys()).issuperset(first_half) - assert set(db_watcher_appointments.keys()).issuperset(second_half) - - # Let's delete the rest - db_manager.batch_delete_watcher_appointments(second_half) - - # Now there should be no appointments left - db_watcher_appointments = db_manager.load_watcher_appointments() - assert not db_watcher_appointments - - -def test_delete_responder_tracker(db_manager, responder_trackers): - # Same for the responder - db_responder_trackers = db_manager.load_responder_trackers() - assert len(db_responder_trackers) != 0 - - for key in responder_trackers.keys(): - db_manager.delete_responder_tracker(key) - - db_responder_trackers = db_manager.load_responder_trackers() - assert len(db_responder_trackers) == 0 - - -def test_batch_delete_responder_trackers(db_manager, responder_trackers): - # Let's start by adding a bunch of appointments - for uuid, value in responder_trackers.items(): - db_manager.store_responder_tracker(uuid, json.dumps({"value": value})) - - first_half = list(responder_trackers.keys())[: len(responder_trackers) // 2] - second_half = list(responder_trackers.keys())[len(responder_trackers) // 2 :] - - # Let's now delete half of them in a batch update - db_manager.batch_delete_responder_trackers(first_half) - - db_responder_trackers = db_manager.load_responder_trackers() - assert not set(db_responder_trackers.keys()).issuperset(first_half) - assert set(db_responder_trackers.keys()).issuperset(second_half) - - # Let's delete the rest - db_manager.batch_delete_responder_trackers(second_half) - - # Now there should be no trackers left - db_responder_trackers = db_manager.load_responder_trackers() - assert not db_responder_trackers - - -def test_store_load_last_block_hash_watcher(db_manager): - # Let's first create a made up block hash - local_last_block_hash = get_random_value_hex(32) - db_manager.store_last_block_hash_watcher(local_last_block_hash) - - db_last_block_hash = db_manager.load_last_block_hash_watcher() - - assert local_last_block_hash == db_last_block_hash - - -def test_store_load_last_block_hash_responder(db_manager): - # Same for the responder - local_last_block_hash = get_random_value_hex(32) - db_manager.store_last_block_hash_responder(local_last_block_hash) - - db_last_block_hash = db_manager.load_last_block_hash_responder() - - assert local_last_block_hash == db_last_block_hash - - -def test_create_triggered_appointment_flag(db_manager): - # Test that flags are added - key = get_random_value_hex(16) - db_manager.create_triggered_appointment_flag(key) - - assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is not None - - # Test to get a random one that we haven't added - key = get_random_value_hex(16) - assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is None - - -def test_batch_create_triggered_appointment_flag(db_manager): - # Test that flags are added in batch - keys = [get_random_value_hex(16) for _ in range(10)] - - # Checked that non of the flags is already in the db - db_flags = db_manager.load_all_triggered_flags() - assert not set(db_flags).issuperset(keys) - - # Make sure that they are now - db_manager.batch_create_triggered_appointment_flag(keys) - db_flags = db_manager.load_all_triggered_flags() - assert set(db_flags).issuperset(keys) - - -def test_load_all_triggered_flags(db_manager): - # There should be a some flags in the db from the previous tests. Let's load them - flags = db_manager.load_all_triggered_flags() - - # We can add another flag and see that there's two now - new_uuid = uuid4().hex - db_manager.create_triggered_appointment_flag(new_uuid) - flags.append(new_uuid) - - assert set(db_manager.load_all_triggered_flags()) == set(flags) - - -def test_delete_triggered_appointment_flag(db_manager): - # Test data is properly deleted. - keys = db_manager.load_all_triggered_flags() - - # Delete all entries - for k in keys: - db_manager.delete_triggered_appointment_flag(k) - - # Try to load them back - for k in keys: - assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + k).encode("utf-8")) is None - - -def test_batch_delete_triggered_appointment_flag(db_manager): - # Let's add some flags first - keys = [get_random_value_hex(16) for _ in range(10)] - db_manager.batch_create_triggered_appointment_flag(keys) - - # And now let's delete in batch - first_half = keys[: len(keys) // 2] - second_half = keys[len(keys) // 2 :] - - db_manager.batch_delete_triggered_appointment_flag(first_half) - db_falgs = db_manager.load_all_triggered_flags() - assert not set(db_falgs).issuperset(first_half) - assert set(db_falgs).issuperset(second_half) - - # Delete the rest - db_manager.batch_delete_triggered_appointment_flag(second_half) - assert not db_manager.load_all_triggered_flags() + with pytest.raises(TypeError): + db_manager.delete_entry(get_random_value_hex(16), prefix=1) diff --git a/test/teos/unit/test_responder.py b/test/teos/unit/test_responder.py index c667cc0..0bec7cc 100644 --- a/test/teos/unit/test_responder.py +++ b/test/teos/unit/test_responder.py @@ -9,8 +9,8 @@ from threading import Thread from teos.carrier import Carrier from teos.tools import bitcoin_cli -from teos.db_manager import DBManager from teos.chain_monitor import ChainMonitor +from teos.appointments_dbm import AppointmentsDBM from teos.responder import Responder, TransactionTracker from common.constants import LOCATOR_LEN_HEX @@ -36,7 +36,7 @@ def responder(db_manager, carrier, block_processor): @pytest.fixture(scope="session") def temp_db_manager(): db_name = get_random_value_hex(8) - db_manager = DBManager(db_name) + db_manager = AppointmentsDBM(db_name) yield db_manager @@ -120,17 +120,6 @@ def test_tracker_to_dict(): ) -def test_tracker_to_json(): - tracker = create_dummy_tracker() - tracker_dict = json.loads(tracker.to_json()) - - assert ( - tracker.locator == tracker_dict["locator"] - and tracker.penalty_rawtx == tracker_dict["penalty_rawtx"] - and tracker.appointment_end == tracker_dict["appointment_end"] - ) - - def test_tracker_from_dict(): tracker_dict = create_dummy_tracker().to_dict() new_tracker = TransactionTracker.from_dict(tracker_dict) @@ -295,7 +284,7 @@ def test_do_watch(temp_db_manager, carrier, block_processor): # We also need to store the info in the db responder.db_manager.create_triggered_appointment_flag(uuid) - responder.db_manager.store_responder_tracker(uuid, tracker.to_json()) + responder.db_manager.store_responder_tracker(uuid, tracker.to_dict()) # Let's start to watch Thread(target=responder.do_watch, daemon=True).start() @@ -472,7 +461,7 @@ def test_rebroadcast(db_manager, carrier, block_processor): # We need to add it to the db too responder.db_manager.create_triggered_appointment_flag(uuid) - responder.db_manager.store_responder_tracker(uuid, tracker.to_json()) + responder.db_manager.store_responder_tracker(uuid, tracker.to_dict()) responder.tx_tracker_map[penalty_txid] = [uuid] responder.unconfirmed_txs.append(penalty_txid) diff --git a/test/teos/unit/test_watcher.py b/test/teos/unit/test_watcher.py index e61a178..77ab810 100644 --- a/test/teos/unit/test_watcher.py +++ b/test/teos/unit/test_watcher.py @@ -9,8 +9,8 @@ from teos.carrier import Carrier from teos.watcher import Watcher from teos.tools import bitcoin_cli from teos.responder import Responder -from teos.db_manager import DBManager from teos.chain_monitor import ChainMonitor +from teos.appointments_dbm import AppointmentsDBM from teos.block_processor import BlockProcessor import common.cryptographer @@ -47,7 +47,7 @@ MAX_APPOINTMENTS = 100 @pytest.fixture(scope="session") def temp_db_manager(): db_name = get_random_value_hex(8) - db_manager = DBManager(db_name) + db_manager = AppointmentsDBM(db_name) yield db_manager @@ -198,7 +198,7 @@ def test_do_watch(watcher, temp_db_manager): for uuid, appointment in appointments.items(): watcher.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time, "size": 200} - watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + watcher.db_manager.store_watcher_appointment(uuid, appointment.to_dict()) watcher.db_manager.create_append_locator_map(appointment.locator, uuid) do_watch_thread = Thread(target=watcher.do_watch, daemon=True) @@ -248,7 +248,7 @@ def test_filter_valid_breaches_random_data(watcher): dummy_appointment, _ = generate_dummy_appointment() uuid = uuid4().hex appointments[uuid] = {"locator": dummy_appointment.locator, "end_time": dummy_appointment.end_time} - watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_json()) + watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_dict()) watcher.db_manager.create_append_locator_map(dummy_appointment.locator, uuid) locator_uuid_map[dummy_appointment.locator] = [uuid] @@ -288,7 +288,7 @@ def test_filter_valid_breaches(watcher): for uuid, appointment in appointments.items(): watcher.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} - watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_json()) + watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_dict()) watcher.db_manager.create_append_locator_map(dummy_appointment.locator, uuid) watcher.locator_uuid_map = locator_uuid_map From a46bab6b207170621e2a2c4ff1c538338a011bbc Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 18:25:14 +0200 Subject: [PATCH 51/62] Adds user database and modifies registered_user content to be a dict instead of an int Future additional data may be stored, so it's better if we store it as something that can hold it --- teos/gatekeeper.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index 2b812fd..935afac 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -28,9 +28,10 @@ class Gatekeeper: registered_users (:obj:`dict`): a map of user_pk:appointment_slots. """ - def __init__(self, default_slots): + def __init__(self, user_db, default_slots): self.default_slots = default_slots - self.registered_users = {} + self.user_db = user_db + self.registered_users = user_db.load_all_users() def add_update_user(self, user_pk): """ @@ -47,11 +48,13 @@ class Gatekeeper: raise ValueError("provided public key does not match expected format (33-byte hex string)") if user_pk not in self.registered_users: - self.registered_users[user_pk] = self.default_slots + self.registered_users[user_pk] = {"available_slots": self.default_slots} else: - self.registered_users[user_pk] += self.default_slots + self.registered_users[user_pk]["available_slots"] += self.default_slots - return self.registered_users[user_pk] + self.user_db.store_user(user_pk, self.registered_users[user_pk]) + + return self.registered_users[user_pk]["available_slots"] def identify_user(self, message, signature): """ @@ -95,9 +98,9 @@ class Gatekeeper: # We are not making sure the value passed is a integer, but the value is computed by the API and rounded before # passing it to the gatekeeper. - # DISCUSS: we may want to return a different exception if teh user does not exist - if user_pk in self.registered_users and n <= self.registered_users.get(user_pk): - self.registered_users[user_pk] -= n + # DISCUSS: we may want to return a different exception if the user does not exist + if user_pk in self.registered_users and n <= self.registered_users.get(user_pk).get("available_slots"): + self.registered_users[user_pk]["available_slots"] -= n else: raise NotEnoughSlots(user_pk, n) @@ -114,4 +117,4 @@ class Gatekeeper: # passing it to the gatekeeper. # DISCUSS: if the user does not exist we may want to log or return an exception. if user_pk in self.registered_users: - self.registered_users[user_pk] += n + self.registered_users[user_pk]["available_slots"] += n From 3009bb83ba824b83290f6e2a87bff987cb4ed612 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 18:26:36 +0200 Subject: [PATCH 52/62] Addaps api and daemon to use userDB --- teos/__init__.py | 1 + teos/api.py | 2 +- teos/teosd.py | 5 ++- teos/users_dbm.py | 100 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 teos/users_dbm.py diff --git a/teos/__init__.py b/teos/__init__.py index 0eba6bd..811e16d 100644 --- a/teos/__init__.py +++ b/teos/__init__.py @@ -23,4 +23,5 @@ DEFAULT_CONF = { "LOG_FILE": {"value": "teos.log", "type": str, "path": True}, "TEOS_SECRET_KEY": {"value": "teos_sk.der", "type": str, "path": True}, "APPOINTMENTS_DB_PATH": {"value": "appointments", "type": str, "path": True}, + "USERS_DB_PATH": {"value": "users", "type": str, "path": True}, } diff --git a/teos/api.py b/teos/api.py index ec65329..ec71206 100644 --- a/teos/api.py +++ b/teos/api.py @@ -213,7 +213,7 @@ class API: response = { "locator": appointment.locator, "signature": signature, - "available_slots": self.gatekeeper.registered_users[user_pk], + "available_slots": self.gatekeeper.registered_users[user_pk].get("available_slots"), } else: diff --git a/teos/teosd.py b/teos/teosd.py index f23447a..694b664 100644 --- a/teos/teosd.py +++ b/teos/teosd.py @@ -14,12 +14,13 @@ from teos.help import show_usage from teos.watcher import Watcher from teos.builder import Builder from teos.carrier import Carrier +from teos.users_dbm import UsersDBM from teos.inspector import Inspector from teos.responder import Responder -from teos.appointments_dbm import AppointmentsDBM from teos.gatekeeper import Gatekeeper from teos.chain_monitor import ChainMonitor from teos.block_processor import BlockProcessor +from teos.appointments_dbm import AppointmentsDBM from teos.tools import can_connect_to_bitcoind, in_correct_network from teos import LOG_PREFIX, DATA_DIR, DEFAULT_CONF, CONF_FILE_NAME @@ -151,7 +152,7 @@ def main(command_line_conf): # Fire the API and the ChainMonitor # FIXME: 92-block-data-during-bootstrap-db chain_monitor.monitor_chain() - gatekeeper = Gatekeeper(config.get("DEFAULT_SLOTS")) + gatekeeper = Gatekeeper(UsersDBM(config.get("USERS_DB_PATH")), config.get("DEFAULT_SLOTS")) API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, gatekeeper).start() except Exception as e: logger.error("An error occurred: {}. Shutting down".format(e)) diff --git a/teos/users_dbm.py b/teos/users_dbm.py new file mode 100644 index 0000000..d0caac2 --- /dev/null +++ b/teos/users_dbm.py @@ -0,0 +1,100 @@ +import json +import plyvel + +from teos import LOG_PREFIX +from teos.db_manager import DBManager + +from common.logger import Logger + +logger = Logger(actor="UsersDBM", log_name_prefix=LOG_PREFIX) + + +class UsersDBM(DBManager): + """ + The :class:`UsersDBM` is the class in charge of interacting with the users database (``LevelDB``). + Keys and values are stored as bytes in the database but processed as strings by the manager. + + Args: + db_path (:obj:`str`): the path (relative or absolute) to the system folder containing the database. A fresh + database will be create if the specified path does not contain one. + + Raises: + ValueError: If the provided ``db_path`` is not a string. + plyvel.Error: If the db is currently unavailable (being used by another process). + """ + + def __init__(self, db_path): + if not isinstance(db_path, str): + raise ValueError("db_path must be a valid path/name") + + try: + super().__init__(db_path) + + except plyvel.Error as e: + if "LOCK: Resource temporarily unavailable" in str(e): + logger.info("The db is already being used by another process (LOCK)") + + raise e + + def store_user(self, user_pk, user_data): + """ + Stores a user record to the database. ``user_pk`` is used as identifier. + + Args: + user_pk (:obj:`str`): a 33-byte hex-encoded string identifying the user. + user_data (:obj:`dict`): the user associated data, as a dictionary. + """ + + self.create_entry(user_pk, json.dumps(user_data)) + logger.info("Adding user to Gatekeeper's db", uuid=user_pk) + + def load_user(self, user_pk): + """ + Loads a user record from the database using the ``user_pk`` as identifier. + + use_pk (:obj:`str`): a 33-byte hex-encoded string identifying the user. + + Returns: + :obj:`dict`: A dictionary containing the appointment data if they ``key`` is found. + + Returns ``None`` otherwise. + """ + + data = self.load_entry(user_pk) + + try: + data = json.loads(data) + except (TypeError, json.decoder.JSONDecodeError): + data = None + + return data + + def delete_user(self, user_pk): + """ + Deletes a user record from the database. + + Args: + user_pk (:obj:`str`): a 33-byte hex-encoded string identifying the user. + """ + + self.delete_entry(user_pk) + logger.info("Deleting user from Gatekeeper's db", uuid=user_pk) + + def load_all_users(self): + """ + Loads all user records from the database. + + Returns: + :obj:`dict`: A dictionary containing all users indexed by ``user_pk``. + + Returns an empty dictionary if no data is found. + """ + + data = {} + + for k, v in self.db.iterator(): + # Get uuid and appointment_data from the db + user_pk = k.decode("utf-8") + data[user_pk] = json.loads(v) + + return data From f31f356a1a568315bc716648814fe3ba04858991 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 18:27:27 +0200 Subject: [PATCH 53/62] Adapts tests to use userDB and registered_users content as dict --- test/teos/e2e/test_basic_e2e.py | 10 ++----- test/teos/unit/conftest.py | 18 ++++++++++++ test/teos/unit/test_api.py | 24 ++++++++-------- test/teos/unit/test_gatekeeper.py | 46 ++++++++++++++++--------------- 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/test/teos/e2e/test_basic_e2e.py b/test/teos/e2e/test_basic_e2e.py index 9bc63c2..5e23cee 100644 --- a/test/teos/e2e/test_basic_e2e.py +++ b/test/teos/e2e/test_basic_e2e.py @@ -328,11 +328,8 @@ def test_appointment_shutdown_teos_trigger_back_online(create_txs, bitcoin_cli): assert teos_pid != teosd_process.pid - # FIXME: We have to cheat here since users are not kept between restarts atm - sleep(1) - teos_cli.register(compressed_cli_pk, teos_base_endpoint) - # Check that the appointment is still in the Watcher + sleep(1) appointment_info = get_appointment_info(locator) assert appointment_info is not None @@ -376,11 +373,8 @@ def test_appointment_shutdown_teos_trigger_while_offline(create_txs, bitcoin_cli teosd_process = run_teosd() assert teos_pid != teosd_process.pid - # FIXME: We have to cheat here since users are not kept between restarts atm - sleep(1) - teos_cli.register(compressed_cli_pk, teos_base_endpoint) - # The appointment should have been moved to the Responder + sleep(1) appointment_info = get_appointment_info(locator) assert appointment_info is not None diff --git a/test/teos/unit/conftest.py b/test/teos/unit/conftest.py index 6076b52..cbf3fc5 100644 --- a/test/teos/unit/conftest.py +++ b/test/teos/unit/conftest.py @@ -12,6 +12,8 @@ from bitcoind_mock.transaction import create_dummy_transaction from teos.carrier import Carrier from teos.tools import bitcoin_cli +from teos.users_dbm import UsersDBM +from teos.gatekeeper import Gatekeeper from teos import LOG_PREFIX, DEFAULT_CONF from teos.responder import TransactionTracker from teos.block_processor import BlockProcessor @@ -62,6 +64,17 @@ def db_manager(): rmtree("test_db") +@pytest.fixture(scope="module") +def user_db_manager(): + manager = UsersDBM("test_user_db") + # Add last know block for the Responder in the db + + yield manager + + manager.db.close() + rmtree("test_user_db") + + @pytest.fixture(scope="module") def carrier(): return Carrier(bitcoind_connect_params) @@ -72,6 +85,11 @@ def block_processor(): return BlockProcessor(bitcoind_connect_params) +@pytest.fixture(scope="module") +def gatekeeper(user_db_manager): + return Gatekeeper(user_db_manager, get_config().get("DEFAULT_SLOTS")) + + def generate_keypair(): sk = PrivateKey() pk = sk.public_key diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 00b0844..1cfed89 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -58,13 +58,12 @@ def get_all_db_manager(): @pytest.fixture(scope="module", autouse=True) -def api(db_manager, carrier, block_processor, run_bitcoind): +def api(db_manager, carrier, block_processor, gatekeeper, run_bitcoind): sk, pk = generate_keypair() responder = Responder(db_manager, carrier, block_processor) watcher = Watcher(db_manager, block_processor, responder, sk.to_der(), MAX_APPOINTMENTS, config.get("EXPIRY_DELTA")) - gatekeeper = Gatekeeper(config.get("DEFAULT_SLOTS")) api = API(Inspector(block_processor, config.get("MIN_TO_SELF_DELAY")), watcher, gatekeeper) return api @@ -146,7 +145,7 @@ def test_register_json_no_inner_dict(client): def test_add_appointment(api, client, appointment): # Simulate the user registration - api.gatekeeper.registered_users[compressed_client_pk] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": 1} # Properly formatted appointment appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) @@ -159,7 +158,7 @@ def test_add_appointment(api, client, appointment): def test_add_appointment_no_json(api, client, appointment): # Simulate the user registration - api.gatekeeper.registered_users[compressed_client_pk] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": 1} # Properly formatted appointment r = client.post(add_appointment_endpoint, data="random_message") @@ -168,7 +167,7 @@ def test_add_appointment_no_json(api, client, appointment): def test_add_appointment_json_no_inner_dict(api, client, appointment): # Simulate the user registration - api.gatekeeper.registered_users[compressed_client_pk] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": 1} # Properly formatted appointment r = client.post(add_appointment_endpoint, json="random_message") @@ -204,7 +203,7 @@ def test_add_appointment_not_registered(api, client, appointment): def test_add_appointment_registered_no_free_slots(api, client, appointment): # Empty the user slots - api.gatekeeper.registered_users[compressed_client_pk] = 0 + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": 0} # Properly formatted appointment appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) @@ -237,7 +236,7 @@ def test_add_appointment_multiple_times_same_user(api, client, appointment, n=MU appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) # Simulate registering enough slots - api.gatekeeper.registered_users[compressed_client_pk] = n + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": n} for _ in range(n): r = add_appointment( client, {"appointment": appointment.to_dict(), "signature": appointment_signature}, compressed_client_pk @@ -257,7 +256,8 @@ def test_add_appointment_multiple_times_different_users(api, client, appointment # Add one slot per public key for pair in user_keys: - api.gatekeeper.registered_users[hexlify(pair[1].format(compressed=True)).decode("utf-8")] = 2 + tmp_compressed_pk = hexlify(pair[1].format(compressed=True)).decode("utf-8") + api.gatekeeper.registered_users[tmp_compressed_pk] = {"available_slots": 2} # Send the appointments for compressed_pk, signature in zip(compressed_pks, signatures): @@ -271,7 +271,7 @@ def test_add_appointment_multiple_times_different_users(api, client, appointment def test_add_appointment_update_same_size(api, client, appointment): # Update an appointment by one of the same size and check that no additional slots are filled - api.gatekeeper.registered_users[compressed_client_pk] = 1 + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": 1} appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) # # Since we will replace the appointment, we won't added to appointments @@ -292,7 +292,7 @@ def test_add_appointment_update_same_size(api, client, appointment): def test_add_appointment_update_bigger(api, client, appointment): # Update an appointment by one bigger, and check additional slots are filled - api.gatekeeper.registered_users[compressed_client_pk] = 2 + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": 2} appointment_signature = Cryptographer.sign(appointment.serialize(), client_sk) r = add_appointment( @@ -320,7 +320,7 @@ def test_add_appointment_update_bigger(api, client, appointment): def test_add_appointment_update_smaller(api, client, appointment): # Update an appointment by one bigger, and check slots are freed - api.gatekeeper.registered_users[compressed_client_pk] = 2 + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": 2} # This should take 2 slots appointment.encrypted_blob.data = TWO_SLOTS_BLOTS @@ -341,7 +341,7 @@ def test_add_appointment_update_smaller(api, client, appointment): def test_add_too_many_appointment(api, client): # Give slots to the user - api.gatekeeper.registered_users[compressed_client_pk] = 200 + api.gatekeeper.registered_users[compressed_client_pk] = {"available_slots": 200} free_appointment_slots = MAX_APPOINTMENTS - len(api.watcher.appointments) diff --git a/test/teos/unit/test_gatekeeper.py b/test/teos/unit/test_gatekeeper.py index 5608092..bfb916f 100644 --- a/test/teos/unit/test_gatekeeper.py +++ b/test/teos/unit/test_gatekeeper.py @@ -1,31 +1,33 @@ import pytest -from teos.gatekeeper import Gatekeeper, IdentificationFailure, NotEnoughSlots +from teos.gatekeeper import IdentificationFailure, NotEnoughSlots from common.cryptographer import Cryptographer -from test.teos.unit.conftest import get_random_value_hex, generate_keypair - -DEFAULT_SLOTS = 42 -gatekeeper = Gatekeeper(DEFAULT_SLOTS) +from test.teos.unit.conftest import get_random_value_hex, generate_keypair, get_config -def test_init(): - assert isinstance(gatekeeper.default_slots, int) and gatekeeper.default_slots == DEFAULT_SLOTS +config = get_config() + + +def test_init(gatekeeper): + assert isinstance(gatekeeper.default_slots, int) and gatekeeper.default_slots == config.get("DEFAULT_SLOTS") assert isinstance(gatekeeper.registered_users, dict) and len(gatekeeper.registered_users) == 0 -def test_add_update_user(): +def test_add_update_user(gatekeeper): # add_update_user adds DEFAULT_SLOTS to a given user as long as the identifier is {02, 03}| 32-byte hex str user_pk = "02" + get_random_value_hex(32) for _ in range(10): current_slots = gatekeeper.registered_users.get(user_pk) - current_slots = current_slots if current_slots is not None else 0 + current_slots = current_slots.get("available_slots") if current_slots is not None else 0 gatekeeper.add_update_user(user_pk) - assert gatekeeper.registered_users.get(user_pk) == current_slots + DEFAULT_SLOTS + assert gatekeeper.registered_users.get(user_pk).get("available_slots") == current_slots + config.get( + "DEFAULT_SLOTS" + ) # The same can be checked for multiple users for _ in range(10): @@ -33,10 +35,10 @@ def test_add_update_user(): user_pk = "03" + get_random_value_hex(32) gatekeeper.add_update_user(user_pk) - assert gatekeeper.registered_users.get(user_pk) == DEFAULT_SLOTS + assert gatekeeper.registered_users.get(user_pk).get("available_slots") == config.get("DEFAULT_SLOTS") -def test_add_update_user_wrong_pk(): +def test_add_update_user_wrong_pk(gatekeeper): # Passing a wrong pk defaults to the errors in check_user_pk. We can try with one. wrong_pk = get_random_value_hex(32) @@ -44,7 +46,7 @@ def test_add_update_user_wrong_pk(): gatekeeper.add_update_user(wrong_pk) -def test_add_update_user_wrong_pk_prefix(): +def test_add_update_user_wrong_pk_prefix(gatekeeper): # Prefixes must be 02 or 03, anything else should fail wrong_pk = "04" + get_random_value_hex(32) @@ -52,7 +54,7 @@ def test_add_update_user_wrong_pk_prefix(): gatekeeper.add_update_user(wrong_pk) -def test_identify_user(): +def test_identify_user(gatekeeper): # Identify user should return a user_pk for registered users. It raises # IdentificationFailure for invalid parameters or non-registered users. @@ -67,7 +69,7 @@ def test_identify_user(): assert gatekeeper.identify_user(message.encode(), signature) == compressed_pk -def test_identify_user_non_registered(): +def test_identify_user_non_registered(gatekeeper): # Non-registered user won't be identified sk, pk = generate_keypair() @@ -78,7 +80,7 @@ def test_identify_user_non_registered(): gatekeeper.identify_user(message.encode(), signature) -def test_identify_user_invalid_signature(): +def test_identify_user_invalid_signature(gatekeeper): # If the signature does not match the message given a public key, the user won't be identified message = "Hey, it's me" signature = get_random_value_hex(72) @@ -87,7 +89,7 @@ def test_identify_user_invalid_signature(): gatekeeper.identify_user(message.encode(), signature) -def test_identify_user_wrong(): +def test_identify_user_wrong(gatekeeper): # Wrong parameters shouldn't verify either sk, pk = generate_keypair() @@ -107,13 +109,13 @@ def test_identify_user_wrong(): gatekeeper.identify_user(message, signature.encode()) -def test_fill_slots(): +def test_fill_slots(gatekeeper): # Free slots will decrease the slot count of a user as long as he has enough slots, otherwise raise NotEnoughSlots user_pk = "02" + get_random_value_hex(32) gatekeeper.add_update_user(user_pk) - gatekeeper.fill_slots(user_pk, DEFAULT_SLOTS - 1) - assert gatekeeper.registered_users.get(user_pk) == 1 + gatekeeper.fill_slots(user_pk, config.get("DEFAULT_SLOTS") - 1) + assert gatekeeper.registered_users.get(user_pk).get("available_slots") == 1 with pytest.raises(NotEnoughSlots): gatekeeper.fill_slots(user_pk, 2) @@ -123,13 +125,13 @@ def test_fill_slots(): gatekeeper.fill_slots(get_random_value_hex(33), 2) -def test_free_slots(): +def test_free_slots(gatekeeper): # Free slots simply adds slots to the user as long as it exists. user_pk = "03" + get_random_value_hex(32) gatekeeper.add_update_user(user_pk) gatekeeper.free_slots(user_pk, 42) - assert gatekeeper.registered_users.get(user_pk) == DEFAULT_SLOTS + 42 + assert gatekeeper.registered_users.get(user_pk).get("available_slots") == config.get("DEFAULT_SLOTS") + 42 # Just making sure it does not crash for non-registered user assert gatekeeper.free_slots(get_random_value_hex(33), 10) is None From 9a29e597818cd0758c6360aedb87424decdc5244 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 21:30:12 +0200 Subject: [PATCH 54/62] Adds additional checks to DBManagers (Appointments and Users) --- teos/appointments_dbm.py | 112 ++++++++++++++++++++++++++++++++------- teos/users_dbm.py | 38 ++++++++++--- 2 files changed, 126 insertions(+), 24 deletions(-) diff --git a/teos/appointments_dbm.py b/teos/appointments_dbm.py index 52cb946..ba1a9f8 100644 --- a/teos/appointments_dbm.py +++ b/teos/appointments_dbm.py @@ -103,9 +103,8 @@ class AppointmentsDBM(DBManager): Returns ``None`` otherwise. """ - data = self.load_entry(key, prefix=WATCHER_PREFIX) - try: + data = self.load_entry(key, prefix=WATCHER_PREFIX) data = json.loads(data) except (TypeError, json.decoder.JSONDecodeError): data = None @@ -122,9 +121,8 @@ class AppointmentsDBM(DBManager): Returns ``None`` otherwise. """ - data = self.load_entry(key, prefix=RESPONDER_PREFIX) - try: + data = self.load_entry(key, prefix=RESPONDER_PREFIX) data = json.loads(data) except (TypeError, json.decoder.JSONDecodeError): data = None @@ -171,10 +169,23 @@ class AppointmentsDBM(DBManager): Args: uuid (:obj:`str`): the identifier of the appointment to be stored. appointment (:obj: `dict`): an appointment encoded as dictionary. + + Returns: + :obj:`bool`: True if the appointment was stored in the db. False otherwise. """ - self.create_entry(uuid, json.dumps(appointment), prefix=WATCHER_PREFIX) - logger.info("Adding appointment to Watchers's db", uuid=uuid) + try: + self.create_entry(uuid, json.dumps(appointment), prefix=WATCHER_PREFIX) + logger.info("Adding appointment to Watchers's db", uuid=uuid) + return True + + except json.JSONDecodeError: + logger.info("Could't add appointment to db. Wrong appointment format.", uuid=uuid, appoinent=appointment) + return False + + except TypeError: + logger.info("Could't add appointment to db.", uuid=uuid, appoinent=appointment) + return False def store_responder_tracker(self, uuid, tracker): """ @@ -183,10 +194,23 @@ class AppointmentsDBM(DBManager): Args: uuid (:obj:`str`): the identifier of the appointment to be stored. tracker (:obj: `dict`): a tracker encoded as dictionary. + + Returns: + :obj:`bool`: True if the tracker was stored in the db. False otherwise. """ - self.create_entry(uuid, json.dumps(tracker), prefix=RESPONDER_PREFIX) - logger.info("Adding appointment to Responder's db", uuid=uuid) + try: + self.create_entry(uuid, json.dumps(tracker), prefix=RESPONDER_PREFIX) + logger.info("Adding tracker to Responder's db", uuid=uuid) + return True + + except json.JSONDecodeError: + logger.info("Could't add tracker to db. Wrong tracker format.", uuid=uuid, tracker=tracker) + return False + + except TypeError: + logger.info("Could't add tracker to db.", uuid=uuid, tracker=tracker) + return False def load_locator_map(self, locator): """ @@ -265,10 +289,19 @@ class AppointmentsDBM(DBManager): Args: locator (:obj:`str`): a 16-byte hex-encoded string identifying the map to delete. + + Returns: + :obj:`bool`: True if the locator map was deleted from the database or it was non-existent, False otherwise. """ - self.delete_entry(locator, prefix=LOCATOR_MAP_PREFIX) - logger.info("Deleting locator map from db", uuid=locator) + try: + self.delete_entry(locator, prefix=LOCATOR_MAP_PREFIX) + logger.info("Deleting locator map from db", locator=locator) + return True + + except TypeError: + logger.info("Couldn't delete locator map from db, locator has wrong type", locator=locator) + return False def delete_watcher_appointment(self, uuid): """ @@ -276,10 +309,19 @@ class AppointmentsDBM(DBManager): Args: uuid (:obj:`str`): a 16-byte hex-encoded string identifying the appointment to be deleted. + + Returns: + :obj:`bool`: True if the appointment was deleted from the database or it was non-existent, False otherwise. """ - self.delete_entry(uuid, prefix=WATCHER_PREFIX) - logger.info("Deleting appointment from Watcher's db", uuid=uuid) + try: + self.delete_entry(uuid, prefix=WATCHER_PREFIX) + logger.info("Deleting appointment from Watcher's db", uuid=uuid) + return True + + except TypeError: + logger.info("Couldn't delete appointment from db, uuid has wrong type", uuid=uuid) + return False def batch_delete_watcher_appointments(self, uuids): """ @@ -300,10 +342,19 @@ class AppointmentsDBM(DBManager): Args: uuid (:obj:`str`): a 16-byte hex-encoded string identifying the tracker to be deleted. + + Returns: + :obj:`bool`: True if the tracker was deleted from the database or it was non-existent, False otherwise. """ - self.delete_entry(uuid, prefix=RESPONDER_PREFIX) - logger.info("Deleting appointment from Responder's db", uuid=uuid) + try: + self.delete_entry(uuid, prefix=RESPONDER_PREFIX) + logger.info("Deleting tracker from Responder's db", uuid=uuid) + return True + + except TypeError: + logger.info("Couldn't delete tracker from db, uuid has wrong type", uuid=uuid) + return False def batch_delete_responder_trackers(self, uuids): """ @@ -346,9 +397,17 @@ class AppointmentsDBM(DBManager): Args: block_hash (:obj:`str`): the block hash to be stored (32-byte hex-encoded) + + Returns: + :obj:`bool`: True if the block hash was stored in the db. False otherwise. """ - self.create_entry(WATCHER_LAST_BLOCK_KEY, block_hash) + try: + self.create_entry(WATCHER_LAST_BLOCK_KEY, block_hash) + return True + + except (TypeError, json.JSONDecodeError): + return False def store_last_block_hash_responder(self, block_hash): """ @@ -356,9 +415,17 @@ class AppointmentsDBM(DBManager): Args: block_hash (:obj:`str`): the block hash to be stored (32-byte hex-encoded) + + Returns: + :obj:`bool`: True if the block hash was stored in the db. False otherwise. """ - self.create_entry(RESPONDER_LAST_BLOCK_KEY, block_hash) + try: + self.create_entry(RESPONDER_LAST_BLOCK_KEY, block_hash) + return True + + except (TypeError, json.JSONDecodeError): + return False def create_triggered_appointment_flag(self, uuid): """ @@ -403,10 +470,19 @@ class AppointmentsDBM(DBManager): Args: uuid (:obj:`str`): the identifier of the flag to be removed. + + Returns: + :obj:`bool`: True if the flag was deleted from the database or it was non-existent, False otherwise. """ - self.delete_entry(uuid, prefix=TRIGGERED_APPOINTMENTS_PREFIX) - logger.info("Removing triggered flag from appointment appointment", uuid=uuid) + try: + self.delete_entry(uuid, prefix=TRIGGERED_APPOINTMENTS_PREFIX) + logger.info("Removing triggered flag from appointment appointment", uuid=uuid) + return True + + except TypeError: + logger.info("Couldn't delete triggered flag from db, uuid has wrong type", uuid=uuid) + return False def batch_delete_triggered_appointment_flag(self, uuids): """ diff --git a/teos/users_dbm.py b/teos/users_dbm.py index d0caac2..d9809be 100644 --- a/teos/users_dbm.py +++ b/teos/users_dbm.py @@ -5,6 +5,7 @@ from teos import LOG_PREFIX from teos.db_manager import DBManager from common.logger import Logger +from common.tools import check_compressed_pk_format logger = Logger(actor="UsersDBM", log_name_prefix=LOG_PREFIX) @@ -43,10 +44,27 @@ class UsersDBM(DBManager): Args: user_pk (:obj:`str`): a 33-byte hex-encoded string identifying the user. user_data (:obj:`dict`): the user associated data, as a dictionary. + + Returns: + :obj:`bool`: True if the user was stored in the database, false otherwise. """ - self.create_entry(user_pk, json.dumps(user_data)) - logger.info("Adding user to Gatekeeper's db", uuid=user_pk) + if check_compressed_pk_format(user_pk): + try: + self.create_entry(user_pk, json.dumps(user_data)) + logger.info("Adding user to Gatekeeper's db", user_pk=user_pk) + return True + + except json.JSONDecodeError: + logger.info("Could't add user to db. Wrong user data format.", user_pk=user_pk, user_data=user_data) + return False + + except TypeError: + logger.info("Could't add user to db.", user_pk=user_pk, user_data=user_data) + return False + else: + logger.info("Could't add user to db. Wrong pk format.", user_pk=user_pk, user_data=user_data) + return False def load_user(self, user_pk): """ @@ -60,9 +78,8 @@ class UsersDBM(DBManager): Returns ``None`` otherwise. """ - data = self.load_entry(user_pk) - try: + data = self.load_entry(user_pk) data = json.loads(data) except (TypeError, json.decoder.JSONDecodeError): data = None @@ -75,10 +92,19 @@ class UsersDBM(DBManager): Args: user_pk (:obj:`str`): a 33-byte hex-encoded string identifying the user. + + Returns: + :obj:`bool`: True if the user was deleted from the database or it was non-existent, False otherwise. """ - self.delete_entry(user_pk) - logger.info("Deleting user from Gatekeeper's db", uuid=user_pk) + try: + self.delete_entry(user_pk) + logger.info("Deleting user from Gatekeeper's db", uuid=user_pk) + return True + + except TypeError: + logger.info("Cant delete user from db, user key has wrong type", uuid=user_pk) + return False def load_all_users(self): """ From 7abb4ada004420890ed90b12deae62a5664d6bcd Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 21:30:34 +0200 Subject: [PATCH 55/62] Updates AppoitmentDBM unit tests and adds UserDBM's --- test/teos/unit/test_appointments_dbm.py | 68 ++++++++++++++++---- test/teos/unit/test_users_dbm.py | 82 +++++++++++++++++++++++++ 2 files changed, 139 insertions(+), 11 deletions(-) create mode 100644 test/teos/unit/test_users_dbm.py diff --git a/test/teos/unit/test_appointments_dbm.py b/test/teos/unit/test_appointments_dbm.py index b6cd69f..48928f6 100644 --- a/test/teos/unit/test_appointments_dbm.py +++ b/test/teos/unit/test_appointments_dbm.py @@ -169,15 +169,32 @@ def test_delete_locator_map(db_manager): assert len(locator_maps) != 0 for locator, uuids in locator_maps.items(): - db_manager.delete_locator_map(locator) + assert db_manager.delete_locator_map(locator) is True locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX) assert len(locator_maps) == 0 + # Keys of wrong type should fail + assert db_manager.delete_locator_map(42) is False + + +def test_store_watcher_appointment_wrong(db_manager, watcher_appointments): + # Wrong uuid types should fail + for _, appointment in watcher_appointments.items(): + assert db_manager.store_watcher_appointment(42, appointment.to_dict()) is False + + +def test_load_watcher_appointment_wrong(db_manager): + # Random keys should fail + assert db_manager.load_watcher_appointment(get_random_value_hex(16)) is None + + # Wrong format keys should also return None + assert db_manager.load_watcher_appointment(42) is None + def test_store_load_watcher_appointment(db_manager, watcher_appointments): for uuid, appointment in watcher_appointments.items(): - db_manager.store_watcher_appointment(uuid, appointment.to_dict()) + assert db_manager.store_watcher_appointment(uuid, appointment.to_dict()) is True db_watcher_appointments = db_manager.load_watcher_appointments() @@ -200,7 +217,7 @@ def test_store_load_triggered_appointment(db_manager): # Create an appointment flagged as triggered triggered_appointment, _ = generate_dummy_appointment(real_height=False) uuid = uuid4().hex - db_manager.store_watcher_appointment(uuid, triggered_appointment.to_dict()) + assert db_manager.store_watcher_appointment(uuid, triggered_appointment.to_dict()) is True db_manager.create_triggered_appointment_flag(uuid) # The new appointment is grabbed only if we set include_triggered @@ -208,9 +225,23 @@ def test_store_load_triggered_appointment(db_manager): assert uuid in db_manager.load_watcher_appointments(include_triggered=True) +def test_store_responder_trackers_wrong(db_manager, responder_trackers): + # Wrong uuid types should fail + for _, tracker in responder_trackers.items(): + assert db_manager.store_responder_tracker(42, {"value": tracker}) is False + + +def test_load_responder_tracker_wrong(db_manager): + # Random keys should fail + assert db_manager.load_responder_tracker(get_random_value_hex(16)) is None + + # Wrong format keys should also return None + assert db_manager.load_responder_tracker(42) is None + + def test_store_load_responder_trackers(db_manager, responder_trackers): for key, value in responder_trackers.items(): - db_manager.store_responder_tracker(key, {"value": value}) + assert db_manager.store_responder_tracker(key, {"value": value}) is True db_responder_trackers = db_manager.load_responder_trackers() @@ -226,16 +257,19 @@ def test_delete_watcher_appointment(db_manager, watcher_appointments): assert len(db_watcher_appointments) != 0 for key in watcher_appointments.keys(): - db_manager.delete_watcher_appointment(key) + assert db_manager.delete_watcher_appointment(key) is True db_watcher_appointments = db_manager.load_watcher_appointments() assert len(db_watcher_appointments) == 0 + # Keys of wrong type should fail + assert db_manager.delete_watcher_appointment(42) is False + def test_batch_delete_watcher_appointments(db_manager, watcher_appointments): # Let's start by adding a bunch of appointments for uuid, appointment in watcher_appointments.items(): - db_manager.store_watcher_appointment(uuid, appointment.to_dict()) + assert db_manager.store_watcher_appointment(uuid, appointment.to_dict()) is True first_half = list(watcher_appointments.keys())[: len(watcher_appointments) // 2] second_half = list(watcher_appointments.keys())[len(watcher_appointments) // 2 :] @@ -261,16 +295,19 @@ def test_delete_responder_tracker(db_manager, responder_trackers): assert len(db_responder_trackers) != 0 for key in responder_trackers.keys(): - db_manager.delete_responder_tracker(key) + assert db_manager.delete_responder_tracker(key) is True db_responder_trackers = db_manager.load_responder_trackers() assert len(db_responder_trackers) == 0 + # Keys of wrong type should fail + assert db_manager.delete_responder_tracker(42) is False + def test_batch_delete_responder_trackers(db_manager, responder_trackers): # Let's start by adding a bunch of appointments for uuid, value in responder_trackers.items(): - db_manager.store_responder_tracker(uuid, {"value": value}) + assert db_manager.store_responder_tracker(uuid, {"value": value}) is True first_half = list(responder_trackers.keys())[: len(responder_trackers) // 2] second_half = list(responder_trackers.keys())[len(responder_trackers) // 2 :] @@ -293,22 +330,28 @@ def test_batch_delete_responder_trackers(db_manager, responder_trackers): def test_store_load_last_block_hash_watcher(db_manager): # Let's first create a made up block hash local_last_block_hash = get_random_value_hex(32) - db_manager.store_last_block_hash_watcher(local_last_block_hash) + assert db_manager.store_last_block_hash_watcher(local_last_block_hash) is True db_last_block_hash = db_manager.load_last_block_hash_watcher() assert local_last_block_hash == db_last_block_hash + # Wrong types for last block should fail for both store and load + assert db_manager.store_last_block_hash_watcher(42) is False + def test_store_load_last_block_hash_responder(db_manager): # Same for the responder local_last_block_hash = get_random_value_hex(32) - db_manager.store_last_block_hash_responder(local_last_block_hash) + assert db_manager.store_last_block_hash_responder(local_last_block_hash) is True db_last_block_hash = db_manager.load_last_block_hash_responder() assert local_last_block_hash == db_last_block_hash + # Wrong types for last block should fail for both store and load + assert db_manager.store_last_block_hash_responder(42) is False + def test_create_triggered_appointment_flag(db_manager): # Test that flags are added @@ -354,12 +397,15 @@ def test_delete_triggered_appointment_flag(db_manager): # Delete all entries for k in keys: - db_manager.delete_triggered_appointment_flag(k) + assert db_manager.delete_triggered_appointment_flag(k) is True # Try to load them back for k in keys: assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + k).encode("utf-8")) is None + # Keys of wrong type should fail + assert db_manager.delete_triggered_appointment_flag(42) is False + def test_batch_delete_triggered_appointment_flag(db_manager): # Let's add some flags first diff --git a/test/teos/unit/test_users_dbm.py b/test/teos/unit/test_users_dbm.py new file mode 100644 index 0000000..5066561 --- /dev/null +++ b/test/teos/unit/test_users_dbm.py @@ -0,0 +1,82 @@ +from teos.appointments_dbm import AppointmentsDBM + +from test.teos.unit.conftest import get_random_value_hex + + +stored_users = {} + + +def open_create_db(db_path): + + try: + db_manager = AppointmentsDBM(db_path) + + return db_manager + + except ValueError: + return False + + +def test_store_user(user_db_manager): + # Store user should work as long as the user_pk is properly formatted and data is a dictionary + user_pk = "02" + get_random_value_hex(32) + user_data = {"available_slots": 42} + stored_users[user_pk] = user_data + assert user_db_manager.store_user(user_pk, user_data) is True + + # Wrong pks should return False on adding + user_pk = "04" + get_random_value_hex(32) + user_data = {"available_slots": 42} + assert user_db_manager.store_user(user_pk, user_data) is False + + # Same for wrong types + assert user_db_manager.store_user(42, user_data) is False + + # And for wrong type user data + assert user_db_manager.store_user(user_pk, 42) is False + + +def test_load_user(user_db_manager): + # Loading a user we have stored should work + for user_pk, user_data in stored_users.items(): + assert user_db_manager.load_user(user_pk) == user_data + + # Random keys should fail + assert user_db_manager.load_user(get_random_value_hex(33)) is None + + # Wrong format keys should also return None + assert user_db_manager.load_user(42) is None + + +def test_delete_user(user_db_manager): + # Deleting an existing user should work + for user_pk, user_data in stored_users.items(): + assert user_db_manager.delete_user(user_pk) is True + + for user_pk, user_data in stored_users.items(): + assert user_db_manager.load_user(user_pk) is None + + # But deleting a non existing one should not fail + assert user_db_manager.delete_user(get_random_value_hex(32)) is True + + # Keys of wrong type should fail + assert user_db_manager.delete_user(42) is False + + +def test_load_all_users(user_db_manager): + # There should be no users at the moment + assert user_db_manager.load_all_users() == {} + stored_users = {} + + # Adding some and checking we get them all + for i in range(10): + user_pk = "02" + get_random_value_hex(32) + user_data = {"available_slots": i} + user_db_manager.store_user(user_pk, user_data) + stored_users[user_pk] = user_data + + all_users = user_db_manager.load_all_users() + + assert set(all_users.keys()) == set(stored_users.keys()) + for k, v in all_users.items(): + assert stored_users[k] == v From b93aa6482799930327d9ab4d72f03a1b4776f6f0 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 2 Apr 2020 10:02:11 +0200 Subject: [PATCH 56/62] Patches unchanged variable after renaming in 3e32d8a85e41dcf2255f7905b467cfa7ef791783 --- cli/teos_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/teos_cli.py b/cli/teos_cli.py index bdd4989..498ead5 100644 --- a/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -391,7 +391,7 @@ def main(args, command_line_conf): if command in commands: if command == "register": - register_data = register(compress_cli_pk, teos_url) + register_data = register(compressed_cli_pk, teos_url) if register_data: print(register_data) From 099ec5d1ce12d5e60dccc71854b13dd9f6ac57d4 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 2 Apr 2020 10:02:50 +0200 Subject: [PATCH 57/62] PEP8 fixes --- common/appointment.py | 1 - teos/responder.py | 1 - test/common/unit/test_appointment.py | 1 - test/teos/unit/test_api.py | 1 - test/teos/unit/test_db_manager.py | 1 - test/teos/unit/test_responder.py | 1 - 6 files changed, 6 deletions(-) diff --git a/common/appointment.py b/common/appointment.py index 164fb67..2d5f7ad 100644 --- a/common/appointment.py +++ b/common/appointment.py @@ -1,4 +1,3 @@ -import json import struct from binascii import unhexlify diff --git a/teos/responder.py b/teos/responder.py index 9cbb21a..7f6dce9 100644 --- a/teos/responder.py +++ b/teos/responder.py @@ -1,4 +1,3 @@ -import json from queue import Queue from threading import Thread diff --git a/test/common/unit/test_appointment.py b/test/common/unit/test_appointment.py index 36488e7..d5738a4 100644 --- a/test/common/unit/test_appointment.py +++ b/test/common/unit/test_appointment.py @@ -1,4 +1,3 @@ -import json import struct import binascii from pytest import fixture diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index 1cfed89..f19fbc2 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -7,7 +7,6 @@ from teos import HOST, PORT import teos.errors as errors from teos.watcher import Watcher from teos.inspector import Inspector -from teos.gatekeeper import Gatekeeper from teos.appointments_dbm import AppointmentsDBM from teos.responder import Responder, TransactionTracker diff --git a/test/teos/unit/test_db_manager.py b/test/teos/unit/test_db_manager.py index d4b275d..2ee337d 100644 --- a/test/teos/unit/test_db_manager.py +++ b/test/teos/unit/test_db_manager.py @@ -1,5 +1,4 @@ import os -import json import shutil import pytest diff --git a/test/teos/unit/test_responder.py b/test/teos/unit/test_responder.py index 0bec7cc..7c4d53d 100644 --- a/test/teos/unit/test_responder.py +++ b/test/teos/unit/test_responder.py @@ -1,4 +1,3 @@ -import json import pytest import random from uuid import uuid4 From 39f2628b79cc1adb20845d89ca253d632dfa3054 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 2 Apr 2020 15:15:34 +0200 Subject: [PATCH 58/62] Fixes docs and adds register help --- cli/help.py | 20 +++++++++++++---- cli/teos_cli.py | 59 ++++++++++++++++++++++++++++++------------------- 2 files changed, 52 insertions(+), 27 deletions(-) diff --git a/cli/help.py b/cli/help.py index 3b079ad..4ecf172 100644 --- a/cli/help.py +++ b/cli/help.py @@ -3,6 +3,7 @@ def show_usage(): "USAGE: " "\n\tpython teos_cli.py [global options] command [command options] [arguments]" "\n\nCOMMANDS:" + "\n\tregister \tRegisters your user public key with the tower." "\n\tadd_appointment \tRegisters a json formatted appointment with the tower." "\n\tget_appointment \tGets json formatted data about an appointment from the tower." "\n\thelp \t\t\tShows a list of commands or help for a specific command." @@ -14,12 +15,23 @@ def show_usage(): ) +def help_register(): + return ( + "NAME:" + "\n\n\tregister" + "\n\nUSAGE:" + "\n\n\tpython teos_cli.py register" + "\n\nDESCRIPTION:" + "\n\n\tRegisters your user public key with the tower." + ) + + def help_add_appointment(): return ( "NAME:" - "\tpython teos_cli add_appointment - Registers a json formatted appointment to the tower." + "\n\tadd_appointment - Registers a json formatted appointment to the tower." "\n\nUSAGE:" - "\tpython teos_cli add_appointment [command options] appointment/path_to_appointment_file" + "\n\tpython teos_cli.py add_appointment [command options] appointment/path_to_appointment_file" "\n\nDESCRIPTION:" "\n\n\tRegisters a json formatted appointment to the tower." "\n\tif -f, --file *is* specified, then the command expects a path to a json file instead of a json encoded " @@ -33,9 +45,9 @@ def help_add_appointment(): def help_get_appointment(): return ( "NAME:" - "\tpython teos_cli get_appointment - Gets json formatted data about an appointment from the tower." + "\n\tget_appointment - Gets json formatted data about an appointment from the tower." "\n\nUSAGE:" - "\tpython teos_cli get_appointment appointment_locator" + "\n\tpython teos_cli.py get_appointment appointment_locator" "\n\nDESCRIPTION:" "\n\n\tGets json formatted data about an appointment from the tower.\n" ) diff --git a/cli/teos_cli.py b/cli/teos_cli.py index 498ead5..b53f8c3 100644 --- a/cli/teos_cli.py +++ b/cli/teos_cli.py @@ -10,7 +10,7 @@ from getopt import getopt, GetoptError from requests import ConnectTimeout, ConnectionError from requests.exceptions import MissingSchema, InvalidSchema, InvalidURL -from cli.help import show_usage, help_add_appointment, help_get_appointment +from cli.help import show_usage, help_add_appointment, help_get_appointment, help_register from cli import DEFAULT_CONF, DATA_DIR, CONF_FILE_NAME, LOG_PREFIX import common.cryptographer @@ -21,14 +21,26 @@ from common.appointment import Appointment from common.config_loader import ConfigLoader from common.cryptographer import Cryptographer from common.tools import setup_logging, setup_data_folder -from common.tools import check_sha256_hex_format, check_locator_format, compute_locator, check_compressed_pk_format +from common.tools import is_256b_hex_str, is_locator, compute_locator, is_compressed_pk logger = Logger(actor="Client", log_name_prefix=LOG_PREFIX) common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_PREFIX) def register(compressed_pk, teos_url): - if not check_compressed_pk_format(compressed_pk): + """ + Registers the user to the tower. + + Args: + compressed_pk (:obj:`str`): a 33-byte hex-encoded compressed public key representing the user. + teos_url (:obj:`str`): the teos base url. + + Returns: + :obj:`dict` or :obj:`None`: a dictionary containing the tower response if the registration succeeded. ``None`` + otherwise. + """ + + if not is_compressed_pk(compressed_pk): logger.error("The cli public key is not valid") return None @@ -45,8 +57,7 @@ def register(compressed_pk, teos_url): def add_appointment(appointment_data, cli_sk, teos_pk, teos_url, appointments_folder_path): """ - Manages the add_appointment command, from argument parsing, trough sending the appointment to the tower, until - saving the appointment receipt. + Manages the add_appointment command. The life cycle of the function is as follows: - Check that the given commitment_txid is correct (proper format and not missing) @@ -58,8 +69,6 @@ def add_appointment(appointment_data, cli_sk, teos_pk, teos_url, appointments_fo - Check the tower's response and signature - Store the receipt (appointment + signature) on disk - If any of the above-mentioned steps fails, the method returns false, otherwise it returns true. - Args: appointment_data (:obj:`dict`): a dictionary containing the appointment data. cli_sk (:obj:`PrivateKey`): the client's private key. @@ -69,7 +78,7 @@ def add_appointment(appointment_data, cli_sk, teos_pk, teos_url, appointments_fo Returns: - :obj:`bool`: True if the appointment is accepted by the tower and the receipt is properly stored, false if any + :obj:`bool`: True if the appointment is accepted by the tower and the receipt is properly stored. False if any error occurs during the process. """ @@ -77,7 +86,7 @@ def add_appointment(appointment_data, cli_sk, teos_pk, teos_url, appointments_fo logger.error("The provided appointment JSON is empty") return False - if not check_sha256_hex_format(appointment_data.get("tx_id")): + if not is_256b_hex_str(appointment_data.get("tx_id")): logger.error("The provided txid is not valid") return False @@ -141,13 +150,13 @@ def get_appointment(locator, cli_sk, teos_pk, teos_url): teos_url (:obj:`str`): the teos base url. Returns: - :obj:`dict` or :obj:`None`: a dictionary containing thew appointment data if the locator is valid and the tower + :obj:`dict` or :obj:`None`: a dictionary containing the appointment data if the locator is valid and the tower responds. ``None`` otherwise. """ # FIXME: All responses from the tower should be signed. Not using teos_pk atm. - valid_locator = check_locator_format(locator) + valid_locator = is_locator(locator) if not valid_locator: logger.error("The provided locator is not valid", locator=locator) @@ -171,13 +180,14 @@ def load_keys(teos_pk_path, cli_sk_path, cli_pk_path): Loads all the keys required so sign, send, and verify the appointment. Args: - teos_pk_path (:obj:`str`): path to the TEOS public key file. + teos_pk_path (:obj:`str`): path to the tower public key file. cli_sk_path (:obj:`str`): path to the client private key file. cli_pk_path (:obj:`str`): path to the client public key file. Returns: - :obj:`tuple` or ``None``: a three item tuple containing a teos_pk object, cli_sk object and the cli_sk_der - encoded key if all keys can be loaded. ``None`` otherwise. + :obj:`tuple` or ``None``: a three-item tuple containing a ``PrivateKey``, a ``PublicKey`` and a ``str`` + representing the tower pk, user sk and user compressed pk respectively if all keys can be loaded. + ``None`` otherwise. """ if teos_pk_path is None: @@ -228,7 +238,7 @@ def post_request(data, endpoint): Returns: :obj:`dict` or ``None``: a json-encoded dictionary with the server response if the data can be posted. - None otherwise. + ``None`` otherwise. """ try: @@ -251,14 +261,14 @@ def post_request(data, endpoint): def process_post_response(response): """ - Processes the server response to an post request. + Processes the server response to a post request. Args: - response (:obj:`requests.models.Response`): a ``Response`` object obtained from the sent request. + response (:obj:`requests.models.Response`): a ``Response`` object obtained from the request. Returns: - :obj:`dict` or :obj:`None`: a dictionary containing the tower's response data if it can be properly parsed and - the response type is ``HTTP_OK``. ``None`` otherwise. + :obj:`dict` or :obj:`None`: a dictionary containing the tower's response data if the response type is + ``HTTP_OK`` and the response can be properly parsed. ``None`` otherwise. """ if not response: @@ -287,8 +297,8 @@ def parse_add_appointment_args(args): Parses the arguments of the add_appointment command. Args: - args (:obj:`list`): a list of arguments to pass to ``parse_add_appointment_args``. Must contain a json encoded - appointment, or the file option and the path to a file containing a json encoded appointment. + args (:obj:`list`): a list of command line arguments that must contain a json encoded appointment, or the file + option and the path to a file containing a json encoded appointment. Returns: :obj:`dict` or :obj:`None`: A dictionary containing the appointment data if it can be loaded. ``None`` @@ -332,7 +342,7 @@ def parse_add_appointment_args(args): def save_appointment_receipt(appointment, signature, appointments_folder_path): """ - Saves an appointment receipt to disk. A receipt consists in an appointment and a signature from the tower. + Saves an appointment receipt to disk. A receipt consists of an appointment and a signature from the tower. Args: appointment (:obj:`Appointment `): the appointment to be saved on disk. @@ -340,7 +350,7 @@ def save_appointment_receipt(appointment, signature, appointments_folder_path): appointments_folder_path (:obj:`str`): the path to the appointments folder. Returns: - :obj:`bool`: True if the appointment if properly saved, false otherwise. + :obj:`bool`: True if the appointment if properly saved. False otherwise. Raises: IOError: if an error occurs whilst writing the file on disk. @@ -420,6 +430,9 @@ def main(args, command_line_conf): if args: command = args.pop(0) + if command == "register": + sys.exit(help_register()) + if command == "add_appointment": sys.exit(help_add_appointment()) From fe73ee7298bf20b34c21d26bc680550e905f9bf1 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 2 Apr 2020 15:20:04 +0200 Subject: [PATCH 59/62] Fixes comments, docstrings and some renamings --- common/appointment.py | 18 +++++------ common/config_loader.py | 17 +++++----- common/cryptographer.py | 59 ++++++++++++++++++---------------- common/logger.py | 11 ++++--- common/tools.py | 25 +++++++------- teos/api.py | 42 ++++++++++++------------ teos/appointments_dbm.py | 36 +++++++++++++-------- teos/carrier.py | 17 +++++----- teos/chain_monitor.py | 8 ++--- teos/cleaner.py | 12 ++++--- teos/db_manager.py | 2 +- teos/gatekeeper.py | 27 +++++++--------- teos/inspector.py | 31 ++++++++---------- teos/responder.py | 26 ++++++++------- teos/teosd.py | 7 ++-- teos/tools.py | 6 ++-- teos/users_dbm.py | 22 +++++++------ teos/watcher.py | 22 ++++++------- test/common/unit/test_tools.py | 28 ++++++++-------- test/teos/unit/test_tools.py | 30 ----------------- 20 files changed, 213 insertions(+), 233 deletions(-) diff --git a/common/appointment.py b/common/appointment.py index 2d5f7ad..7f0f5d4 100644 --- a/common/appointment.py +++ b/common/appointment.py @@ -9,18 +9,17 @@ class Appointment: The :class:`Appointment` contains the information regarding an appointment between a client and the Watchtower. Args: - locator (:mod:`str`): A 16-byte hex-encoded value used by the tower to detect channel breaches. It serves as a + locator (:obj:`str`): A 16-byte hex-encoded value used by the tower to detect channel breaches. It serves as a trigger for the tower to decrypt and broadcast the penalty transaction. - start_time (:mod:`int`): The block height where the tower is hired to start watching for breaches. - end_time (:mod:`int`): The block height where the tower will stop watching for breaches. - to_self_delay (:mod:`int`): The ``to_self_delay`` encoded in the ``csv`` of the ``htlc`` that this appointment - is covering. + start_time (:obj:`int`): The block height where the tower is hired to start watching for breaches. + end_time (:obj:`int`): The block height where the tower will stop watching for breaches. + to_self_delay (:obj:`int`): The ``to_self_delay`` encoded in the ``csv`` of the ``to_remote`` output of the + commitment transaction that this appointment is covering. encrypted_blob (:obj:`EncryptedBlob `): An ``EncryptedBlob`` object containing an encrypted penalty transaction. The tower will decrypt it and broadcast the penalty transaction upon seeing a breach on the blockchain. """ - # DISCUSS: 35-appointment-checks def __init__(self, locator, start_time, end_time, to_self_delay, encrypted_blob): self.locator = locator self.start_time = start_time # ToDo: #4-standardize-appointment-fields @@ -36,7 +35,7 @@ class Appointment: This method is useful to load data from a database. Args: - appointment_data (:mod:`dict`): a dictionary containing the following keys: + appointment_data (:obj:`dict`): a dictionary containing the following keys: ``{locator, start_time, end_time, to_self_delay, encrypted_blob}`` Returns: @@ -62,11 +61,10 @@ class Appointment: def to_dict(self): """ - Exports an appointment as a dictionary. + Encodes an appointment as a dictionary. Returns: :obj:`dict`: A dictionary containing the appointment attributes. - """ # ToDO: #3-improve-appointment-structure @@ -90,7 +88,7 @@ class Appointment: All values are big endian. Returns: - :mod:`bytes`: The serialized data to be signed. + :obj:`bytes`: The serialized data to be signed. """ return ( unhexlify(self.locator) diff --git a/common/config_loader.py b/common/config_loader.py index a7c1dd8..d0bafb0 100644 --- a/common/config_loader.py +++ b/common/config_loader.py @@ -21,14 +21,14 @@ class ConfigLoader: data_dir (:obj:`str`): the path to the data directory where the configuration file may be found. conf_file_path (:obj:`str`): the path to the config file (the file may not exist). conf_fields (:obj:`dict`): a dictionary populated with the configuration params and the expected types. - follows the same format as default_conf. + It follows the same format as default_conf. command_line_conf (:obj:`dict`): a dictionary containing the command line parameters that may replace the ones in default / config file. """ def __init__(self, data_dir, conf_file_name, default_conf, command_line_conf): self.data_dir = data_dir - self.conf_file_path = self.data_dir + conf_file_name + self.conf_file_path = os.path.join(self.data_dir, conf_file_name) self.conf_fields = default_conf self.command_line_conf = command_line_conf @@ -36,13 +36,13 @@ class ConfigLoader: """ Builds a config dictionary from command line, config file and default configuration parameters. - The priority if as follows: + The priority is as follows: - command line - config file - defaults Returns: - obj:`dict`: a dictionary containing all the configuration parameters. + :obj:`dict`: a dictionary containing all the configuration parameters. """ @@ -50,6 +50,7 @@ class ConfigLoader: file_config = configparser.ConfigParser() file_config.read(self.conf_file_path) + # Load parameters and cast them to int if necessary if file_config: for sec in file_config.sections(): for k, v in file_config.items(sec): @@ -82,10 +83,10 @@ class ConfigLoader: Returns: :obj:`dict`: A dictionary with the same keys as the provided one, but containing only the "value" field as - value if the provided ``conf_fields`` where correct. + value if the provided ``conf_fields`` are correct. Raises: - ValueError: If any of the dictionary elements does not have the expected type + :obj:`ValueError`: If any of the dictionary elements does not have the expected type. """ conf_dict = {} @@ -104,11 +105,11 @@ class ConfigLoader: def extend_paths(self): """ - Extends the relative paths of the ``conf_fields`` dictionary with ``data_dir``. + Extends the relative paths of the ``conf_fields`` dictionary with ``data_dir``. If an absolute path is given, it'll remain the same. """ for key, field in self.conf_fields.items(): - if field.get("path") is True and isinstance(field.get("value"), str): + if field.get("path") and isinstance(field.get("value"), str): self.conf_fields[key]["value"] = os.path.join(self.data_dir, self.conf_fields[key]["value"]) diff --git a/common/cryptographer.py b/common/cryptographer.py index e7d4fac..e4fa7f8 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -6,14 +6,14 @@ from coincurve import PrivateKey, PublicKey from cryptography.exceptions import InvalidTag from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305 -from common.tools import check_sha256_hex_format +from common.tools import is_256b_hex_str LN_MESSAGE_PREFIX = b"Lightning Signed Message:" def sha256d(message): """ - Compute the sha245d (double sha256) of a given by message. + Compute the sha256 (double sha256) of a given by message. Args: message(:obj:`bytes`): the message to be used as input to the hash function. @@ -46,10 +46,9 @@ def hash_160(message): # NOTCOVERED def sigrec_encode(rsig_rid): """ - Encodes a pk-recoverable signature to be used in LN. ```rsig_rid`` can be obtained trough + Encodes a pk-recoverable signature to be used in LN. ``rsig_rid`` can be obtained trough ``PrivateKey.sign_recoverable``. The required format has the recovery id as the last byte, and for signing LN - messages we need it as the first. - From: https://twitter.com/rusty_twit/status/1182102005914800128 + messages we need it as the first. From: https://twitter.com/rusty_twit/status/1182102005914800128 Args: rsig_rid(:obj:`bytes`): the signature to be encoded. @@ -94,7 +93,7 @@ logger = None class Cryptographer: """ - The :class:`Cryptographer` is the class in charge of all the cryptography in the tower. + The :class:`Cryptographer` is in charge of all the cryptography in the tower. """ @staticmethod @@ -104,21 +103,21 @@ class Cryptographer: formatted. Args: - data(:mod:`str`): the data to be encrypted. - secret(:mod:`str`): the secret used to derive the encryption key. + data(:obj:`str`): the data to be encrypted. + secret(:obj:`str`): the secret used to derive the encryption key. Returns: :obj:`bool`: Whether or not the ``key`` and ``data`` are properly formatted. Raises: - ValueError: if either the ``key`` or ``data`` is not properly formatted. + :obj:`ValueError`: if either the ``key`` or ``data`` is not properly formatted. """ if len(data) % 2: error = "Incorrect (Odd-length) value" raise ValueError(error) - if not check_sha256_hex_format(secret): + if not is_256b_hex_str(secret): error = "Secret must be a 32-byte hex value (64 hex chars)" raise ValueError(error) @@ -127,16 +126,19 @@ class Cryptographer: @staticmethod def encrypt(blob, secret): """ - Encrypts a given :mod:`Blob ` data using ``CHACHA20POLY1305``. + Encrypts a given :obj:`Blob ` data using ``CHACHA20POLY1305``. ``SHA256(secret)`` is used as ``key``, and ``0 (12-byte)`` as ``iv``. Args: - blob (:mod:`Blob `): a ``Blob`` object containing a raw penalty transaction. - secret (:mod:`str`): a value to used to derive the encryption key. Should be the dispute txid. + blob (:obj:`Blob `): a ``Blob`` object containing a raw penalty transaction. + secret (:obj:`str`): a value to used to derive the encryption key. Should be the dispute txid. Returns: :obj:`str`: The encrypted data (hex encoded). + + Raises: + :obj:`ValueError`: if either the ``secret`` or ``blob`` is not properly formatted. """ Cryptographer.check_data_key_format(blob.data, secret) @@ -162,17 +164,20 @@ class Cryptographer: # ToDo: #20-test-tx-decrypting-edge-cases def decrypt(encrypted_blob, secret): """ - Decrypts a given :mod:`EncryptedBlob ` using ``CHACHA20POLY1305``. + Decrypts a given :obj:`EncryptedBlob ` using ``CHACHA20POLY1305``. ``SHA256(secret)`` is used as ``key``, and ``0 (12-byte)`` as ``iv``. Args: - encrypted_blob(:mod:`EncryptedBlob `): an ``EncryptedBlob`` + encrypted_blob(:obj:`EncryptedBlob `): an ``EncryptedBlob`` potentially containing a penalty transaction. - secret (:mod:`str`): a value to used to derive the decryption key. Should be the dispute txid. + secret (:obj:`str`): a value to used to derive the decryption key. Should be the dispute txid. Returns: :obj:`str`: The decrypted data (hex encoded). + + Raises: + :obj:`ValueError`: if either the ``secret`` or ``encrypted_blob`` is not properly formatted. """ Cryptographer.check_data_key_format(encrypted_blob.data, secret) @@ -234,17 +239,14 @@ class Cryptographer: @staticmethod def load_private_key_der(sk_der): """ - Creates a :mod:`PrivateKey` object from a given ``DER`` encoded private key. + Creates a :obj:`PrivateKey` from a given ``DER`` encoded private key. Args: - sk_der(:mod:`str`): a private key encoded in ``DER`` format. + sk_der(:obj:`str`): a private key encoded in ``DER`` format. Returns: - :mod:`PrivateKey`: A ``PrivateKey`` object. - - Raises: - ValueError: if the provided ``pk_der`` data cannot be deserialized (wrong size or format). - TypeError: if the provided ``pk_der`` data is not a string. + :obj:`PrivateKey` or :obj:`None`: A ``PrivateKey`` object. if the private key can be loaded. `None` + otherwise. """ try: sk = PrivateKey.from_der(sk_der) @@ -261,14 +263,14 @@ class Cryptographer: @staticmethod def sign(message, sk): """ - Signs a given data using a given secret key using ECDSA. + Signs a given data using a given secret key using ECDSA over secp256k1. Args: message(:obj:`bytes`): the data to be signed. sk(:obj:`PrivateKey`): the ECDSA secret key used to signed the data. Returns: - :obj:`str`: The zbase32 signature of the given message. + :obj:`str` or :obj:`None`: The zbase32 signature of the given message is it can be signed. `None` otherwise. """ if not isinstance(message, bytes): @@ -295,7 +297,7 @@ class Cryptographer: zb32_sig(:obj:`str`): the zbase32 signature of the message. Returns: - :obj:`PublicKey`: The recovered public key. + :obj:`PublicKey` or :obj:`None`: The recovered public key if it can be recovered. `None` otherwise. """ if not isinstance(message, bytes): @@ -345,13 +347,14 @@ class Cryptographer: @staticmethod def get_compressed_pk(pk): """ - Computes a compressed, hex encoded, public key given a ``PublicKey`` object. + Computes a compressed, hex-encoded, public key given a ``PublicKey``. Args: pk(:obj:`PublicKey`): a given public key. Returns: - :obj:`str`: A compressed, hex encoded, public key (33-byte long) + :obj:`str` or :obj:`None`: A compressed, hex-encoded, public key (33-byte long) if it can be compressed. + `None` oterwise. """ if not isinstance(pk, PublicKey): diff --git a/common/logger.py b/common/logger.py index 136b330..791a0ed 100644 --- a/common/logger.py +++ b/common/logger.py @@ -15,9 +15,10 @@ class _StructuredMessage: class Logger: """ - The :class:`Logger` is the class in charge of logging events into the log file. + The :class:`Logger` is in charge of logging events into the log file. Args: + log_name_prefix (:obj:`str`): the prefix of the logger where the data will be stored in (server, client, ...). actor (:obj:`str`): the system actor that is logging the event (e.g. ``Watcher``, ``Cryptographer``, ...). """ @@ -52,7 +53,7 @@ class Logger: Args: msg (:obj:`str`): the message to be logged. - kwargs: a ``key:value`` collection parameters to be added to the output. + kwargs (:obj:`dict`): a ``key:value`` collection parameters to be added to the output. """ self.f_logger.info(self._create_file_message(msg, **kwargs)) @@ -64,7 +65,7 @@ class Logger: Args: msg (:obj:`str`): the message to be logged. - kwargs: a ``key:value`` collection parameters to be added to the output. + kwargs (:obj:`dict`): a ``key:value`` collection parameters to be added to the output. """ self.f_logger.debug(self._create_file_message(msg, **kwargs)) @@ -76,7 +77,7 @@ class Logger: Args: msg (:obj:`str`): the message to be logged. - kwargs: a ``key:value`` collection parameters to be added to the output. + kwargs (:obj:`dict`): a ``key:value`` collection parameters to be added to the output. """ self.f_logger.error(self._create_file_message(msg, **kwargs)) @@ -88,7 +89,7 @@ class Logger: Args: msg (:obj:`str`): the message to be logged. - kwargs: a ``key:value`` collection parameters to be added to the output. + kwargs (:obj:`dict`): a ``key:value`` collection parameters to be added to the output. """ self.f_logger.warning(self._create_file_message(msg, **kwargs)) diff --git a/common/tools.py b/common/tools.py index ccd5973..7609a2c 100644 --- a/common/tools.py +++ b/common/tools.py @@ -4,21 +4,21 @@ from pathlib import Path from common.constants import LOCATOR_LEN_HEX -def check_compressed_pk_format(compressed_pk): +def is_compressed_pk(value): """ - Checks if a given value is a 33-byte hex encoded string. + Checks if a given value is a 33-byte hex-encoded string starting by 02 or 03. Args: - compressed_pk(:obj:`str`): the value to be checked. + value(:obj:`str`): the value to be checked. Returns: :obj:`bool`: Whether or not the value matches the format. """ - return isinstance(compressed_pk, str) and re.match(r"^0[2-3][0-9A-Fa-f]{64}$", compressed_pk) is not None + return isinstance(value, str) and re.match(r"^0[2-3][0-9A-Fa-f]{64}$", value) is not None -def check_sha256_hex_format(value): +def is_256b_hex_str(value): """ Checks if a given value is a 32-byte hex encoded string. @@ -31,7 +31,7 @@ def check_sha256_hex_format(value): return isinstance(value, str) and re.match(r"^[0-9A-Fa-f]{64}$", value) is not None -def check_locator_format(value): +def is_locator(value): """ Checks if a given value is a 16-byte hex encoded string. @@ -61,7 +61,7 @@ def setup_data_folder(data_folder): Create a data folder for either the client or the server side if the folder does not exists. Args: - data_folder (:obj:`str`): the path of the folder + data_folder (:obj:`str`): the path of the folder. """ Path(data_folder).mkdir(parents=True, exist_ok=True) @@ -69,9 +69,12 @@ def setup_data_folder(data_folder): def setup_logging(log_file_path, log_name_prefix): """ - Setups a couple of loggers (console and file) given a prefix and a file path. The log names are: + Setups a couple of loggers (console and file) given a prefix and a file path. - prefix | _file_log and prefix | _console_log + The log names are: + + prefix | _file_log + prefix | _console_log Args: log_file_path (:obj:`str`): the path of the file to output the file log. @@ -80,10 +83,10 @@ def setup_logging(log_file_path, log_name_prefix): if not isinstance(log_file_path, str): print(log_file_path) - raise ValueError("Wrong log file path.") + raise ValueError("Wrong log file path") if not isinstance(log_name_prefix, str): - raise ValueError("Wrong log file name.") + raise ValueError("Wrong log file name") # Create the file logger f_logger = logging.getLogger("{}_file_log".format(log_name_prefix)) diff --git a/teos/api.py b/teos/api.py index ec71206..0859c86 100644 --- a/teos/api.py +++ b/teos/api.py @@ -45,7 +45,7 @@ def get_remote_addr(): # NOTCOVERED: not sure how to monkey path this one. May be related to #77 def get_request_data_json(request): """ - Gets the content of a json POST request and makes sure ir decodes to a Python dictionary. + Gets the content of a json POST request and makes sure it decodes to a dictionary. Args: request (:obj:`Request`): the request sent by the user. @@ -54,7 +54,7 @@ def get_request_data_json(request): :obj:`dict`: the dictionary parsed from the json request. Raises: - :obj:`TypeError`: if the request is not json encoded or it does not decodes to a Python dictionary. + :obj:`TypeError`: if the request is not json encoded or it does not decodes to a dictionary. """ if request.is_json: @@ -69,13 +69,14 @@ def get_request_data_json(request): class API: """ - The :class:`API` is in charge of the interface between the user and the tower. It handles and server user requests. + The :class:`API` is in charge of the interface between the user and the tower. It handles and serves user requests. Args: inspector (:obj:`Inspector `): an ``Inspector`` instance to check the correctness of - the received data. + the received appointment data. watcher (:obj:`Watcher `): a ``Watcher`` instance to pass the requests to. - gatekeeper (:obj:`Watcher `): a `Gatekeeper` instance in charge to gatekeep the API. + gatekeeper (:obj:`Watcher `): a `Gatekeeper` instance in charge to control the user + access. """ def __init__(self, inspector, watcher, gatekeeper): @@ -104,12 +105,11 @@ class API: Users register by sending a public key to the proper endpoint. This is exploitable atm, but will be solved when payments are introduced. - Returns: - :obj:`tuple`: A tuple containing the response (``json``) and response code (``int``). For accepted requests, - the ``rcode`` is always 200 and the response contains a json with the public key and number of slots in the - subscription. For rejected requests, the ``rcode`` is a 404 and the value contains an application specific - error, and an error message. Error messages can be found at :mod:`Errors `. + :obj:`tuple`: A tuple containing the response (:obj:`str`) and response code (:obj:`int`). For accepted + requests, the ``rcode`` is always 200 and the response contains a json with the public key and number of + slots in the subscription. For rejected requests, the ``rcode`` is a 404 and the value contains an + application error, and an error message. Error messages can be found at :mod:`Errors `. """ remote_addr = get_remote_addr() @@ -150,12 +150,12 @@ class API: Main endpoint of the Watchtower. The client sends requests (appointments) to this endpoint to request a job to the Watchtower. Requests must be - json encoded and contain an ``appointment`` field and optionally a ``signature`` and ``public_key`` fields. + json encoded and contain an ``appointment`` and ``signature`` fields. Returns: - :obj:`tuple`: A tuple containing the response (``json``) and response code (``int``). For accepted - appointments, the ``rcode`` is always 200 and the response contains the receipt signature. For rejected - appointments, the ``rcode`` is a 404 and the value contains an application specific error, and an error + :obj:`tuple`: A tuple containing the response (:obj:`str`) and response code (:obj:`int`). For accepted + appointments, the ``rcode`` is always 200 and the response contains the receipt signature (json). For + rejected appointments, the ``rcode`` is a 404 and the value contains an application error, and an error message. Error messages can be found at :mod:`Errors `. """ @@ -185,16 +185,16 @@ class API: appointment_uuid = hash_160("{}{}".format(appointment.locator, user_pk)) appointment_summary = self.watcher.get_appointment_summary(appointment_uuid) - # For updates we only reserve the slot difference provided the new one is bigger. if appointment_summary: used_slots = ceil(appointment_summary.get("size") / ENCRYPTED_BLOB_MAX_SIZE_HEX) required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) slot_diff = required_slots - used_slots + # For updates we only reserve the slot difference provided the new one is bigger. required_slots = slot_diff if slot_diff > 0 else 0 - # For regular appointments 1 slot is reserved per ENCRYPTED_BLOB_MAX_SIZE_HEX block. else: + # For regular appointments 1 slot is reserved per ENCRYPTED_BLOB_MAX_SIZE_HEX block. slot_diff = 0 required_slots = ceil(len(appointment.encrypted_blob.data) / ENCRYPTED_BLOB_MAX_SIZE_HEX) @@ -245,7 +245,9 @@ class API: The information is requested by ``locator``. Returns: - :obj:`dict`: A json formatted dictionary containing information about the requested appointment. + :obj:`str`: A json formatted dictionary containing information about the requested appointment. + + Returns not found if the user does not have the requested appointment or the locator is invalid. A ``status`` flag is added to the data provided by either the :obj:`Watcher ` or the :obj:`Responder ` that signals the status of the appointment. @@ -312,10 +314,8 @@ class API: This endpoint should only be accessible by the administrator. Requests are only allowed from localhost. Returns: - :obj:`dict`: A json formatted dictionary containing all the appointments hold by the - :obj:`Watcher ` (``watcher_appointments``) and by the - :obj:`Responder ` (``responder_trackers``). - + :obj:`str`: A json formatted dictionary containing all the appointments hold by the ``Watcher`` + (``watcher_appointments``) and by the ``Responder>`` (``responder_trackers``). """ # ToDo: #15-add-system-monitor diff --git a/teos/appointments_dbm.py b/teos/appointments_dbm.py index ba1a9f8..ebd027a 100644 --- a/teos/appointments_dbm.py +++ b/teos/appointments_dbm.py @@ -19,7 +19,7 @@ TRIGGERED_APPOINTMENTS_PREFIX = "ta" class AppointmentsDBM(DBManager): """ - The :class:`AppointmentsDBM` is the class in charge of interacting with the appointments database (``LevelDB``). + The :class:`AppointmentsDBM` is in charge of interacting with the appointments database (``LevelDB``). Keys and values are stored as bytes in the database but processed as strings by the manager. The database is split in six prefixes: @@ -36,8 +36,8 @@ class AppointmentsDBM(DBManager): database will be create if the specified path does not contain one. Raises: - ValueError: If the provided ``db_path`` is not a string. - plyvel.Error: If the db is currently unavailable (being used by another process). + :obj:`ValueError`: If the provided ``db_path`` is not a string. + :obj:`plyvel.Error`: If the db is currently unavailable (being used by another process). """ def __init__(self, db_path): @@ -78,7 +78,11 @@ class AppointmentsDBM(DBManager): def get_last_known_block(self, key): """ - Loads the last known block given a key (either ``WATCHER_LAST_BLOCK_KEY`` or ``RESPONDER_LAST_BLOCK_KEY``). + Loads the last known block given a key. + + Args: + key (:obj:`str`): the identifier of the db to look into (either ``WATCHER_LAST_BLOCK_KEY`` or + ``RESPONDER_LAST_BLOCK_KEY``). Returns: :obj:`str` or :obj:`None`: A 16-byte hex-encoded str representing the last known block hash. @@ -93,9 +97,12 @@ class AppointmentsDBM(DBManager): return last_block - def load_watcher_appointment(self, key): + def load_watcher_appointment(self, uuid): """ - Loads an appointment from the database using ``WATCHER_PREFIX`` as prefix to the given ``key``. + Loads an appointment from the database using ``WATCHER_PREFIX`` as prefix to the given ``uuid``. + + Args: + uuid (:obj:`str`): the appointment's unique identifier. Returns: :obj:`dict`: A dictionary containing the appointment data if they ``key`` is found. @@ -104,16 +111,19 @@ class AppointmentsDBM(DBManager): """ try: - data = self.load_entry(key, prefix=WATCHER_PREFIX) + data = self.load_entry(uuid, prefix=WATCHER_PREFIX) data = json.loads(data) except (TypeError, json.decoder.JSONDecodeError): data = None return data - def load_responder_tracker(self, key): + def load_responder_tracker(self, uuid): """ - Loads a tracker from the database using ``RESPONDER_PREFIX`` as a prefix to the given ``key``. + Loads a tracker from the database using ``RESPONDER_PREFIX`` as a prefix to the given ``uuid``. + + Args: + uuid (:obj:`str`): the tracker's unique identifier. Returns: :obj:`dict`: A dictionary containing the tracker data if they ``key`` is found. @@ -122,7 +132,7 @@ class AppointmentsDBM(DBManager): """ try: - data = self.load_entry(key, prefix=RESPONDER_PREFIX) + data = self.load_entry(uuid, prefix=RESPONDER_PREFIX) data = json.loads(data) except (TypeError, json.decoder.JSONDecodeError): data = None @@ -134,7 +144,7 @@ class AppointmentsDBM(DBManager): Loads all the appointments from the database (all entries with the ``WATCHER_PREFIX`` prefix). Args: - include_triggered (:obj:`bool`): Whether to include the appointments flagged as triggered or not. ``False`` + include_triggered (:obj:`bool`): whether to include the appointments flagged as triggered or not. ``False`` by default. Returns: @@ -168,7 +178,7 @@ class AppointmentsDBM(DBManager): Args: uuid (:obj:`str`): the identifier of the appointment to be stored. - appointment (:obj: `dict`): an appointment encoded as dictionary. + appointment (:obj:`dict`): an appointment encoded as dictionary. Returns: :obj:`bool`: True if the appointment was stored in the db. False otherwise. @@ -193,7 +203,7 @@ class AppointmentsDBM(DBManager): Args: uuid (:obj:`str`): the identifier of the appointment to be stored. - tracker (:obj: `dict`): a tracker encoded as dictionary. + tracker (:obj:`dict`): a tracker encoded as dictionary. Returns: :obj:`bool`: True if the tracker was stored in the db. False otherwise. diff --git a/teos/carrier.py b/teos/carrier.py index 587afa4..c8bba1c 100644 --- a/teos/carrier.py +++ b/teos/carrier.py @@ -36,12 +36,12 @@ class Receipt: class Carrier: """ - The :class:`Carrier` is the class in charge of interacting with ``bitcoind`` to send/get transactions. It uses - :obj:`Receipt` objects to report about the sending outcome. + The :class:`Carrier` is in charge of interacting with ``bitcoind`` to send/get transactions. It uses :obj:`Receipt` + objects to report about the sending outcome. Args: btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind - (rpc user, rpc passwd, host and port) + (rpc user, rpc password, host and port) Attributes: issued_receipts (:obj:`dict`): a dictionary of issued receipts to prevent resending the same transaction over @@ -135,18 +135,17 @@ class Carrier: Returns: :obj:`dict` or :obj:`None`: A dictionary with the transaction data if the transaction can be found on the - chain. - Returns ``None`` otherwise. + chain. ``None`` otherwise. """ try: tx_info = bitcoin_cli(self.btc_connect_params).getrawtransaction(txid, 1) + return tx_info except JSONRPCException as e: - tx_info = None # While it's quite unlikely, the transaction that was already in the blockchain could have been - # reorged while we were querying bitcoind to get the confirmation count. In such a case we just - # restart the tracker + # reorged while we were querying bitcoind to get the confirmation count. In that case we just restart + # the tracker if e.error.get("code") == rpc_errors.RPC_INVALID_ADDRESS_OR_KEY: logger.info("Transaction not found in mempool nor blockchain", txid=txid) @@ -154,4 +153,4 @@ class Carrier: # If something else happens (unlikely but possible) log it so we can treat it in future releases logger.error("JSONRPCException", method="Carrier.get_transaction", error=e.error) - return tx_info + return None diff --git a/teos/chain_monitor.py b/teos/chain_monitor.py index 186a31d..654fffa 100644 --- a/teos/chain_monitor.py +++ b/teos/chain_monitor.py @@ -10,8 +10,8 @@ logger = Logger(actor="ChainMonitor", log_name_prefix=LOG_PREFIX) class ChainMonitor: """ - The :class:`ChainMonitor` is the class in charge of monitoring the blockchain (via ``bitcoind``) to detect new - blocks on top of the best chain. If a new best block is spotted, the chain monitor will notify the + The :class:`ChainMonitor` is in charge of monitoring the blockchain (via ``bitcoind``) to detect new blocks on top + of the best chain. If a new best block is spotted, the chain monitor will notify the :obj:`Watcher ` and the :obj:`Responder ` using ``Queues``. The :class:`ChainMonitor` monitors the chain using two methods: ``zmq`` and ``polling``. Blocks are only notified @@ -34,7 +34,6 @@ class ChainMonitor: watcher_queue (:obj:`Queue`): a queue to send new best tips to the :obj:`Watcher `. responder_queue (:obj:`Queue`): a queue to send new best tips to the :obj:`Responder `. - polling_delta (:obj:`int`): time between polls (in seconds). max_block_window_size (:obj:`int`): max size of last_tips. block_processor (:obj:`BlockProcessor `): a blockProcessor instance. @@ -75,7 +74,6 @@ class ChainMonitor: Args: block_hash (:obj:`str`): the new block hash to be sent to the subscribers. - block_hash (:obj:`str`): the new block hash to be sent to the subscribers. """ self.watcher_queue.put(block_hash) @@ -90,7 +88,7 @@ class ChainMonitor: block_hash (:obj:`block_hash`): the new best tip. Returns: - (:obj:`bool`): ``True`` is the state was successfully updated, ``False`` otherwise. + :obj:`bool`: True is the state was successfully updated, False otherwise. """ if block_hash != self.best_tip and block_hash not in self.last_tips: diff --git a/teos/cleaner.py b/teos/cleaner.py index e0aefdc..a9cba0a 100644 --- a/teos/cleaner.py +++ b/teos/cleaner.py @@ -7,7 +7,7 @@ logger = Logger(actor="Cleaner", log_name_prefix=LOG_PREFIX) class Cleaner: """ - The :class:`Cleaner` is the class in charge of removing expired/completed data from the tower. + The :class:`Cleaner` is in charge of removing expired/completed data from the tower. Mutable objects (like dicts) are passed-by-reference in Python, so no return is needed for the Cleaner. """ @@ -15,15 +15,16 @@ class Cleaner: @staticmethod def delete_appointment_from_memory(uuid, appointments, locator_uuid_map): """ - Deletes an appointment from memory (appointments and locator_uuid_map dictionaries). If the given appointment - does not share locator with any other, the map will completely removed, otherwise, the uuid will be removed from - the map. + Deletes an appointment from memory (``appointments`` and ``locator_uuid_map`` dictionaries). If the given + appointment does not share locator with any other, the map will completely removed, otherwise, the uuid will be + removed from the map. Args: uuid (:obj:`str`): the identifier of the appointment to be deleted. appointments (:obj:`dict`): the appointments dictionary from where the appointment should be removed. locator_uuid_map (:obj:`dict`): the locator:uuid map from where the appointment should also be removed. """ + locator = appointments[uuid].get("locator") # Delete the appointment @@ -136,6 +137,7 @@ class Cleaner: db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance to interact with the database. """ + locator_maps_to_update = {} for uuid in completed_appointments: @@ -161,7 +163,7 @@ class Cleaner: @staticmethod def flag_triggered_appointments(triggered_appointments, appointments, locator_uuid_map, db_manager): """ - Deletes a list of triggered appointment from memory (:obj:`Watcher `) and flags them as + Deletes a list of triggered appointment from memory (:obj:`Watcher `) and flags them as triggered on disk. Args: diff --git a/teos/db_manager.py b/teos/db_manager.py index d911a4b..678147e 100644 --- a/teos/db_manager.py +++ b/teos/db_manager.py @@ -3,7 +3,7 @@ import plyvel class DBManager: """ - The :class:`DBManager` is the class in charge of interacting with a database (``LevelDB``). + The :class:`DBManager` is in charge of interacting with a database (``LevelDB``). Keys and values are stored as bytes in the database but processed as strings by the manager. Args: diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index 935afac..a4a65fb 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -1,9 +1,9 @@ -from common.tools import check_compressed_pk_format +from common.tools import is_compressed_pk from common.cryptographer import Cryptographer class NotEnoughSlots(ValueError): - """Raise this when trying to subtract more slots than a user has available.""" + """Raise this when trying to subtract more slots than a user has available""" def __init__(self, user_pk, requested_slots): self.user_pk = user_pk @@ -21,8 +21,8 @@ class IdentificationFailure(Exception): class Gatekeeper: """ - The Gatekeeper is in charge of managing the access to the tower. Only registered users are allowed to perform - actions. + The :class:`Gatekeeper` is in charge of managing the access to the tower. Only registered users are allowed to + perform actions. Attributes: registered_users (:obj:`dict`): a map of user_pk:appointment_slots. @@ -44,7 +44,7 @@ class Gatekeeper: :obj:`int`: the number of available slots in the user subscription. """ - if not check_compressed_pk_format(user_pk): + if not is_compressed_pk(user_pk): raise ValueError("provided public key does not match expected format (33-byte hex string)") if user_pk not in self.registered_users: @@ -58,17 +58,17 @@ class Gatekeeper: def identify_user(self, message, signature): """ - Checks if the provided user signature comes from a registered user. + Checks if a request comes from a registered user by ec-recovering their public key from a signed message. Args: message (:obj:`bytes`): byte representation of the original message from where the signature was generated. - signature (:obj:`str`): the user's signature (hex encoded). + signature (:obj:`str`): the user's signature (hex-encoded). Returns: :obj:`str`: a compressed key recovered from the signature and matching a registered user. Raises: - :obj:``: if the user cannot be identified. + :obj:`IdentificationFailure`: if the user cannot be identified. """ if isinstance(message, bytes) and isinstance(signature, str): @@ -89,18 +89,16 @@ class Gatekeeper: Args: user_pk(:obj:`str`): the public key that identifies the user (33-bytes hex str). - n: the number of slots to fill. n (:obj:`int`): the number of slots to fill. Raises: - :obj:``: if the user subscription does not have enough slots. + :obj:`NotEnoughSlots`: if the user subscription does not have enough slots. """ - # We are not making sure the value passed is a integer, but the value is computed by the API and rounded before - # passing it to the gatekeeper. # DISCUSS: we may want to return a different exception if the user does not exist if user_pk in self.registered_users and n <= self.registered_users.get(user_pk).get("available_slots"): self.registered_users[user_pk]["available_slots"] -= n + self.user_db.store_user(user_pk, self.registered_users[user_pk]) else: raise NotEnoughSlots(user_pk, n) @@ -110,11 +108,10 @@ class Gatekeeper: Args: user_pk(:obj:`str`): the public key that identifies the user (33-bytes hex str). - n: the number of slots to free. + n (:obj:`int`): the number of slots to free. """ - # We are not making sure the value passed is a integer, but the value is computed by the API and rounded before - # passing it to the gatekeeper. # DISCUSS: if the user does not exist we may want to log or return an exception. if user_pk in self.registered_users: self.registered_users[user_pk]["available_slots"] += n + self.user_db.store_user(user_pk, self.registered_users[user_pk]) diff --git a/teos/inspector.py b/teos/inspector.py index c5fa9af..235de12 100644 --- a/teos/inspector.py +++ b/teos/inspector.py @@ -2,6 +2,7 @@ import re import common.cryptographer from common.logger import Logger +from common.tools import is_locator from common.constants import LOCATOR_LEN_HEX from common.appointment import Appointment @@ -50,11 +51,10 @@ class Inspector: Returns: - :obj:`Appointment `: An appointment initialized with the - provided data. + :obj:`Appointment `: An appointment initialized with the provided data. Raises: - :obj:`InspectionFailed `: if any of the fields is wrong. + :obj:`InspectionFailed`: if any of the fields is wrong. """ if appointment_data is None: @@ -64,7 +64,7 @@ class Inspector: block_height = self.block_processor.get_block_count() if block_height is None: - raise InspectionFailed(errors.UNKNOWN_JSON_RPC_EXCEPTION, "Unexpected error occurred") + raise InspectionFailed(errors.UNKNOWN_JSON_RPC_EXCEPTION, "unexpected error occurred") self.check_locator(appointment_data.get("locator")) self.check_start_time(appointment_data.get("start_time"), block_height) @@ -79,13 +79,13 @@ class Inspector: """ Checks if the provided ``locator`` is correct. - Locators must be 16-byte hex encoded strings. + Locators must be 16-byte hex-encoded strings. Args: locator (:obj:`str`): the locator to be checked. Raises: - :obj:`InspectionFailed `: if any of the fields is wrong. + :obj:`InspectionFailed`: if any of the fields is wrong. """ if locator is None: @@ -99,7 +99,7 @@ class Inspector: elif len(locator) != LOCATOR_LEN_HEX: raise InspectionFailed(errors.APPOINTMENT_WRONG_FIELD_SIZE, "wrong locator size ({})".format(len(locator))) - elif re.search(r"^[0-9A-Fa-f]+$", locator) is None: + elif not is_locator(locator): raise InspectionFailed(errors.APPOINTMENT_WRONG_FIELD_FORMAT, "wrong locator format ({})".format(locator)) @staticmethod @@ -114,12 +114,9 @@ class Inspector: block_height (:obj:`int`): the chain height. Raises: - :obj:`InspectionFailed `: if any of the fields is wrong. + :obj:`InspectionFailed`: if any of the fields is wrong. """ - # TODO: What's too close to the current height is not properly defined. Right now any appointment that is in the - # future will be accepted (even if it's only one block away). - if start_time is None: raise InspectionFailed(errors.APPOINTMENT_EMPTY_FIELD, "empty start_time received") @@ -156,7 +153,7 @@ class Inspector: block_height (:obj:`int`): the chain height. Raises: - :obj:`InspectionFailed `: if any of the fields is wrong. + :obj:`InspectionFailed`: if any of the fields is wrong. """ # TODO: What's too close to the current height is not properly defined. Right now any appointment that ends in @@ -193,11 +190,11 @@ class Inspector: To self delays must be greater or equal to ``MIN_TO_SELF_DELAY``. Args: - to_self_delay (:obj:`int`): The ``to_self_delay`` encoded in the ``csv`` of the ``htlc`` that this - appointment is covering. + to_self_delay (:obj:`int`): The ``to_self_delay`` encoded in the ``csv`` of ``to_remote`` output of the + commitment transaction this appointment is covering. Raises: - :obj:`InspectionFailed `: if any of the fields is wrong. + :obj:`InspectionFailed`: if any of the fields is wrong. """ if to_self_delay is None: @@ -229,10 +226,10 @@ class Inspector: Checks if the provided ``encrypted_blob`` may be correct. Args: - encrypted_blob (:obj:`str`): the encrypted blob to be checked (hex encoded). + encrypted_blob (:obj:`str`): the encrypted blob to be checked (hex-encoded). Raises: - :obj:`InspectionFailed `: if any of the fields is wrong. + :obj:`InspectionFailed`: if any of the fields is wrong. """ if encrypted_blob is None: diff --git a/teos/responder.py b/teos/responder.py index 7f6dce9..a5d6843 100644 --- a/teos/responder.py +++ b/teos/responder.py @@ -13,7 +13,7 @@ logger = Logger(actor="Responder", log_name_prefix=LOG_PREFIX) class TransactionTracker: """ - A :class:`TransactionTracker` is used to monitor a ``penalty_tx``. Once the dispute is seen by the + A :class:`TransactionTracker` is used to monitor a ``penalty_tx``. Once the dispute is seen by the :obj:`Watcher ` the penalty transaction is decrypted and the relevant appointment data is passed along to the :obj:`Responder`. @@ -53,7 +53,7 @@ class TransactionTracker: :obj:`TransactionTracker`: A ``TransactionTracker`` instantiated with the provided data. Raises: - ValueError: if any of the required fields is missing. + :obj:`ValueError`: if any of the required fields is missing. """ locator = tx_tracker_data.get("locator") @@ -72,7 +72,7 @@ class TransactionTracker: def to_dict(self): """ - Exports a :obj:`TransactionTracker` as a dictionary. + Encodes a :obj:`TransactionTracker` as a dictionary. Returns: :obj:`dict`: A dictionary containing the :obj:`TransactionTracker` data. @@ -91,13 +91,16 @@ class TransactionTracker: class Responder: """ - The :class:`Responder` is the class in charge of ensuring that channel breaches are dealt with. It does so handling + The :class:`Responder` is in charge of ensuring that channel breaches are dealt with. It does so handling the decrypted ``penalty_txs`` handed by the :obj:`Watcher ` and ensuring the they make it to the blockchain. Args: db_manager (:obj:`AppointmentsDBM `): a ``AppointmentsDBM`` instance to interact with the database. + carrier (:obj:`Carrier `): a ``Carrier`` instance to send transactions to bitcoind. + block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance to + get data from bitcoind. Attributes: trackers (:obj:`dict`): A dictionary containing the minimum information about the :obj:`TransactionTracker` @@ -116,7 +119,6 @@ class Responder: block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance to get data from bitcoind. last_known_block (:obj:`str`): the last block known by the ``Responder``. - """ def __init__(self, db_manager, carrier, block_processor): @@ -131,6 +133,7 @@ class Responder: self.last_known_block = db_manager.load_last_block_hash_responder() def awake(self): + """Starts a new thread to monitor the blockchain to make sure triggered appointments get enough depth""" responder_thread = Thread(target=self.do_watch, daemon=True) responder_thread.start() @@ -140,7 +143,7 @@ class Responder: """ Whether the :obj:`Responder` is on sync with ``bitcoind`` or not. Used when recovering from a crash. - The Watchtower can be instantiated with fresh or with backed up data. In the later, some triggers may have been + The Watchtower can be instantiated with fresh or with backed up data. In the later, some triggers may have been missed. In order to go back on sync both the :obj:`Watcher ` and the :obj:`Responder` need to perform the state transitions until they catch up. @@ -205,9 +208,8 @@ class Responder: """ Creates a :obj:`TransactionTracker` after successfully broadcasting a ``penalty_tx``. - A reduction of :obj:`TransactionTracker` is stored in ``trackers`` and ``tx_tracker_map`` and the - ``penalty_txid`` added to ``unconfirmed_txs`` if ``confirmations=0``. Finally, all the data is stored in the - database. + A summary of :obj:`TransactionTracker` is stored in ``trackers`` and ``tx_tracker_map`` and the ``penalty_txid`` + added to ``unconfirmed_txs`` if ``confirmations=0``. Finally, all the data is stored in the database. Args: uuid (:obj:`str`): a unique identifier for the appointment. @@ -248,7 +250,7 @@ class Responder: def do_watch(self): """ - Monitors the blockchain whilst there are pending trackers. + Monitors the blockchain for reorgs and appointment ends. This is the main method of the :obj:`Responder` and triggers tracker cleaning, rebroadcasting, reorg managing, etc. @@ -384,9 +386,9 @@ class Responder: def rebroadcast(self, txs_to_rebroadcast): """ Rebroadcasts a ``penalty_tx`` that has missed too many confirmations. In the current approach this would loop - forever si the transaction keeps not getting it. + forever if the transaction keeps not getting it. - Potentially the fees could be bumped here if the transaction has some tower dedicated outputs (or allows it + Potentially, the fees could be bumped here if the transaction has some tower dedicated outputs (or allows it trough ``ANYONECANPAY`` or something similar). Args: diff --git a/teos/teosd.py b/teos/teosd.py index 694b664..3287581 100644 --- a/teos/teosd.py +++ b/teos/teosd.py @@ -45,9 +45,10 @@ def main(command_line_conf): signal(SIGQUIT, handle_signals) # Loads config and sets up the data folder and log file - config_loader = ConfigLoader(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF, command_line_conf) + data_dir = command_line_conf.get("DATA_DIR") if "DATA_DIR" in command_line_conf else DATA_DIR + config_loader = ConfigLoader(data_dir, CONF_FILE_NAME, DEFAULT_CONF, command_line_conf) config = config_loader.build_config() - setup_data_folder(DATA_DIR) + setup_data_folder(data_dir) setup_logging(config.get("LOG_FILE"), LOG_PREFIX) logger.info("Starting TEOS") @@ -183,7 +184,7 @@ if __name__ == "__main__": except ValueError: exit("btcrpcport must be an integer") if opt in ["--datadir"]: - DATA_DIR = os.path.expanduser(arg) + command_line_conf["DATA_DIR"] = os.path.expanduser(arg) if opt in ["-h", "--help"]: exit(show_usage()) diff --git a/teos/tools.py b/teos/tools.py index dded387..269a41d 100644 --- a/teos/tools.py +++ b/teos/tools.py @@ -15,7 +15,7 @@ def bitcoin_cli(btc_connect_params): Args: btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind - (rpc user, rpc passwd, host and port) + (rpc user, rpc password, host and port) Returns: :obj:`AuthServiceProxy `: An authenticated service proxy to ``bitcoind`` @@ -40,7 +40,7 @@ def can_connect_to_bitcoind(btc_connect_params): Args: btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind - (rpc user, rpc passwd, host and port) + (rpc user, rpc password, host and port) Returns: :obj:`bool`: ``True`` if the connection can be established. ``False`` otherwise. """ @@ -62,7 +62,7 @@ def in_correct_network(btc_connect_params, network): Args: btc_connect_params (:obj:`dict`): a dictionary with the parameters to connect to bitcoind - (rpc user, rpc passwd, host and port) + (rpc user, rpc password, host and port) network (:obj:`str`): the network the tower is connected to. Returns: diff --git a/teos/users_dbm.py b/teos/users_dbm.py index d9809be..b9f3b21 100644 --- a/teos/users_dbm.py +++ b/teos/users_dbm.py @@ -5,14 +5,14 @@ from teos import LOG_PREFIX from teos.db_manager import DBManager from common.logger import Logger -from common.tools import check_compressed_pk_format +from common.tools import is_compressed_pk logger = Logger(actor="UsersDBM", log_name_prefix=LOG_PREFIX) class UsersDBM(DBManager): """ - The :class:`UsersDBM` is the class in charge of interacting with the users database (``LevelDB``). + The :class:`UsersDBM` is in charge of interacting with the users database (``LevelDB``). Keys and values are stored as bytes in the database but processed as strings by the manager. Args: @@ -20,8 +20,8 @@ class UsersDBM(DBManager): database will be create if the specified path does not contain one. Raises: - ValueError: If the provided ``db_path`` is not a string. - plyvel.Error: If the db is currently unavailable (being used by another process). + :obj:`ValueError`: If the provided ``db_path`` is not a string. + :obj:`plyvel.Error`: If the db is currently unavailable (being used by another process). """ def __init__(self, db_path): @@ -46,31 +46,33 @@ class UsersDBM(DBManager): user_data (:obj:`dict`): the user associated data, as a dictionary. Returns: - :obj:`bool`: True if the user was stored in the database, false otherwise. + :obj:`bool`: True if the user was stored in the database, False otherwise. """ - if check_compressed_pk_format(user_pk): + if is_compressed_pk(user_pk): try: self.create_entry(user_pk, json.dumps(user_data)) logger.info("Adding user to Gatekeeper's db", user_pk=user_pk) return True except json.JSONDecodeError: - logger.info("Could't add user to db. Wrong user data format.", user_pk=user_pk, user_data=user_data) + logger.info("Could't add user to db. Wrong user data format", user_pk=user_pk, user_data=user_data) return False except TypeError: - logger.info("Could't add user to db.", user_pk=user_pk, user_data=user_data) + logger.info("Could't add user to db", user_pk=user_pk, user_data=user_data) return False else: - logger.info("Could't add user to db. Wrong pk format.", user_pk=user_pk, user_data=user_data) + logger.info("Could't add user to db. Wrong pk format", user_pk=user_pk, user_data=user_data) return False def load_user(self, user_pk): """ Loads a user record from the database using the ``user_pk`` as identifier. - use_pk (:obj:`str`): a 33-byte hex-encoded string identifying the user. + Args: + + user_pk (:obj:`str`): a 33-byte hex-encoded string identifying the user. Returns: :obj:`dict`: A dictionary containing the appointment data if they ``key`` is found. diff --git a/teos/watcher.py b/teos/watcher.py index 79c5cfe..bb02ff7 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -16,8 +16,7 @@ common.cryptographer.logger = Logger(actor="Cryptographer", log_name_prefix=LOG_ class Watcher: """ - The :class:`Watcher` is the class in charge to watch for channel breaches for the appointments accepted by the - tower. + The :class:`Watcher` is in charge of watching for channel breaches for the appointments accepted by the tower. The :class:`Watcher` keeps track of the accepted appointments in ``appointments`` and, for new received block, checks if any breach has happened by comparing the txids with the appointment locators. If a breach is seen, the @@ -36,7 +35,7 @@ class Watcher: get block from bitcoind. responder (:obj:`Responder `): a ``Responder`` instance. sk_der (:obj:`bytes`): a DER encoded private key used to sign appointment receipts (signaling acceptance). - max_appointments (:obj:`int`): the maximum ammount of appointments accepted by the ``Watcher`` at the same time. + max_appointments (:obj:`int`): the maximum amount of appointments accepted by the ``Watcher`` at the same time. expiry_delta (:obj:`int`): the additional time the ``Watcher`` will keep an expired appointment around. Attributes: @@ -53,7 +52,7 @@ class Watcher: get block from bitcoind. responder (:obj:`Responder `): a ``Responder`` instance. signing_key (:mod:`PrivateKey`): a private key used to sign accepted appointments. - max_appointments (:obj:`int`): the maximum ammount of appointments accepted by the ``Watcher`` at the same time. + max_appointments (:obj:`int`): the maximum amount of appointments accepted by the ``Watcher`` at the same time. expiry_delta (:obj:`int`): the additional time the ``Watcher`` will keep an expired appointment around. Raises: @@ -73,9 +72,7 @@ class Watcher: self.signing_key = Cryptographer.load_private_key_der(sk_der) def awake(self): - """ - Starts a new thread to monitor the blockchain for channel breaches. - """ + """Starts a new thread to monitor the blockchain for channel breaches""" watcher_thread = Thread(target=self.do_watch, daemon=True) watcher_thread.start() @@ -85,13 +82,13 @@ class Watcher: def get_appointment_summary(self, uuid): """ Returns the summary of an appointment. The summary consists of the data kept in memory: - locator, end_time, and size. + {locator, end_time, and size} Args: uuid (:obj:`str`): a 16-byte hex string identifying the appointment. Returns: - :obj:`dict` or :obj:`None`: a dictionary with the appointment summary, or None if the appointment is not + :obj:`dict` or :obj:`None`: a dictionary with the appointment summary, or ``None`` if the appointment is not found. """ return self.appointments.get(uuid) @@ -100,8 +97,8 @@ class Watcher: """ Adds a new appointment to the ``appointments`` dictionary if ``max_appointments`` has not been reached. - ``add_appointment`` is the entry point of the Watcher. Upon receiving a new appointment it will start monitoring - the blockchain (``do_watch``) until ``appointments`` is empty. + ``add_appointment`` is the entry point of the ``Watcher``. Upon receiving a new appointment it will start + monitoring the blockchain (``do_watch``) until ``appointments`` is empty. Once a breach is seen on the blockchain, the :obj:`Watcher` will decrypt the corresponding :obj:`EncryptedBlob ` and pass the information to the @@ -123,7 +120,6 @@ class Watcher: - ``(True, signature)`` if the appointment has been accepted. - ``(False, None)`` otherwise. - """ if len(self.appointments) < self.max_appointments: @@ -164,7 +160,7 @@ class Watcher: def do_watch(self): """ - Monitors the blockchain whilst there are pending appointments. + Monitors the blockchain for channel breaches. This is the main method of the :obj:`Watcher` and the one in charge to pass appointments to the :obj:`Responder ` upon detecting a breach. diff --git a/test/common/unit/test_tools.py b/test/common/unit/test_tools.py index 3e862b1..4276444 100644 --- a/test/common/unit/test_tools.py +++ b/test/common/unit/test_tools.py @@ -3,9 +3,9 @@ import logging from common.constants import LOCATOR_LEN_BYTES from common.tools import ( - check_compressed_pk_format, - check_sha256_hex_format, - check_locator_format, + is_compressed_pk, + is_256b_hex_str, + is_locator, compute_locator, setup_data_folder, setup_logging, @@ -13,7 +13,7 @@ from common.tools import ( from test.common.unit.conftest import get_random_value_hex -def test_check_compressed_pk_format(): +def test_is_compressed_pk(): wrong_values = [ None, 3, @@ -34,21 +34,21 @@ def test_check_compressed_pk_format(): prefix = "02" else: prefix = "03" - assert check_compressed_pk_format(prefix + get_random_value_hex(32)) + assert is_compressed_pk(prefix + get_random_value_hex(32)) # check_user_pk must only accept values that is not a 33-byte hex string for value in wrong_values: - assert not check_compressed_pk_format(value) + assert not is_compressed_pk(value) -def test_check_sha256_hex_format(): +def test_is_256b_hex_str(): # Only 32-byte hex encoded strings should pass the test wrong_inputs = [None, str(), 213, 46.67, dict(), "A" * 63, "C" * 65, bytes(), get_random_value_hex(31)] for wtype in wrong_inputs: - assert check_sha256_hex_format(wtype) is False + assert is_256b_hex_str(wtype) is False for v in range(100): - assert check_sha256_hex_format(get_random_value_hex(32)) is True + assert is_256b_hex_str(get_random_value_hex(32)) is True def test_check_locator_format(): @@ -66,20 +66,20 @@ def test_check_locator_format(): get_random_value_hex(LOCATOR_LEN_BYTES - 1), ] for wtype in wrong_inputs: - assert check_locator_format(wtype) is False + assert is_locator(wtype) is False for _ in range(100): - assert check_locator_format(get_random_value_hex(LOCATOR_LEN_BYTES)) is True + assert is_locator(get_random_value_hex(LOCATOR_LEN_BYTES)) is True def test_compute_locator(): - # The best way of checking that compute locator is correct is by using check_locator_format + # The best way of checking that compute locator is correct is by using is_locator for _ in range(100): - assert check_locator_format(compute_locator(get_random_value_hex(LOCATOR_LEN_BYTES))) is True + assert is_locator(compute_locator(get_random_value_hex(LOCATOR_LEN_BYTES))) is True # String of length smaller than LOCATOR_LEN_BYTES bytes must fail for i in range(1, LOCATOR_LEN_BYTES): - assert check_locator_format(compute_locator(get_random_value_hex(i))) is False + assert is_locator(compute_locator(get_random_value_hex(i))) is False def test_setup_data_folder(): diff --git a/test/teos/unit/test_tools.py b/test/teos/unit/test_tools.py index 45bceab..9a68a19 100644 --- a/test/teos/unit/test_tools.py +++ b/test/teos/unit/test_tools.py @@ -1,5 +1,4 @@ from teos.tools import can_connect_to_bitcoind, in_correct_network, bitcoin_cli -from common.tools import check_sha256_hex_format from test.teos.unit.conftest import bitcoind_connect_params @@ -27,32 +26,3 @@ def test_bitcoin_cli(): except Exception: assert False - - -def test_check_sha256_hex_format(): - assert check_sha256_hex_format(None) is False - assert check_sha256_hex_format("") is False - assert ( - check_sha256_hex_format(0x0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF) is False - ) # wrong type - assert ( - check_sha256_hex_format("abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd") is True - ) # lowercase - assert ( - check_sha256_hex_format("ABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCDEFABCD") is True - ) # uppercase - assert ( - check_sha256_hex_format("0123456789abcdef0123456789ABCDEF0123456789abcdef0123456789ABCDEF") is True - ) # mixed case - assert ( - check_sha256_hex_format("0123456789012345678901234567890123456789012345678901234567890123") is True - ) # only nums - assert ( - check_sha256_hex_format("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdf") is False - ) # too short - assert ( - check_sha256_hex_format("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0") is False - ) # too long - assert ( - check_sha256_hex_format("g123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") is False - ) # non-hex From 10e856ae745902df462653d21941c46f64aec37c Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 3 Apr 2020 16:38:46 +0200 Subject: [PATCH 60/62] Fixes typos according to the review --- CONTRIBUTING.md | 2 +- common/cryptographer.py | 2 +- teos/appointments_dbm.py | 6 +++--- teos/builder.py | 2 +- teos/gatekeeper.py | 6 +++--- teos/users_dbm.py | 4 ++-- test/teos/e2e/test_basic_e2e.py | 2 +- test/teos/unit/test_api.py | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2781ab7..c3c1280 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ In additon, we use [flake8](https://flake8.pycqa.org/en/latest/) to detect style flake8 --max-line-length=120 {source_file_or_directory} ``` - Not all outputs from flake8 are mandatory. For instance, splitting **bullet points in docstrings (E501)** will cause issues when generating the docuementation, so we will leave that longer than the line lenght limit . Another example are **whitespaces before colons in inline fors (E203)**. `black` places them in that way, so we'll leave them like that. + Not all outputs from flake8 are mandatory. For instance, splitting **bullet points in docstrings (E501)** will cause issues when generating the documentation, so we will leave that longer than the line length limit . Another example are **whitespaces before colons in inline fors (E203)**. `black` places them in that way, so we'll leave them like that. On top of that, there are a few rules to also have in mind. diff --git a/common/cryptographer.py b/common/cryptographer.py index e4fa7f8..c5e4af5 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -13,7 +13,7 @@ LN_MESSAGE_PREFIX = b"Lightning Signed Message:" def sha256d(message): """ - Compute the sha256 (double sha256) of a given by message. + Computes the double sha256 of a given by message. Args: message(:obj:`bytes`): the message to be used as input to the hash function. diff --git a/teos/appointments_dbm.py b/teos/appointments_dbm.py index ebd027a..cc1fcd0 100644 --- a/teos/appointments_dbm.py +++ b/teos/appointments_dbm.py @@ -33,7 +33,7 @@ class AppointmentsDBM(DBManager): Args: db_path (:obj:`str`): the path (relative or absolute) to the system folder containing the database. A fresh - database will be create if the specified path does not contain one. + database will be created if the specified path does not contain one. Raises: :obj:`ValueError`: If the provided ``db_path`` is not a string. @@ -148,7 +148,7 @@ class AppointmentsDBM(DBManager): by default. Returns: - :obj:`dict`: A dictionary with all the appointments stored in the database. An empty dictionary is there + :obj:`dict`: A dictionary with all the appointments stored in the database. An empty dictionary if there are none. """ @@ -453,7 +453,7 @@ class AppointmentsDBM(DBManager): Creates a flag that signals that an appointment has been triggered for every appointment in the given list Args: - uuids (:obj:`list`): a list of identifier for the appointments to flag. + uuids (:obj:`list`): a list of identifiers for the appointments to flag. """ with self.db.write_batch() as b: diff --git a/teos/builder.py b/teos/builder.py index 6d087fa..831236f 100644 --- a/teos/builder.py +++ b/teos/builder.py @@ -99,7 +99,7 @@ class Builder: def update_states(watcher, missed_blocks_watcher, missed_blocks_responder): """ Updates the states of both the :mod:`Watcher ` and the - :mod:`Responder `. If both have pending blocks to process they need to be updates at + :mod:`Responder `. If both have pending blocks to process they need to be updated at the same time, block by block. If only one instance has to be updated, ``populate_block_queue`` should be used. diff --git a/teos/gatekeeper.py b/teos/gatekeeper.py index a4a65fb..79b5efc 100644 --- a/teos/gatekeeper.py +++ b/teos/gatekeeper.py @@ -3,7 +3,7 @@ from common.cryptographer import Cryptographer class NotEnoughSlots(ValueError): - """Raise this when trying to subtract more slots than a user has available""" + """Raised when trying to subtract more slots than a user has available""" def __init__(self, user_pk, requested_slots): self.user_pk = user_pk @@ -12,7 +12,7 @@ class NotEnoughSlots(ValueError): class IdentificationFailure(Exception): """ - Raise this when a user can not be identified. Either the user public key cannot be recovered or the user is + Raised when a user can not be identified. Either the user public key cannot be recovered or the user is not found within the registered ones. """ @@ -45,7 +45,7 @@ class Gatekeeper: """ if not is_compressed_pk(user_pk): - raise ValueError("provided public key does not match expected format (33-byte hex string)") + raise ValueError("Provided public key does not match expected format (33-byte hex string)") if user_pk not in self.registered_users: self.registered_users[user_pk] = {"available_slots": self.default_slots} diff --git a/teos/users_dbm.py b/teos/users_dbm.py index b9f3b21..c71ee5a 100644 --- a/teos/users_dbm.py +++ b/teos/users_dbm.py @@ -17,7 +17,7 @@ class UsersDBM(DBManager): Args: db_path (:obj:`str`): the path (relative or absolute) to the system folder containing the database. A fresh - database will be create if the specified path does not contain one. + database will be created if the specified path does not contain one. Raises: :obj:`ValueError`: If the provided ``db_path`` is not a string. @@ -75,7 +75,7 @@ class UsersDBM(DBManager): user_pk (:obj:`str`): a 33-byte hex-encoded string identifying the user. Returns: - :obj:`dict`: A dictionary containing the appointment data if they ``key`` is found. + :obj:`dict`: A dictionary containing the appointment data if the ``key`` is found. Returns ``None`` otherwise. """ diff --git a/test/teos/e2e/test_basic_e2e.py b/test/teos/e2e/test_basic_e2e.py index 5e23cee..ab47c49 100644 --- a/test/teos/e2e/test_basic_e2e.py +++ b/test/teos/e2e/test_basic_e2e.py @@ -54,7 +54,7 @@ def add_appointment(appointment_data, sk=cli_sk): def test_commands_non_registered(bitcoin_cli, create_txs): - # All commands should failed if the user is not registered + # All commands should fail if the user is not registered # Add appointment commitment_tx, penalty_tx = create_txs diff --git a/test/teos/unit/test_api.py b/test/teos/unit/test_api.py index f19fbc2..210240f 100644 --- a/test/teos/unit/test_api.py +++ b/test/teos/unit/test_api.py @@ -388,7 +388,7 @@ def test_request_appointment_not_registered_user(client): tmp_sk, tmp_pk = generate_keypair() # The tower is designed so a not found appointment and a request from a non-registered user return the same error to - # prevent proving. + # prevent probing. test_request_random_appointment_registered_user(client, tmp_sk) From dcb221ea36bc9335a9ed58deeba76e8dbe633707 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 3 Apr 2020 20:57:01 +0200 Subject: [PATCH 61/62] Adds missing cli register test --- test/cli/unit/test_teos_cli.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/test/cli/unit/test_teos_cli.py b/test/cli/unit/test_teos_cli.py index 7ac0d32..52a32d6 100644 --- a/test/cli/unit/test_teos_cli.py +++ b/test/cli/unit/test_teos_cli.py @@ -2,6 +2,7 @@ import os import json import shutil import responses +from binascii import hexlify from coincurve import PrivateKey from requests.exceptions import ConnectionError @@ -28,6 +29,7 @@ another_sk = PrivateKey.from_int(3) teos_url = "http://{}:{}".format(config.get("TEOS_SERVER"), config.get("TEOS_PORT")) add_appointment_endpoint = "{}/add_appointment".format(teos_url) +register_endpoint = "{}/register".format(teos_url) get_appointment_endpoint = "{}/get_appointment".format(teos_url) dummy_appointment_data = { @@ -56,11 +58,22 @@ def get_signature(message, sk): return Cryptographer.sign(message, sk) -def test_register(): - pass - - # TODO: 90-add-more-add-appointment-tests +@responses.activate +def test_register(): + # Simulate a register response + compressed_pk_hex = hexlify(dummy_cli_compressed_pk).decode("utf-8") + response = {"public_key": compressed_pk_hex, "available_slots": 100} + responses.add(responses.POST, register_endpoint, json=response, status=200) + result = teos_cli.register(compressed_pk_hex, teos_url) + + assert len(responses.calls) == 1 + assert responses.calls[0].request.url == register_endpoint + assert result.get("public_key") == compressed_pk_hex and result.get("available_slots") == response.get( + "available_slots" + ) + + @responses.activate def test_add_appointment(): # Simulate a request to add_appointment for dummy_appointment, make sure that the right endpoint is requested From f35018201260291c60fef04e83fb35da34f73a16 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Fri, 3 Apr 2020 21:56:42 +0200 Subject: [PATCH 62/62] sign_recoverable can raise a ValueError, handling it. --- common/cryptographer.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/common/cryptographer.py b/common/cryptographer.py index c5e4af5..ecef7cd 100644 --- a/common/cryptographer.py +++ b/common/cryptographer.py @@ -281,9 +281,14 @@ class Cryptographer: logger.error("The value passed as sk is not a private key (EllipticCurvePrivateKey)") return None - rsig_rid = sk.sign_recoverable(LN_MESSAGE_PREFIX + message, hasher=sha256d) - sigrec = sigrec_encode(rsig_rid) - zb32_sig = pyzbase32.encode_bytes(sigrec).decode() + try: + rsig_rid = sk.sign_recoverable(LN_MESSAGE_PREFIX + message, hasher=sha256d) + sigrec = sigrec_encode(rsig_rid) + zb32_sig = pyzbase32.encode_bytes(sigrec).decode() + + except ValueError: + logger.error("Couldn't sign the message") + return None return zb32_sig