From 15f9a9efe11dee99549343468faaddce5d26dd90 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 9 Jan 2020 18:13:24 +0100 Subject: [PATCH 1/6] Removes triggered from to_json Triggered has been deatached from the appointment having it's own entry into the db, so not part of the appointment data anymore --- common/appointment.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/common/appointment.py b/common/appointment.py index 3b31365..7b21471 100644 --- a/common/appointment.py +++ b/common/appointment.py @@ -81,28 +81,18 @@ class Appointment: return appointment - def to_json(self, triggered=False): + def to_json(self): """ Exports an appointment as a deterministic json encoded string. This method ensures that multiple invocations with the same data yield the same value. This is the format used to store appointments in the database. - Args: - triggered (:mod:`bool`): Whether the dispute has been triggered or not. When an appointment passes from the - :mod:`Watcher ` to the :mod:`Responder ` it is not deleted straightaway. - Instead, the appointment is stored in the DB flagged as ``triggered``. This aims to ease handling block - reorgs in the future. - Returns: :obj:`str`: A json-encoded str representing the appointment. """ - appointment = self.to_dict() - - appointment["triggered"] = triggered - - return json.dumps(appointment, sort_keys=True, separators=(",", ":")) + return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) def serialize(self): """ From f6af67be87d7b0b50e6abdd6ca8d570db763e904 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 9 Jan 2020 18:14:19 +0100 Subject: [PATCH 2/6] Refactors db_manager to have a new entry type for the triggered appointmnets. Adds methods to create, load a and remove this new entries --- pisa/db_manager.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/pisa/db_manager.py b/pisa/db_manager.py index d9c566c..e337065 100644 --- a/pisa/db_manager.py +++ b/pisa/db_manager.py @@ -10,6 +10,7 @@ WATCHER_LAST_BLOCK_KEY = "bw" RESPONDER_PREFIX = "r" RESPONDER_LAST_BLOCK_KEY = "br" LOCATOR_MAP_PREFIX = "m" +TRIGGERED_APPOINTMENTS_PREFIX = "ta" class DBManager: @@ -17,13 +18,14 @@ class DBManager: The :class:`DBManager` is the class in charge of interacting with the appointments database (``LevelDB``). Keys and values are stored as bytes in the database but processed as strings by the manager. - The database is split in five prefixes: + The database is split in six prefixes: - ``WATCHER_PREFIX``, defined as ``b'w``, is used to store :obj:`Watcher ` appointments. - ``RESPONDER_PREFIX``, defines as ``b'r``, is used to store :obj:`Responder ` trackers. - ``WATCHER_LAST_BLOCK_KEY``, defined as ``b'bw``, is used to store the last block hash known by the :obj:`Watcher `. - ``RESPONDER_LAST_BLOCK_KEY``, defined as ``b'br``, is used to store the last block hash known by the :obj:`Responder `. - ``LOCATOR_MAP_PREFIX``, defined as ``b'm``, is used to store the ``locator:uuid`` maps. + - ``TRIGGERED_APPOINTMENTS_PREFIX``, defined as ``b'ta``, is used to stored triggered appointments (appointments that have been handed to the :obj:`Responder `.) Args: db_path (:obj:`str`): the path (relative or absolute) to the system folder containing the database. A fresh @@ -160,10 +162,9 @@ class DBManager: def load_watcher_appointments(self, include_triggered=False): """ Loads all the appointments from the database (all entries with the ``WATCHER_PREFIX`` prefix). - Args: - include_triggered (:obj:`bool`): Whether to include the appointments flagged as triggered or not. ``False`` by - default. + include_triggered (:obj:`bool`): Whether to include the appointments flagged as triggered or not. ``False`` + by default. Returns: :obj:`dict`: A dictionary with all the appointments stored in the database. An empty dictionary is there @@ -171,10 +172,11 @@ class DBManager: """ appointments = self.load_appointments_db(prefix=WATCHER_PREFIX) + triggered_appointments = self.load_all_triggered_flags() if not include_triggered: appointments = { - uuid: appointment for uuid, appointment in appointments.items() if appointment["triggered"] is False + uuid: appointment for uuid, appointment in appointments.items() if uuid not in triggered_appointments } return appointments @@ -332,3 +334,30 @@ class DBManager: """ self.create_entry(RESPONDER_LAST_BLOCK_KEY, block_hash) + + def create_triggered_appointment_flag(self, uuid): + """ + Creates a flag that signals that an appointment has been triggered. + """ + + self.db.put((TRIGGERED_APPOINTMENTS_PREFIX + uuid).encode("utf-8"), "".encode("utf-8")) + + def load_all_triggered_flags(self): + """ + Loads all the appointment triggered flags from the database. + + Returns: + :obj:`list`: a list of all the uuids of the triggered appointments. + """ + + return [ + k.decode()[len(TRIGGERED_APPOINTMENTS_PREFIX) :] + for k, v in self.db.iterator(prefix=TRIGGERED_APPOINTMENTS_PREFIX.encode("utf-8")) + ] + + def delete_triggered_appointment_flag(self, uuid): + """ + Deletes a flag that signals that an appointment has been triggered. + """ + + self.delete_entry(uuid, prefix=TRIGGERED_APPOINTMENTS_PREFIX) From d9cccfb26acaf73583d1f2a0540df6ada0f1551b Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 9 Jan 2020 18:15:43 +0100 Subject: [PATCH 3/6] Refactors the API to used the new triggered functions --- pisa/api.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pisa/api.py b/pisa/api.py index 9468166..519b3cb 100644 --- a/pisa/api.py +++ b/pisa/api.py @@ -109,17 +109,16 @@ class API: return jsonify(response) locator_map = self.watcher.db_manager.load_locator_map(locator) + triggered_appointments = self.watcher.db_manager.load_all_triggered_flags() if locator_map is not None: for uuid in locator_map: - appointment_data = self.watcher.db_manager.load_watcher_appointment(uuid) + if uuid not in triggered_appointments: + appointment_data = self.watcher.db_manager.load_watcher_appointment(uuid) - if appointment_data is not None and appointment_data["triggered"] is False: - # Triggered is an internal flag - del appointment_data["triggered"] - - appointment_data["status"] = "being_watched" - response.append(appointment_data) + if appointment_data is not None: + appointment_data["status"] = "being_watched" + response.append(appointment_data) tracker_data = self.watcher.db_manager.load_responder_tracker(uuid) From 34636ab8e0fc6332bbdff894bb736c5f8da172a7 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 9 Jan 2020 18:18:12 +0100 Subject: [PATCH 4/6] Unit tests for the db_manager triggered_flag functions --- test/pisa/unit/test_db_manager.py | 47 +++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/test/pisa/unit/test_db_manager.py b/test/pisa/unit/test_db_manager.py index 49a5e43..10483a1 100644 --- a/test/pisa/unit/test_db_manager.py +++ b/test/pisa/unit/test_db_manager.py @@ -5,7 +5,12 @@ import shutil from uuid import uuid4 from pisa.db_manager import DBManager -from pisa.db_manager import WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY, LOCATOR_MAP_PREFIX +from pisa.db_manager import ( + WATCHER_LAST_BLOCK_KEY, + RESPONDER_LAST_BLOCK_KEY, + LOCATOR_MAP_PREFIX, + TRIGGERED_APPOINTMENTS_PREFIX, +) from common.constants import LOCATOR_LEN_BYTES @@ -221,7 +226,8 @@ def test_store_load_triggered_appointment(db_manager): # Create an appointment flagged as triggered triggered_appointment, _ = generate_dummy_appointment(real_height=False) uuid = uuid4().hex - db_manager.store_watcher_appointment(uuid, triggered_appointment.to_json(triggered=True)) + db_manager.store_watcher_appointment(uuid, triggered_appointment.to_json()) + db_manager.create_triggered_appointment_flag(uuid) # The new appointment is grabbed only if we set include_triggered assert db_watcher_appointments == db_manager.load_watcher_appointments() @@ -282,3 +288,40 @@ def test_store_load_last_block_hash_responder(db_manager): db_last_block_hash = db_manager.load_last_block_hash_responder() assert local_last_block_hash == db_last_block_hash + + +def test_create_triggered_appointment_flag(db_manager): + # Test that flags are added + key = get_random_value_hex(16) + db_manager.create_triggered_appointment_flag(key) + + assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is not None + + # Test to get a random one that we haven't added + key = get_random_value_hex(16) + assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is None + + +def test_load_all_triggered_flags(db_manager): + # There should be a some flags in the db from the previous tests. Let's load them + flags = db_manager.load_all_triggered_flags() + + # We can add another flag and see that there's two now + new_uuid = uuid4().hex + db_manager.create_triggered_appointment_flag(new_uuid) + flags.append(new_uuid) + + assert set(db_manager.load_all_triggered_flags()) == set(flags) + + +def test_delete_triggered_appointment_flag(db_manager): + # Test data is properly deleted. + keys = db_manager.load_all_triggered_flags() + + # Delete all entries + for k in keys: + db_manager.delete_triggered_appointment_flag(k) + + # Try to load them back + for k in keys: + assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + k).encode("utf-8")) is None From 5196f5df2974455733da0f715bd0d7402e3fd434 Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 9 Jan 2020 18:18:26 +0100 Subject: [PATCH 5/6] Removes most appointment data from memory Data used to be stored both in memory and disk (db). This commits modifies the Watcher, Responder and Cleaner so they only keep the needed maps and load information from disk when necessary. --- pisa/cleaner.py | 26 +++++++++++++------------- pisa/responder.py | 35 +++++++++++++++++++++++------------ pisa/watcher.py | 23 +++++++++++++---------- 3 files changed, 49 insertions(+), 35 deletions(-) diff --git a/pisa/cleaner.py b/pisa/cleaner.py index a00cc22..6b8d73e 100644 --- a/pisa/cleaner.py +++ b/pisa/cleaner.py @@ -1,4 +1,5 @@ from common.logger import Logger +from common.appointment import Appointment logger = Logger("Cleaner") @@ -27,7 +28,7 @@ class Cleaner: """ for uuid in expired_appointments: - locator = appointments[uuid].locator + locator = appointments[uuid].get("locator") appointments.pop(uuid) @@ -58,20 +59,21 @@ class Cleaner: database. """ + locator = appointments[uuid].get("locator") + # Delete the appointment - appointment = appointments.pop(uuid) + appointments.pop(uuid) # If there was only one appointment that matches the locator we can delete the whole list - if len(locator_uuid_map[appointment.locator]) == 1: - locator_uuid_map.pop(appointment.locator) + if len(locator_uuid_map[locator]) == 1: + locator_uuid_map.pop(locator) else: # Otherwise we just delete the appointment that matches locator:appointment_pos - locator_uuid_map[appointment.locator].remove(uuid) + locator_uuid_map[locator].remove(uuid) # DISCUSS: instead of deleting the appointment, we will mark it as triggered and delete it from both # the watcher's and responder's db after fulfilled - # Update appointment in the db - db_manager.store_watcher_appointment(uuid, appointment.to_json(triggered=True)) + db_manager.create_triggered_appointment_flag(uuid) @staticmethod def delete_completed_trackers(completed_trackers, height, trackers, tx_tracker_map, db_manager): @@ -98,8 +100,8 @@ class Cleaner: confirmations=confirmations, ) - penalty_txid = trackers[uuid].penalty_txid - locator = trackers[uuid].locator + penalty_txid = trackers[uuid].get("penalty_txid") + locator = trackers[uuid].get("locator") trackers.pop(uuid) if len(tx_tracker_map[penalty_txid]) == 1: @@ -110,9 +112,10 @@ class Cleaner: else: tx_tracker_map[penalty_txid].remove(uuid) - # Delete appointment from the db (both watchers's and responder's) + # Delete appointment from the db (from watchers's and responder's db) and remove flag db_manager.delete_watcher_appointment(uuid) db_manager.delete_responder_tracker(uuid) + db_manager.delete_triggered_appointment_flag(uuid) # Update / delete the locator map locator_map = db_manager.load_locator_map(locator) @@ -120,13 +123,10 @@ class Cleaner: if uuid in locator_map: if len(locator_map) == 1: db_manager.delete_locator_map(locator) - else: locator_map.remove(uuid) db_manager.store_update_locator_map(locator, locator_map) - else: logger.error("UUID not found in the db", uuid=uuid) - else: logger.error("Locator not found in the db", uuid=uuid) diff --git a/pisa/responder.py b/pisa/responder.py index 0fce1cb..aa5f672 100644 --- a/pisa/responder.py +++ b/pisa/responder.py @@ -118,8 +118,9 @@ class Responder: database. Attributes: - trackers (:obj:`dict`): A dictionary containing all the :obj:`TransactionTracker` handled by the - :obj:`Responder`. Each entry is identified by a ``uuid``. + trackers (:obj:`dict`): A dictionary containing the minimum information about the :obj:`TransactionTracker` + required by the :obj:`Responder` (``penalty_txid``, ``locator`` and ``end_time``). + Each entry is identified by a ``uuid``. tx_tracker_map (:obj:`dict`): A ``penalty_txid:uuid`` map used to allow the :obj:`Responder` to deal with several trackers triggered by the same ``penalty_txid``. unconfirmed_txs (:obj:`list`): A list that keeps track of all unconfirmed ``penalty_txs``. @@ -221,8 +222,9 @@ class Responder: """ Creates a :obj:`TransactionTracker` after successfully broadcasting a ``penalty_tx``. - The :obj:`TransactionTracker` is stored in ``trackers`` and ``tx_tracker_map`` and the ``penalty_txid`` added to - ``unconfirmed_txs`` if ``confirmations=0``. Finally, the data is also stored in the database. + A reduction of :obj:`TransactionTracker` is stored in ``trackers`` and ``tx_tracker_map`` and the + ``penalty_txid`` added to ``unconfirmed_txs`` if ``confirmations=0``. Finally, all the data is stored in the + database. ``add_tracker`` awakes the :obj:`Responder` and creates a connection with the :obj:`ZMQSubscriber ` if he is asleep. @@ -240,7 +242,13 @@ class Responder: """ tracker = TransactionTracker(locator, dispute_txid, penalty_txid, penalty_rawtx, appointment_end) - self.trackers[uuid] = tracker + + # We only store the penalty_txid, locator and appointment_end in memory. The rest is dumped into the db. + self.trackers[uuid] = { + "penalty_txid": tracker.penalty_txid, + "locator": locator, + "appointment_end": appointment_end, + } if penalty_txid in self.tx_tracker_map: self.tx_tracker_map[penalty_txid].append(uuid) @@ -392,15 +400,16 @@ class Responder: completed_trackers = [] - for uuid, tracker in self.trackers.items(): - if tracker.appointment_end <= height and tracker.penalty_txid not in self.unconfirmed_txs: - tx = Carrier.get_transaction(tracker.penalty_txid) + for uuid, tracker_data in self.trackers.items(): + appointment_end = tracker_data.get("appointment_end") + penalty_txid = tracker_data.get("penalty_txid") + if appointment_end <= height and penalty_txid not in self.unconfirmed_txs: + tx = Carrier.get_transaction(penalty_txid) - # FIXME: Should be improved with the librarian if tx is not None: confirmations = tx.get("confirmations") - if confirmations >= MIN_CONFIRMATIONS: + if confirmations is not None and confirmations >= MIN_CONFIRMATIONS: # The end of the appointment has been reached completed_trackers.append((uuid, confirmations)) @@ -434,7 +443,7 @@ class Responder: # FIXME: This would potentially grab multiple instances of the same transaction and try to send them. # should we do it only once? for uuid in self.tx_tracker_map[txid]: - tracker = self.trackers[uuid] + tracker = TransactionTracker.from_dict(self.db_manager.load_responder_tracker(uuid)) logger.warning( "Transaction has missed many confirmations. Rebroadcasting", penalty_txid=tracker.penalty_txid ) @@ -460,7 +469,9 @@ class Responder: """ carrier = Carrier() - for uuid, tracker in self.trackers.items(): + for uuid in self.trackers.keys(): + tracker = TransactionTracker.from_dict(self.db_manager.load_responder_tracker(uuid)) + # First we check if the dispute transaction is known (exists either in mempool or blockchain) dispute_tx = carrier.get_transaction(tracker.dispute_txid) diff --git a/pisa/watcher.py b/pisa/watcher.py index 9d659db..62ba514 100644 --- a/pisa/watcher.py +++ b/pisa/watcher.py @@ -4,8 +4,9 @@ from threading import Thread from common.cryptographer import Cryptographer from common.constants import LOCATOR_LEN_HEX - +from common.appointment import Appointment from common.logger import Logger + from pisa.cleaner import Cleaner from pisa.responder import Responder from pisa.block_processor import BlockProcessor @@ -40,8 +41,9 @@ class Watcher: Attributes: - appointments (:obj:`dict`): a dictionary containing all the appointments (:obj:`Appointment - ` instances) accepted by the tower. It's populated trough ``add_appointment``. + appointments (:obj:`dict`): a dictionary containing a simplification of the appointments (:obj:`Appointment + ` instances) accepted by the tower (``locator`` and ``end_time``). + It's populated trough ``add_appointment``. locator_uuid_map (:obj:`dict`): a ``locator:uuid`` map used to allow the :obj:`Watcher` to deal with several appointments with the same ``locator``. asleep (:obj:`bool`): A flag that signals whether the :obj:`Watcher` is asleep or awake. @@ -116,8 +118,9 @@ class Watcher: """ if len(self.appointments) < self.max_appointments: + # Appointments are stored in disk, we only keep the end_time, locator and locator_uuid map in memory uuid = uuid4().hex - self.appointments[uuid] = appointment + self.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} if appointment.locator in self.locator_uuid_map: self.locator_uuid_map[appointment.locator].append(uuid) @@ -138,11 +141,10 @@ class Watcher: self.db_manager.store_update_locator_map(appointment.locator, uuid) appointment_added = True + signature = Cryptographer.sign(appointment.serialize(), self.signing_key) logger.info("New appointment accepted", locator=appointment.locator) - signature = Cryptographer.sign(appointment.serialize(), self.signing_key) - else: appointment_added = False signature = None @@ -181,8 +183,8 @@ class Watcher: expired_appointments = [ uuid - for uuid, appointment in self.appointments.items() - if block["height"] > appointment.end_time + EXPIRY_DELTA + for uuid, appointment_data in self.appointments.items() + if block["height"] > appointment_data.get("end_time") + EXPIRY_DELTA ] Cleaner.delete_expired_appointment( @@ -207,7 +209,7 @@ class Watcher: filtered_breach["dispute_txid"], filtered_breach["penalty_txid"], filtered_breach["penalty_rawtx"], - self.appointments[uuid].end_time, + self.appointments[uuid].get("end_time"), block_hash, ) @@ -274,9 +276,10 @@ class Watcher: for locator, dispute_txid in breaches.items(): for uuid in self.locator_uuid_map[locator]: + appointment = Appointment.from_dict(self.db_manager.load_watcher_appointment(uuid)) try: - penalty_rawtx = Cryptographer.decrypt(self.appointments[uuid].encrypted_blob, dispute_txid) + penalty_rawtx = Cryptographer.decrypt(appointment.encrypted_blob, dispute_txid) except ValueError: penalty_rawtx = None From db4644434817a58d4ce9ab1eb976f921cde0c5cb Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Thu, 9 Jan 2020 18:20:00 +0100 Subject: [PATCH 6/6] Adapts the tests to work with data in db instead of memory --- test/pisa/unit/test_cleaner.py | 19 +++++----- test/pisa/unit/test_responder.py | 59 ++++++++++++++++++++++---------- test/pisa/unit/test_watcher.py | 21 ++++++++++-- 3 files changed, 67 insertions(+), 32 deletions(-) diff --git a/test/pisa/unit/test_cleaner.py b/test/pisa/unit/test_cleaner.py index b7467f3..5b9eaf2 100644 --- a/test/pisa/unit/test_cleaner.py +++ b/test/pisa/unit/test_cleaner.py @@ -4,7 +4,7 @@ from uuid import uuid4 from pisa.responder import TransactionTracker from pisa.cleaner import Cleaner from common.appointment import Appointment -from pisa.db_manager import WATCHER_PREFIX +from pisa.db_manager import WATCHER_PREFIX, TRIGGERED_APPOINTMENTS_PREFIX from test.pisa.unit.conftest import get_random_value_hex @@ -26,7 +26,7 @@ def set_up_appointments(db_manager, total_appointments): locator = get_random_value_hex(LOCATOR_LEN_BYTES) appointment = Appointment(locator, None, None, None, None) - appointments[uuid] = appointment + appointments[uuid] = {"locator": appointment.locator} locator_uuid_map[locator] = [uuid] db_manager.store_watcher_appointment(uuid, appointment.to_json()) @@ -36,7 +36,7 @@ def set_up_appointments(db_manager, total_appointments): if i % 2: uuid = uuid4().hex - appointments[uuid] = appointment + appointments[uuid] = {"locator": appointment.locator} locator_uuid_map[locator].append(uuid) db_manager.store_watcher_appointment(uuid, appointment.to_json()) @@ -59,7 +59,7 @@ def set_up_trackers(db_manager, total_trackers): # Assign both penalty_txid and dispute_txid the same id (it shouldn't matter) tracker = TransactionTracker(locator, dispute_txid, penalty_txid, None, None) - trackers[uuid] = tracker + trackers[uuid] = {"locator": tracker.locator, "penalty_txid": tracker.penalty_txid} tx_tracker_map[penalty_txid] = [uuid] db_manager.store_responder_tracker(uuid, tracker.to_json()) @@ -69,7 +69,7 @@ def set_up_trackers(db_manager, total_trackers): if i % 2: uuid = uuid4().hex - trackers[uuid] = tracker + trackers[uuid] = {"locator": tracker.locator, "penalty_txid": tracker.penalty_txid} tx_tracker_map[penalty_txid].append(uuid) db_manager.store_responder_tracker(uuid, tracker.to_json()) @@ -99,9 +99,8 @@ def test_delete_completed_appointments(db_manager): assert len(appointments) == 0 # Make sure that all appointments are flagged as triggered in the db - db_appointments = db_manager.load_appointments_db(prefix=WATCHER_PREFIX) for uuid in uuids: - assert db_appointments[uuid]["triggered"] is True + assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + uuid).encode("utf-8")) is not None def test_delete_completed_trackers_db_match(db_manager): @@ -128,12 +127,12 @@ def test_delete_completed_trackers_no_db_match(db_manager): # Let's change some uuid's by creating new trackers that are not included in the db and share a penalty_txid # with another tracker that is stored in the db. for uuid in selected_trackers[: ITEMS // 2]: - penalty_txid = trackers[uuid].penalty_txid + penalty_txid = trackers[uuid].get("penalty_txid") dispute_txid = get_random_value_hex(32) locator = dispute_txid[:LOCATOR_LEN_HEX] new_uuid = uuid4().hex - trackers[new_uuid] = TransactionTracker(locator, dispute_txid, penalty_txid, None, None) + trackers[new_uuid] = {"locator": locator, "penalty_txid": penalty_txid} tx_tracker_map[penalty_txid].append(new_uuid) selected_trackers.append(new_uuid) @@ -144,7 +143,7 @@ def test_delete_completed_trackers_no_db_match(db_manager): dispute_txid = get_random_value_hex(32) locator = dispute_txid[:LOCATOR_LEN_HEX] - trackers[uuid] = TransactionTracker(locator, dispute_txid, penalty_txid, None, None) + trackers[uuid] = {"locator": locator, "penalty_txid": penalty_txid} tx_tracker_map[penalty_txid] = [uuid] selected_trackers.append(uuid) diff --git a/test/pisa/unit/test_responder.py b/test/pisa/unit/test_responder.py index a9e99da..cfb0be0 100644 --- a/test/pisa/unit/test_responder.py +++ b/test/pisa/unit/test_responder.py @@ -228,11 +228,9 @@ def test_add_tracker(responder): # Check that the rest of tracker data also matches tracker = responder.trackers[uuid] assert ( - tracker.dispute_txid == dispute_txid - and tracker.penalty_txid == penalty_txid - and tracker.penalty_rawtx == penalty_rawtx - and tracker.appointment_end == appointment_end - and tracker.appointment_end == appointment_end + tracker.get("penalty_txid") == penalty_txid + and tracker.get("locator") == locator + and tracker.get("appointment_end") == appointment_end ) @@ -255,11 +253,9 @@ def test_add_tracker_same_penalty_txid(responder): for uuid in [uuid_1, uuid_2]: tracker = responder.trackers[uuid] assert ( - tracker.dispute_txid == dispute_txid - and tracker.penalty_txid == penalty_txid - and tracker.penalty_rawtx == penalty_rawtx - and tracker.appointment_end == appointment_end - and tracker.appointment_end == appointment_end + tracker.get("penalty_txid") == penalty_txid + and tracker.get("locator") == locator + and tracker.get("appointment_end") == appointment_end ) @@ -308,11 +304,19 @@ def test_do_watch(temp_db_manager): for tracker in trackers: uuid = uuid4().hex - responder.trackers[uuid] = tracker + responder.trackers[uuid] = { + "locator": tracker.locator, + "penalty_txid": tracker.penalty_txid, + "appointment_end": tracker.appointment_end, + } responder.tx_tracker_map[tracker.penalty_txid] = [uuid] responder.missed_confirmations[tracker.penalty_txid] = 0 responder.unconfirmed_txs.append(tracker.penalty_txid) + # We also need to store the info in the db + responder.db_manager.create_triggered_appointment_flag(uuid) + responder.db_manager.store_responder_tracker(uuid, tracker.to_json()) + # Let's start to watch watch_thread = Thread(target=responder.do_watch) watch_thread.daemon = True @@ -434,12 +438,20 @@ def test_get_completed_trackers(db_manager): tracker.appointment_end += 10 trackers_no_end[uuid4().hex] = tracker - # Let's add all to the responder - responder.trackers.update(trackers_end_conf) - responder.trackers.update(trackers_end_no_conf) - responder.trackers.update(trackers_no_end) + all_trackers = {} + all_trackers.update(trackers_end_conf) + all_trackers.update(trackers_end_no_conf) + all_trackers.update(trackers_no_end) - for uuid, tracker in responder.trackers.items(): + # Let's add all to the responder + for uuid, tracker in all_trackers.items(): + responder.trackers[uuid] = { + "locator": tracker.locator, + "penalty_txid": tracker.penalty_txid, + "appointment_end": tracker.appointment_end, + } + + for uuid, tracker in all_trackers.items(): bitcoin_cli().sendrawtransaction(tracker.penalty_rawtx) # The dummy appointments have a end_appointment time of current + 2, but trackers need at least 6 confs by default @@ -474,9 +486,18 @@ def test_rebroadcast(db_manager): penalty_rawtx=TX.create_dummy_transaction() ) - responder.trackers[uuid] = TransactionTracker( - locator, dispute_txid, penalty_txid, penalty_rawtx, appointment_end - ) + tracker = TransactionTracker(locator, dispute_txid, penalty_txid, penalty_rawtx, appointment_end) + + responder.trackers[uuid] = { + "locator": locator, + "penalty_txid": penalty_txid, + "appointment_end": appointment_end, + } + + # We need to add it to the db too + responder.db_manager.create_triggered_appointment_flag(uuid) + responder.db_manager.store_responder_tracker(uuid, tracker.to_json()) + responder.tx_tracker_map[penalty_txid] = [uuid] responder.unconfirmed_txs.append(penalty_txid) diff --git a/test/pisa/unit/test_watcher.py b/test/pisa/unit/test_watcher.py index 8ec331c..a05b6fd 100644 --- a/test/pisa/unit/test_watcher.py +++ b/test/pisa/unit/test_watcher.py @@ -139,7 +139,16 @@ def test_do_subscribe(watcher): def test_do_watch(watcher): # We will wipe all the previous data and add 5 appointments - watcher.appointments, watcher.locator_uuid_map, dispute_txs = create_appointments(APPOINTMENTS) + appointments, locator_uuid_map, dispute_txs = create_appointments(APPOINTMENTS) + + # Set the data into the Watcher and in the db + watcher.locator_uuid_map = locator_uuid_map + watcher.appointments = {} + + for uuid, appointment in appointments.items(): + watcher.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} + watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json()) + watcher.db_manager.store_update_locator_map(appointment.locator, uuid) watch_thread = Thread(target=watcher.do_watch) watch_thread.daemon = True @@ -190,7 +199,9 @@ def test_filter_valid_breaches_random_data(watcher): for i in range(TEST_SET_SIZE): dummy_appointment, _ = generate_dummy_appointment() uuid = uuid4().hex - appointments[uuid] = dummy_appointment + appointments[uuid] = {"locator": dummy_appointment.locator, "end_time": dummy_appointment.end_time} + watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_json()) + watcher.db_manager.store_update_locator_map(dummy_appointment.locator, uuid) locator_uuid_map[dummy_appointment.locator] = [uuid] @@ -226,7 +237,11 @@ def test_filter_valid_breaches(watcher): locator_uuid_map = {dummy_appointment.locator: [uuid]} breaches = {dummy_appointment.locator: dispute_txid} - watcher.appointments = appointments + for uuid, appointment in appointments.items(): + watcher.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time} + watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_json()) + watcher.db_manager.store_update_locator_map(dummy_appointment.locator, uuid) + watcher.locator_uuid_map = locator_uuid_map filtered_valid_breaches = watcher.filter_valid_breaches(breaches)