Addapts tests to match 7031b552f7. Improves DBManager tests.

test/common/unit/test_appointment.py

@@ -71,26 +71,6 @@ def test_to_dict(appointment_data):
     )
 
 
-def test_to_json(appointment_data):
-    appointment = Appointment(
-        appointment_data["locator"],
-        appointment_data["start_time"],
-        appointment_data["end_time"],
-        appointment_data["to_self_delay"],
-        appointment_data["encrypted_blob"],
-    )
-
-    dict_appointment = json.loads(appointment.to_json())
-
-    assert (
-        appointment_data["locator"] == dict_appointment["locator"]
-        and appointment_data["start_time"] == dict_appointment["start_time"]
-        and appointment_data["end_time"] == dict_appointment["end_time"]
-        and appointment_data["to_self_delay"] == dict_appointment["to_self_delay"]
-        and EncryptedBlob(appointment_data["encrypted_blob"]) == EncryptedBlob(dict_appointment["encrypted_blob"])
-    )
-
-
 def test_from_dict(appointment_data):
     # The appointment should be build if we don't miss any field
     appointment = Appointment.from_dict(appointment_data)

test/teos/unit/conftest.py

@@ -12,10 +12,10 @@ from bitcoind_mock.transaction import create_dummy_transaction
 
 from teos.carrier import Carrier
 from teos.tools import bitcoin_cli
-from teos.db_manager import DBManager
 from teos import LOG_PREFIX, DEFAULT_CONF
 from teos.responder import TransactionTracker
 from teos.block_processor import BlockProcessor
+from teos.appointments_dbm import AppointmentsDBM
 
 import common.cryptographer
 from common.blob import Blob
@@ -53,7 +53,7 @@ def prng_seed():
 
 @pytest.fixture(scope="module")
 def db_manager():
-    manager = DBManager("test_db")
+    manager = AppointmentsDBM("test_db")
     # Add last know block for the Responder in the db
 
     yield manager

test/teos/unit/test_api.py

@@ -7,8 +7,8 @@ from teos import HOST, PORT
 import teos.errors as errors
 from teos.watcher import Watcher
 from teos.inspector import Inspector
-from teos.db_manager import DBManager
 from teos.gatekeeper import Gatekeeper
+from teos.appointments_dbm import AppointmentsDBM
 from teos.responder import Responder, TransactionTracker
 
 from test.teos.unit.conftest import get_random_value_hex, generate_dummy_appointment, generate_keypair, get_config
@@ -48,7 +48,7 @@ compressed_client_pk = hexlify(client_pk.format(compressed=True)).decode("utf-8"
 
 @pytest.fixture()
 def get_all_db_manager():
-    manager = DBManager("get_all_tmp_db")
+    manager = AppointmentsDBM("get_all_tmp_db")
     # Add last know block for the Responder in the db
 
     yield manager
@@ -396,7 +396,7 @@ def test_request_appointment_not_registered_user(client):
 def test_request_appointment_in_watcher(api, client, appointment):
     # Mock the appointment in the Watcher
     uuid = hash_160("{}{}".format(appointment.locator, compressed_client_pk))
-    api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json())
+    api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_dict())
 
     # Next we can request it
     message = "get appointment {}".format(appointment.locator)
@@ -426,7 +426,7 @@ def test_request_appointment_in_responder(api, client, appointment):
 
     uuid = hash_160("{}{}".format(appointment.locator, compressed_client_pk))
     api.watcher.db_manager.create_triggered_appointment_flag(uuid)
-    api.watcher.responder.db_manager.store_responder_tracker(uuid, tx_tracker.to_json())
+    api.watcher.responder.db_manager.store_responder_tracker(uuid, tx_tracker.to_dict())
 
     # Request back the data
     message = "get appointment {}".format(appointment.locator)
@@ -464,14 +464,14 @@ def test_get_all_appointments_watcher(api, client, get_all_db_manager, appointme
         uuid = get_random_value_hex(16)
         appointment.locator = get_random_value_hex(16)
         non_triggered_appointments[uuid] = appointment.to_dict()
-        api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json())
+        api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_dict())
 
     triggered_appointments = {}
     for _ in range(10):
         uuid = get_random_value_hex(16)
         appointment.locator = get_random_value_hex(16)
         triggered_appointments[uuid] = appointment.to_dict()
-        api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json())
+        api.watcher.db_manager.store_watcher_appointment(uuid, appointment.to_dict())
         api.watcher.db_manager.create_triggered_appointment_flag(uuid)
 
     # We should only get check the non-triggered appointments
@@ -508,7 +508,7 @@ def test_get_all_appointments_responder(api, client, get_all_db_manager):
         }
         tracker = TransactionTracker.from_dict(tracker_data)
         tx_trackers[uuid] = tracker.to_dict()
-        api.watcher.responder.db_manager.store_responder_tracker(uuid, tracker.to_json())
+        api.watcher.responder.db_manager.store_responder_tracker(uuid, tracker.to_dict())
         api.watcher.db_manager.create_triggered_appointment_flag(uuid)
 
     # Get all appointments

test/teos/unit/test_appointments_dbm.py

@@ -0,0 +1,380 @@
+import os
+import json
+import pytest
+import shutil
+from uuid import uuid4
+
+from teos.appointments_dbm import AppointmentsDBM
+from teos.appointments_dbm import (
+    WATCHER_LAST_BLOCK_KEY,
+    RESPONDER_LAST_BLOCK_KEY,
+    LOCATOR_MAP_PREFIX,
+    TRIGGERED_APPOINTMENTS_PREFIX,
+)
+
+from common.constants import LOCATOR_LEN_BYTES
+
+from test.teos.unit.conftest import get_random_value_hex, generate_dummy_appointment
+
+
+@pytest.fixture(scope="module")
+def watcher_appointments():
+    return {uuid4().hex: generate_dummy_appointment(real_height=False)[0] for _ in range(10)}
+
+
+@pytest.fixture(scope="module")
+def responder_trackers():
+    return {get_random_value_hex(16): get_random_value_hex(32) for _ in range(10)}
+
+
+def open_create_db(db_path):
+
+    try:
+        db_manager = AppointmentsDBM(db_path)
+
+        return db_manager
+
+    except ValueError:
+        return False
+
+
+def test_load_appointments_db(db_manager):
+    # Let's made up a prefix and try to load data from the database using it
+    prefix = "XX"
+    db_appointments = db_manager.load_appointments_db(prefix)
+
+    assert len(db_appointments) == 0
+
+    # We can add a bunch of data to the db and try again (data is stored in json by the manager)
+    local_appointments = {}
+    for _ in range(10):
+        key = get_random_value_hex(16)
+        value = get_random_value_hex(32)
+        local_appointments[key] = value
+
+        db_manager.db.put((prefix + key).encode("utf-8"), json.dumps({"value": value}).encode("utf-8"))
+
+    db_appointments = db_manager.load_appointments_db(prefix)
+
+    # Check that both keys and values are the same
+    assert db_appointments.keys() == local_appointments.keys()
+
+    values = [appointment["value"] for appointment in db_appointments.values()]
+    assert set(values) == set(local_appointments.values()) and (len(values) == len(local_appointments))
+
+
+def test_get_last_known_block():
+    db_path = "empty_db"
+
+    # First we check if the db exists, and if so we delete it
+    if os.path.isdir(db_path):
+        shutil.rmtree(db_path)
+
+    # Check that the db can be created if it does not exist
+    db_manager = open_create_db(db_path)
+
+    # Trying to get any last block for either the watcher or the responder should return None for an empty db
+
+    for key in [WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY]:
+        assert db_manager.get_last_known_block(key) is None
+
+    # After saving some block in the db we should get that exact value
+    for key in [WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY]:
+        block_hash = get_random_value_hex(32)
+        db_manager.db.put(key.encode("utf-8"), block_hash.encode("utf-8"))
+        assert db_manager.get_last_known_block(key) == block_hash
+
+    # Removing test db
+    shutil.rmtree(db_path)
+
+
+def test_load_watcher_appointments_empty(db_manager):
+    assert len(db_manager.load_watcher_appointments()) == 0
+
+
+def test_load_responder_trackers_empty(db_manager):
+    assert len(db_manager.load_responder_trackers()) == 0
+
+
+def test_load_locator_map_empty(db_manager):
+    assert db_manager.load_locator_map(get_random_value_hex(LOCATOR_LEN_BYTES)) is None
+
+
+def test_create_append_locator_map(db_manager):
+    uuid = uuid4().hex
+    locator = get_random_value_hex(LOCATOR_LEN_BYTES)
+    db_manager.create_append_locator_map(locator, uuid)
+
+    # Check that the locator map has been properly stored
+    assert db_manager.load_locator_map(locator) == [uuid]
+
+    # If we try to add the same uuid again the list shouldn't change
+    db_manager.create_append_locator_map(locator, uuid)
+    assert db_manager.load_locator_map(locator) == [uuid]
+
+    # Add another uuid to the same locator and check that it also works
+    uuid2 = uuid4().hex
+    db_manager.create_append_locator_map(locator, uuid2)
+
+    assert set(db_manager.load_locator_map(locator)) == set([uuid, uuid2])
+
+
+def test_update_locator_map(db_manager):
+    # Let's create a couple of appointments with the same locator
+    locator = get_random_value_hex(32)
+    uuid1 = uuid4().hex
+    uuid2 = uuid4().hex
+    db_manager.create_append_locator_map(locator, uuid1)
+    db_manager.create_append_locator_map(locator, uuid2)
+
+    locator_map = db_manager.load_locator_map(locator)
+    assert uuid1 in locator_map
+
+    locator_map.remove(uuid1)
+    db_manager.update_locator_map(locator, locator_map)
+
+    locator_map_after = db_manager.load_locator_map(locator)
+    assert uuid1 not in locator_map_after and uuid2 in locator_map_after and len(locator_map_after) == 1
+
+
+def test_update_locator_map_wong_data(db_manager):
+    # Let's try to update the locator map with a different list of uuids
+    locator = get_random_value_hex(32)
+    db_manager.create_append_locator_map(locator, uuid4().hex)
+    db_manager.create_append_locator_map(locator, uuid4().hex)
+
+    locator_map = db_manager.load_locator_map(locator)
+    wrong_map_update = [uuid4().hex]
+    db_manager.update_locator_map(locator, wrong_map_update)
+    locator_map_after = db_manager.load_locator_map(locator)
+
+    assert locator_map_after == locator_map
+
+
+def test_update_locator_map_empty(db_manager):
+    # We shouldn't be able to update a map with an empty list
+    locator = get_random_value_hex(32)
+    db_manager.create_append_locator_map(locator, uuid4().hex)
+    db_manager.create_append_locator_map(locator, uuid4().hex)
+
+    locator_map = db_manager.load_locator_map(locator)
+    db_manager.update_locator_map(locator, [])
+    locator_map_after = db_manager.load_locator_map(locator)
+
+    assert locator_map_after == locator_map
+
+
+def test_delete_locator_map(db_manager):
+    locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX)
+    assert len(locator_maps) != 0
+
+    for locator, uuids in locator_maps.items():
+        db_manager.delete_locator_map(locator)
+
+    locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX)
+    assert len(locator_maps) == 0
+
+
+def test_store_load_watcher_appointment(db_manager, watcher_appointments):
+    for uuid, appointment in watcher_appointments.items():
+        db_manager.store_watcher_appointment(uuid, appointment.to_dict())
+
+    db_watcher_appointments = db_manager.load_watcher_appointments()
+
+    # Check that the two appointment collections are equal by checking:
+    # - Their size is equal
+    # - Each element in one collection exists in the other
+
+    assert watcher_appointments.keys() == db_watcher_appointments.keys()
+
+    for uuid, appointment in watcher_appointments.items():
+        assert appointment.to_dict() == db_watcher_appointments[uuid]
+
+
+def test_store_load_triggered_appointment(db_manager):
+    db_watcher_appointments = db_manager.load_watcher_appointments()
+    db_watcher_appointments_with_triggered = db_manager.load_watcher_appointments(include_triggered=True)
+
+    assert db_watcher_appointments == db_watcher_appointments_with_triggered
+
+    # Create an appointment flagged as triggered
+    triggered_appointment, _ = generate_dummy_appointment(real_height=False)
+    uuid = uuid4().hex
+    db_manager.store_watcher_appointment(uuid, triggered_appointment.to_dict())
+    db_manager.create_triggered_appointment_flag(uuid)
+
+    # The new appointment is grabbed only if we set include_triggered
+    assert db_watcher_appointments == db_manager.load_watcher_appointments()
+    assert uuid in db_manager.load_watcher_appointments(include_triggered=True)
+
+
+def test_store_load_responder_trackers(db_manager, responder_trackers):
+    for key, value in responder_trackers.items():
+        db_manager.store_responder_tracker(key, {"value": value})
+
+    db_responder_trackers = db_manager.load_responder_trackers()
+
+    values = [tracker["value"] for tracker in db_responder_trackers.values()]
+
+    assert responder_trackers.keys() == db_responder_trackers.keys()
+    assert set(responder_trackers.values()) == set(values) and len(responder_trackers) == len(values)
+
+
+def test_delete_watcher_appointment(db_manager, watcher_appointments):
+    # Let's delete all we added
+    db_watcher_appointments = db_manager.load_watcher_appointments(include_triggered=True)
+    assert len(db_watcher_appointments) != 0
+
+    for key in watcher_appointments.keys():
+        db_manager.delete_watcher_appointment(key)
+
+    db_watcher_appointments = db_manager.load_watcher_appointments()
+    assert len(db_watcher_appointments) == 0
+
+
+def test_batch_delete_watcher_appointments(db_manager, watcher_appointments):
+    # Let's start by adding a bunch of appointments
+    for uuid, appointment in watcher_appointments.items():
+        db_manager.store_watcher_appointment(uuid, appointment.to_dict())
+
+    first_half = list(watcher_appointments.keys())[: len(watcher_appointments) // 2]
+    second_half = list(watcher_appointments.keys())[len(watcher_appointments) // 2 :]
+
+    # Let's now delete half of them in a batch update
+    db_manager.batch_delete_watcher_appointments(first_half)
+
+    db_watcher_appointments = db_manager.load_watcher_appointments()
+    assert not set(db_watcher_appointments.keys()).issuperset(first_half)
+    assert set(db_watcher_appointments.keys()).issuperset(second_half)
+
+    # Let's delete the rest
+    db_manager.batch_delete_watcher_appointments(second_half)
+
+    # Now there should be no appointments left
+    db_watcher_appointments = db_manager.load_watcher_appointments()
+    assert not db_watcher_appointments
+
+
+def test_delete_responder_tracker(db_manager, responder_trackers):
+    # Same for the responder
+    db_responder_trackers = db_manager.load_responder_trackers()
+    assert len(db_responder_trackers) != 0
+
+    for key in responder_trackers.keys():
+        db_manager.delete_responder_tracker(key)
+
+    db_responder_trackers = db_manager.load_responder_trackers()
+    assert len(db_responder_trackers) == 0
+
+
+def test_batch_delete_responder_trackers(db_manager, responder_trackers):
+    # Let's start by adding a bunch of appointments
+    for uuid, value in responder_trackers.items():
+        db_manager.store_responder_tracker(uuid, {"value": value})
+
+    first_half = list(responder_trackers.keys())[: len(responder_trackers) // 2]
+    second_half = list(responder_trackers.keys())[len(responder_trackers) // 2 :]
+
+    # Let's now delete half of them in a batch update
+    db_manager.batch_delete_responder_trackers(first_half)
+
+    db_responder_trackers = db_manager.load_responder_trackers()
+    assert not set(db_responder_trackers.keys()).issuperset(first_half)
+    assert set(db_responder_trackers.keys()).issuperset(second_half)
+
+    # Let's delete the rest
+    db_manager.batch_delete_responder_trackers(second_half)
+
+    # Now there should be no trackers left
+    db_responder_trackers = db_manager.load_responder_trackers()
+    assert not db_responder_trackers
+
+
+def test_store_load_last_block_hash_watcher(db_manager):
+    # Let's first create a made up block hash
+    local_last_block_hash = get_random_value_hex(32)
+    db_manager.store_last_block_hash_watcher(local_last_block_hash)
+
+    db_last_block_hash = db_manager.load_last_block_hash_watcher()
+
+    assert local_last_block_hash == db_last_block_hash
+
+
+def test_store_load_last_block_hash_responder(db_manager):
+    # Same for the responder
+    local_last_block_hash = get_random_value_hex(32)
+    db_manager.store_last_block_hash_responder(local_last_block_hash)
+
+    db_last_block_hash = db_manager.load_last_block_hash_responder()
+
+    assert local_last_block_hash == db_last_block_hash
+
+
+def test_create_triggered_appointment_flag(db_manager):
+    # Test that flags are added
+    key = get_random_value_hex(16)
+    db_manager.create_triggered_appointment_flag(key)
+
+    assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is not None
+
+    # Test to get a random one that we haven't added
+    key = get_random_value_hex(16)
+    assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is None
+
+
+def test_batch_create_triggered_appointment_flag(db_manager):
+    # Test that flags are added in batch
+    keys = [get_random_value_hex(16) for _ in range(10)]
+
+    # Checked that non of the flags is already in the db
+    db_flags = db_manager.load_all_triggered_flags()
+    assert not set(db_flags).issuperset(keys)
+
+    # Make sure that they are now
+    db_manager.batch_create_triggered_appointment_flag(keys)
+    db_flags = db_manager.load_all_triggered_flags()
+    assert set(db_flags).issuperset(keys)
+
+
+def test_load_all_triggered_flags(db_manager):
+    # There should be a some flags in the db from the previous tests. Let's load them
+    flags = db_manager.load_all_triggered_flags()
+
+    # We can add another flag and see that there's two now
+    new_uuid = uuid4().hex
+    db_manager.create_triggered_appointment_flag(new_uuid)
+    flags.append(new_uuid)
+
+    assert set(db_manager.load_all_triggered_flags()) == set(flags)
+
+
+def test_delete_triggered_appointment_flag(db_manager):
+    # Test data is properly deleted.
+    keys = db_manager.load_all_triggered_flags()
+
+    # Delete all entries
+    for k in keys:
+        db_manager.delete_triggered_appointment_flag(k)
+
+    # Try to load them back
+    for k in keys:
+        assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + k).encode("utf-8")) is None
+
+
+def test_batch_delete_triggered_appointment_flag(db_manager):
+    # Let's add some flags first
+    keys = [get_random_value_hex(16) for _ in range(10)]
+    db_manager.batch_create_triggered_appointment_flag(keys)
+
+    # And now let's delete in batch
+    first_half = keys[: len(keys) // 2]
+    second_half = keys[len(keys) // 2 :]
+
+    db_manager.batch_delete_triggered_appointment_flag(first_half)
+    db_falgs = db_manager.load_all_triggered_flags()
+    assert not set(db_falgs).issuperset(first_half)
+    assert set(db_falgs).issuperset(second_half)
+
+    # Delete the rest
+    db_manager.batch_delete_triggered_appointment_flag(second_half)
+    assert not db_manager.load_all_triggered_flags()

test/teos/unit/test_cleaner.py

@@ -27,7 +27,7 @@ def set_up_appointments(db_manager, total_appointments):
         appointments[uuid] = {"locator": appointment.locator}
         locator_uuid_map[locator] = [uuid]
 
-        db_manager.store_watcher_appointment(uuid, appointment.to_json())
+        db_manager.store_watcher_appointment(uuid, appointment.to_dict())
         db_manager.create_append_locator_map(locator, uuid)
 
         # Each locator can have more than one uuid assigned to it.
@@ -37,7 +37,7 @@ def set_up_appointments(db_manager, total_appointments):
             appointments[uuid] = {"locator": appointment.locator}
             locator_uuid_map[locator].append(uuid)
 
-            db_manager.store_watcher_appointment(uuid, appointment.to_json())
+            db_manager.store_watcher_appointment(uuid, appointment.to_dict())
             db_manager.create_append_locator_map(locator, uuid)
 
     return appointments, locator_uuid_map
@@ -60,7 +60,7 @@ def set_up_trackers(db_manager, total_trackers):
         trackers[uuid] = {"locator": tracker.locator, "penalty_txid": tracker.penalty_txid}
         tx_tracker_map[penalty_txid] = [uuid]
 
-        db_manager.store_responder_tracker(uuid, tracker.to_json())
+        db_manager.store_responder_tracker(uuid, tracker.to_dict())
         db_manager.create_append_locator_map(tracker.locator, uuid)
 
         # Each penalty_txid can have more than one uuid assigned to it.
@@ -70,7 +70,7 @@ def set_up_trackers(db_manager, total_trackers):
             trackers[uuid] = {"locator": tracker.locator, "penalty_txid": tracker.penalty_txid}
             tx_tracker_map[penalty_txid].append(uuid)
 
-            db_manager.store_responder_tracker(uuid, tracker.to_json())
+            db_manager.store_responder_tracker(uuid, tracker.to_dict())
             db_manager.create_append_locator_map(tracker.locator, uuid)
 
     return trackers, tx_tracker_map

test/teos/unit/test_db_manager.py

@@ -1,30 +1,10 @@
 import os
 import json
-import pytest
 import shutil
-from uuid import uuid4
+import pytest
 
 from teos.db_manager import DBManager
-from teos.db_manager import (
+from test.teos.unit.conftest import get_random_value_hex
-    WATCHER_LAST_BLOCK_KEY,
-    RESPONDER_LAST_BLOCK_KEY,
-    LOCATOR_MAP_PREFIX,
-    TRIGGERED_APPOINTMENTS_PREFIX,
-)
-
-from common.constants import LOCATOR_LEN_BYTES
-
-from test.teos.unit.conftest import get_random_value_hex, generate_dummy_appointment
-
-
-@pytest.fixture(scope="module")
-def watcher_appointments():
-    return {uuid4().hex: generate_dummy_appointment(real_height=False)[0] for _ in range(10)}
-
-
-@pytest.fixture(scope="module")
-def responder_trackers():
-    return {get_random_value_hex(16): get_random_value_hex(32) for _ in range(10)}
 
 
 def open_create_db(db_path):
@@ -62,67 +42,15 @@ def test_init():
     shutil.rmtree(db_path)
 
 
-def test_load_appointments_db(db_manager):
-    # Let's made up a prefix and try to load data from the database using it
-    prefix = "XX"
-    db_appointments = db_manager.load_appointments_db(prefix)
-
-    assert len(db_appointments) == 0
-
-    # We can add a bunch of data to the db and try again (data is stored in json by the manager)
-    local_appointments = {}
-    for _ in range(10):
-        key = get_random_value_hex(16)
-        value = get_random_value_hex(32)
-        local_appointments[key] = value
-
-        db_manager.db.put((prefix + key).encode("utf-8"), json.dumps({"value": value}).encode("utf-8"))
-
-    db_appointments = db_manager.load_appointments_db(prefix)
-
-    # Check that both keys and values are the same
-    assert db_appointments.keys() == local_appointments.keys()
-
-    values = [appointment["value"] for appointment in db_appointments.values()]
-    assert set(values) == set(local_appointments.values()) and (len(values) == len(local_appointments))
-
-
-def test_get_last_known_block():
-    db_path = "empty_db"
-
-    # First we check if the db exists, and if so we delete it
-    if os.path.isdir(db_path):
-        shutil.rmtree(db_path)
-
-    # Check that the db can be created if it does not exist
-    db_manager = open_create_db(db_path)
-
-    # Trying to get any last block for either the watcher or the responder should return None for an empty db
-
-    for key in [WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY]:
-        assert db_manager.get_last_known_block(key) is None
-
-    # After saving some block in the db we should get that exact value
-    for key in [WATCHER_LAST_BLOCK_KEY, RESPONDER_LAST_BLOCK_KEY]:
-        block_hash = get_random_value_hex(32)
-        db_manager.db.put(key.encode("utf-8"), block_hash.encode("utf-8"))
-        assert db_manager.get_last_known_block(key) == block_hash
-
-    # Removing test db
-    shutil.rmtree(db_path)
-
-
 def test_create_entry(db_manager):
     key = get_random_value_hex(16)
     value = get_random_value_hex(32)
 
-    # Adding a value with no prefix (create entry encodes values in utf-8 internally)
+    # Adding a value with no prefix should work
     db_manager.create_entry(key, value)
-
-    # We should be able to get it straightaway from the key
     assert db_manager.db.get(key.encode("utf-8")).decode("utf-8") == value
 
-    # If we prefix the key we should be able to get it if we add the prefix, but not otherwise
+    # Prefixing the key would require the prefix to load
     key = get_random_value_hex(16)
     prefix = "w"
     db_manager.create_entry(key, value, prefix=prefix)
@@ -130,22 +58,51 @@ def test_create_entry(db_manager):
     assert db_manager.db.get((prefix + key).encode("utf-8")).decode("utf-8") == value
     assert db_manager.db.get(key.encode("utf-8")) is None
 
-    # Same if we try to use any other prefix
+    # Keys, prefixes, and values of wrong format should fail
-    another_prefix = "r"
+    with pytest.raises(TypeError):
-    assert db_manager.db.get((another_prefix + key).encode("utf-8")) is None
+        db_manager.create_entry(key=None)
+
+    with pytest.raises(TypeError):
+        db_manager.create_entry(key=key, value=None)
+
+    with pytest.raises(TypeError):
+        db_manager.create_entry(key=key, value=value, prefix=1)
+
+
+def test_load_entry(db_manager):
+    key = get_random_value_hex(16)
+    value = get_random_value_hex(32)
+
+    # Loading an existing key should work
+    db_manager.db.put(key.encode("utf-8"), value.encode("utf-8"))
+    assert db_manager.load_entry(key) == value.encode("utf-8")
+
+    # Adding an existing prefix should work
+    assert db_manager.load_entry(key[2:], prefix=key[:2]) == value.encode("utf-8")
+
+    # Adding a non-existing prefix should return None
+    assert db_manager.load_entry(key, prefix=get_random_value_hex(2)) is None
+
+    # Loading a non-existing entry should return None
+    assert db_manager.load_entry(get_random_value_hex(16)) is None
+
+    # Trying to load a non str key or prefix should fail
+    with pytest.raises(TypeError):
+        db_manager.load_entry(None)
+
+    with pytest.raises(TypeError):
+        db_manager.load_entry(get_random_value_hex(16), prefix=1)
 
 
 def test_delete_entry(db_manager):
-    # Let's first get the key all the things we've wrote so far in the db
+    # Let's get the key all the things we've wrote so far in the db and empty the db.
     data = [k.decode("utf-8") for k, v in db_manager.db.iterator()]
-
-    # Let's empty the db now
     for key in data:
         db_manager.delete_entry(key)
 
     assert len([k for k, v in db_manager.db.iterator()]) == 0
 
-    # Let's check that the same works if a prefix is provided.
+    # The same works if a prefix is provided.
     prefix = "r"
     key = get_random_value_hex(16)
     value = get_random_value_hex(32)
@@ -158,294 +115,12 @@ def test_delete_entry(db_manager):
     db_manager.delete_entry(key, prefix)
     assert db_manager.db.get((prefix + key).encode("utf-8")) is None
 
+    # Deleting a non-existing key should be fine
+    db_manager.delete_entry(key, prefix)
 
-def test_load_watcher_appointments_empty(db_manager):
+    # Trying to delete a non str key or prefix should fail
-    assert len(db_manager.load_watcher_appointments()) == 0
+    with pytest.raises(TypeError):
+        db_manager.delete_entry(None)
 
-
+    with pytest.raises(TypeError):
-def test_load_responder_trackers_empty(db_manager):
+        db_manager.delete_entry(get_random_value_hex(16), prefix=1)
-    assert len(db_manager.load_responder_trackers()) == 0
-
-
-def test_load_locator_map_empty(db_manager):
-    assert db_manager.load_locator_map(get_random_value_hex(LOCATOR_LEN_BYTES)) is None
-
-
-def test_create_append_locator_map(db_manager):
-    uuid = uuid4().hex
-    locator = get_random_value_hex(LOCATOR_LEN_BYTES)
-    db_manager.create_append_locator_map(locator, uuid)
-
-    # Check that the locator map has been properly stored
-    assert db_manager.load_locator_map(locator) == [uuid]
-
-    # If we try to add the same uuid again the list shouldn't change
-    db_manager.create_append_locator_map(locator, uuid)
-    assert db_manager.load_locator_map(locator) == [uuid]
-
-    # Add another uuid to the same locator and check that it also works
-    uuid2 = uuid4().hex
-    db_manager.create_append_locator_map(locator, uuid2)
-
-    assert set(db_manager.load_locator_map(locator)) == set([uuid, uuid2])
-
-
-def test_update_locator_map(db_manager):
-    # Let's create a couple of appointments with the same locator
-    locator = get_random_value_hex(32)
-    uuid1 = uuid4().hex
-    uuid2 = uuid4().hex
-    db_manager.create_append_locator_map(locator, uuid1)
-    db_manager.create_append_locator_map(locator, uuid2)
-
-    locator_map = db_manager.load_locator_map(locator)
-    assert uuid1 in locator_map
-
-    locator_map.remove(uuid1)
-    db_manager.update_locator_map(locator, locator_map)
-
-    locator_map_after = db_manager.load_locator_map(locator)
-    assert uuid1 not in locator_map_after and uuid2 in locator_map_after and len(locator_map_after) == 1
-
-
-def test_update_locator_map_wong_data(db_manager):
-    # Let's try to update the locator map with a different list of uuids
-    locator = get_random_value_hex(32)
-    db_manager.create_append_locator_map(locator, uuid4().hex)
-    db_manager.create_append_locator_map(locator, uuid4().hex)
-
-    locator_map = db_manager.load_locator_map(locator)
-    wrong_map_update = [uuid4().hex]
-    db_manager.update_locator_map(locator, wrong_map_update)
-    locator_map_after = db_manager.load_locator_map(locator)
-
-    assert locator_map_after == locator_map
-
-
-def test_update_locator_map_empty(db_manager):
-    # We shouldn't be able to update a map with an empty list
-    locator = get_random_value_hex(32)
-    db_manager.create_append_locator_map(locator, uuid4().hex)
-    db_manager.create_append_locator_map(locator, uuid4().hex)
-
-    locator_map = db_manager.load_locator_map(locator)
-    db_manager.update_locator_map(locator, [])
-    locator_map_after = db_manager.load_locator_map(locator)
-
-    assert locator_map_after == locator_map
-
-
-def test_delete_locator_map(db_manager):
-    locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX)
-    assert len(locator_maps) != 0
-
-    for locator, uuids in locator_maps.items():
-        db_manager.delete_locator_map(locator)
-
-    locator_maps = db_manager.load_appointments_db(prefix=LOCATOR_MAP_PREFIX)
-    assert len(locator_maps) == 0
-
-
-def test_store_load_watcher_appointment(db_manager, watcher_appointments):
-    for uuid, appointment in watcher_appointments.items():
-        db_manager.store_watcher_appointment(uuid, appointment.to_json())
-
-    db_watcher_appointments = db_manager.load_watcher_appointments()
-
-    # Check that the two appointment collections are equal by checking:
-    # - Their size is equal
-    # - Each element in one collection exists in the other
-
-    assert watcher_appointments.keys() == db_watcher_appointments.keys()
-
-    for uuid, appointment in watcher_appointments.items():
-        assert json.dumps(db_watcher_appointments[uuid], sort_keys=True, separators=(",", ":")) == appointment.to_json()
-
-
-def test_store_load_triggered_appointment(db_manager):
-    db_watcher_appointments = db_manager.load_watcher_appointments()
-    db_watcher_appointments_with_triggered = db_manager.load_watcher_appointments(include_triggered=True)
-
-    assert db_watcher_appointments == db_watcher_appointments_with_triggered
-
-    # Create an appointment flagged as triggered
-    triggered_appointment, _ = generate_dummy_appointment(real_height=False)
-    uuid = uuid4().hex
-    db_manager.store_watcher_appointment(uuid, triggered_appointment.to_json())
-    db_manager.create_triggered_appointment_flag(uuid)
-
-    # The new appointment is grabbed only if we set include_triggered
-    assert db_watcher_appointments == db_manager.load_watcher_appointments()
-    assert uuid in db_manager.load_watcher_appointments(include_triggered=True)
-
-
-def test_store_load_responder_trackers(db_manager, responder_trackers):
-    for key, value in responder_trackers.items():
-        db_manager.store_responder_tracker(key, json.dumps({"value": value}))
-
-    db_responder_trackers = db_manager.load_responder_trackers()
-
-    values = [tracker["value"] for tracker in db_responder_trackers.values()]
-
-    assert responder_trackers.keys() == db_responder_trackers.keys()
-    assert set(responder_trackers.values()) == set(values) and len(responder_trackers) == len(values)
-
-
-def test_delete_watcher_appointment(db_manager, watcher_appointments):
-    # Let's delete all we added
-    db_watcher_appointments = db_manager.load_watcher_appointments(include_triggered=True)
-    assert len(db_watcher_appointments) != 0
-
-    for key in watcher_appointments.keys():
-        db_manager.delete_watcher_appointment(key)
-
-    db_watcher_appointments = db_manager.load_watcher_appointments()
-    assert len(db_watcher_appointments) == 0
-
-
-def test_batch_delete_watcher_appointments(db_manager, watcher_appointments):
-    # Let's start by adding a bunch of appointments
-    for uuid, appointment in watcher_appointments.items():
-        db_manager.store_watcher_appointment(uuid, appointment.to_json())
-
-    first_half = list(watcher_appointments.keys())[: len(watcher_appointments) // 2]
-    second_half = list(watcher_appointments.keys())[len(watcher_appointments) // 2 :]
-
-    # Let's now delete half of them in a batch update
-    db_manager.batch_delete_watcher_appointments(first_half)
-
-    db_watcher_appointments = db_manager.load_watcher_appointments()
-    assert not set(db_watcher_appointments.keys()).issuperset(first_half)
-    assert set(db_watcher_appointments.keys()).issuperset(second_half)
-
-    # Let's delete the rest
-    db_manager.batch_delete_watcher_appointments(second_half)
-
-    # Now there should be no appointments left
-    db_watcher_appointments = db_manager.load_watcher_appointments()
-    assert not db_watcher_appointments
-
-
-def test_delete_responder_tracker(db_manager, responder_trackers):
-    # Same for the responder
-    db_responder_trackers = db_manager.load_responder_trackers()
-    assert len(db_responder_trackers) != 0
-
-    for key in responder_trackers.keys():
-        db_manager.delete_responder_tracker(key)
-
-    db_responder_trackers = db_manager.load_responder_trackers()
-    assert len(db_responder_trackers) == 0
-
-
-def test_batch_delete_responder_trackers(db_manager, responder_trackers):
-    # Let's start by adding a bunch of appointments
-    for uuid, value in responder_trackers.items():
-        db_manager.store_responder_tracker(uuid, json.dumps({"value": value}))
-
-    first_half = list(responder_trackers.keys())[: len(responder_trackers) // 2]
-    second_half = list(responder_trackers.keys())[len(responder_trackers) // 2 :]
-
-    # Let's now delete half of them in a batch update
-    db_manager.batch_delete_responder_trackers(first_half)
-
-    db_responder_trackers = db_manager.load_responder_trackers()
-    assert not set(db_responder_trackers.keys()).issuperset(first_half)
-    assert set(db_responder_trackers.keys()).issuperset(second_half)
-
-    # Let's delete the rest
-    db_manager.batch_delete_responder_trackers(second_half)
-
-    # Now there should be no trackers left
-    db_responder_trackers = db_manager.load_responder_trackers()
-    assert not db_responder_trackers
-
-
-def test_store_load_last_block_hash_watcher(db_manager):
-    # Let's first create a made up block hash
-    local_last_block_hash = get_random_value_hex(32)
-    db_manager.store_last_block_hash_watcher(local_last_block_hash)
-
-    db_last_block_hash = db_manager.load_last_block_hash_watcher()
-
-    assert local_last_block_hash == db_last_block_hash
-
-
-def test_store_load_last_block_hash_responder(db_manager):
-    # Same for the responder
-    local_last_block_hash = get_random_value_hex(32)
-    db_manager.store_last_block_hash_responder(local_last_block_hash)
-
-    db_last_block_hash = db_manager.load_last_block_hash_responder()
-
-    assert local_last_block_hash == db_last_block_hash
-
-
-def test_create_triggered_appointment_flag(db_manager):
-    # Test that flags are added
-    key = get_random_value_hex(16)
-    db_manager.create_triggered_appointment_flag(key)
-
-    assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is not None
-
-    # Test to get a random one that we haven't added
-    key = get_random_value_hex(16)
-    assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + key).encode("utf-8")) is None
-
-
-def test_batch_create_triggered_appointment_flag(db_manager):
-    # Test that flags are added in batch
-    keys = [get_random_value_hex(16) for _ in range(10)]
-
-    # Checked that non of the flags is already in the db
-    db_flags = db_manager.load_all_triggered_flags()
-    assert not set(db_flags).issuperset(keys)
-
-    # Make sure that they are now
-    db_manager.batch_create_triggered_appointment_flag(keys)
-    db_flags = db_manager.load_all_triggered_flags()
-    assert set(db_flags).issuperset(keys)
-
-
-def test_load_all_triggered_flags(db_manager):
-    # There should be a some flags in the db from the previous tests. Let's load them
-    flags = db_manager.load_all_triggered_flags()
-
-    # We can add another flag and see that there's two now
-    new_uuid = uuid4().hex
-    db_manager.create_triggered_appointment_flag(new_uuid)
-    flags.append(new_uuid)
-
-    assert set(db_manager.load_all_triggered_flags()) == set(flags)
-
-
-def test_delete_triggered_appointment_flag(db_manager):
-    # Test data is properly deleted.
-    keys = db_manager.load_all_triggered_flags()
-
-    # Delete all entries
-    for k in keys:
-        db_manager.delete_triggered_appointment_flag(k)
-
-    # Try to load them back
-    for k in keys:
-        assert db_manager.db.get((TRIGGERED_APPOINTMENTS_PREFIX + k).encode("utf-8")) is None
-
-
-def test_batch_delete_triggered_appointment_flag(db_manager):
-    # Let's add some flags first
-    keys = [get_random_value_hex(16) for _ in range(10)]
-    db_manager.batch_create_triggered_appointment_flag(keys)
-
-    # And now let's delete in batch
-    first_half = keys[: len(keys) // 2]
-    second_half = keys[len(keys) // 2 :]
-
-    db_manager.batch_delete_triggered_appointment_flag(first_half)
-    db_falgs = db_manager.load_all_triggered_flags()
-    assert not set(db_falgs).issuperset(first_half)
-    assert set(db_falgs).issuperset(second_half)
-
-    # Delete the rest
-    db_manager.batch_delete_triggered_appointment_flag(second_half)
-    assert not db_manager.load_all_triggered_flags()

test/teos/unit/test_responder.py

@@ -9,8 +9,8 @@ from threading import Thread
 
 from teos.carrier import Carrier
 from teos.tools import bitcoin_cli
-from teos.db_manager import DBManager
 from teos.chain_monitor import ChainMonitor
+from teos.appointments_dbm import AppointmentsDBM
 from teos.responder import Responder, TransactionTracker
 
 from common.constants import LOCATOR_LEN_HEX
@@ -36,7 +36,7 @@ def responder(db_manager, carrier, block_processor):
 @pytest.fixture(scope="session")
 def temp_db_manager():
     db_name = get_random_value_hex(8)
-    db_manager = DBManager(db_name)
+    db_manager = AppointmentsDBM(db_name)
 
     yield db_manager
 
@@ -120,17 +120,6 @@ def test_tracker_to_dict():
     )
 
 
-def test_tracker_to_json():
-    tracker = create_dummy_tracker()
-    tracker_dict = json.loads(tracker.to_json())
-
-    assert (
-        tracker.locator == tracker_dict["locator"]
-        and tracker.penalty_rawtx == tracker_dict["penalty_rawtx"]
-        and tracker.appointment_end == tracker_dict["appointment_end"]
-    )
-
-
 def test_tracker_from_dict():
     tracker_dict = create_dummy_tracker().to_dict()
     new_tracker = TransactionTracker.from_dict(tracker_dict)
@@ -295,7 +284,7 @@ def test_do_watch(temp_db_manager, carrier, block_processor):
 
         # We also need to store the info in the db
         responder.db_manager.create_triggered_appointment_flag(uuid)
-        responder.db_manager.store_responder_tracker(uuid, tracker.to_json())
+        responder.db_manager.store_responder_tracker(uuid, tracker.to_dict())
 
     # Let's start to watch
     Thread(target=responder.do_watch, daemon=True).start()
@@ -472,7 +461,7 @@ def test_rebroadcast(db_manager, carrier, block_processor):
 
         # We need to add it to the db too
         responder.db_manager.create_triggered_appointment_flag(uuid)
-        responder.db_manager.store_responder_tracker(uuid, tracker.to_json())
+        responder.db_manager.store_responder_tracker(uuid, tracker.to_dict())
 
         responder.tx_tracker_map[penalty_txid] = [uuid]
         responder.unconfirmed_txs.append(penalty_txid)

test/teos/unit/test_watcher.py

@@ -9,8 +9,8 @@ from teos.carrier import Carrier
 from teos.watcher import Watcher
 from teos.tools import bitcoin_cli
 from teos.responder import Responder
-from teos.db_manager import DBManager
 from teos.chain_monitor import ChainMonitor
+from teos.appointments_dbm import AppointmentsDBM
 from teos.block_processor import BlockProcessor
 
 import common.cryptographer
@@ -47,7 +47,7 @@ MAX_APPOINTMENTS = 100
 @pytest.fixture(scope="session")
 def temp_db_manager():
     db_name = get_random_value_hex(8)
-    db_manager = DBManager(db_name)
+    db_manager = AppointmentsDBM(db_name)
 
     yield db_manager
 
@@ -198,7 +198,7 @@ def test_do_watch(watcher, temp_db_manager):
 
     for uuid, appointment in appointments.items():
         watcher.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time, "size": 200}
-        watcher.db_manager.store_watcher_appointment(uuid, appointment.to_json())
+        watcher.db_manager.store_watcher_appointment(uuid, appointment.to_dict())
         watcher.db_manager.create_append_locator_map(appointment.locator, uuid)
 
     do_watch_thread = Thread(target=watcher.do_watch, daemon=True)
@@ -248,7 +248,7 @@ def test_filter_valid_breaches_random_data(watcher):
         dummy_appointment, _ = generate_dummy_appointment()
         uuid = uuid4().hex
         appointments[uuid] = {"locator": dummy_appointment.locator, "end_time": dummy_appointment.end_time}
-        watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_json())
+        watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_dict())
         watcher.db_manager.create_append_locator_map(dummy_appointment.locator, uuid)
 
         locator_uuid_map[dummy_appointment.locator] = [uuid]
@@ -288,7 +288,7 @@ def test_filter_valid_breaches(watcher):
 
     for uuid, appointment in appointments.items():
         watcher.appointments[uuid] = {"locator": appointment.locator, "end_time": appointment.end_time}
-        watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_json())
+        watcher.db_manager.store_watcher_appointment(uuid, dummy_appointment.to_dict())
         watcher.db_manager.create_append_locator_map(dummy_appointment.locator, uuid)
 
     watcher.locator_uuid_map = locator_uuid_map