[Mint] Migrate duplicate keysets to db (#511)

* wip

* remove all deprecated keyset tests

* fix more tests

* fixups

cashu/core/base.py

@@ -656,7 +656,6 @@ class WalletKeyset:
         valid_to=None,
         first_seen=None,
         active=True,
-        use_deprecated_id=False,  # BACKWARDS COMPATIBILITY < 0.15.0
     ):
         self.valid_from = valid_from
         self.valid_to = valid_to
@@ -671,19 +670,10 @@ class WalletKeyset:
         else:
             self.id = id
 
-        # BEGIN BACKWARDS COMPATIBILITY < 0.15.0
-        if use_deprecated_id:
-            logger.warning(
-                "Using deprecated keyset id derivation for backwards compatibility <"
-                " 0.15.0"
-            )
-            self.id = derive_keyset_id_deprecated(self.public_keys)
-        # END BACKWARDS COMPATIBILITY < 0.15.0
-
         self.unit = Unit[unit]
 
         logger.trace(f"Derived keyset id {self.id} from public keys.")
-        if id and id != self.id and use_deprecated_id:
+        if id and id != self.id:
             logger.warning(
                 f"WARNING: Keyset id {self.id} does not match the given id {id}."
                 " Overwriting."
@@ -738,8 +728,6 @@ class MintKeyset:
     first_seen: Optional[str] = None
     version: Optional[str] = None
 
-    duplicate_keyset_id: Optional[str] = None  # BACKWARDS COMPATIBILITY < 0.15.0
-
     def __init__(
         self,
         *,
@@ -820,6 +808,12 @@ class MintKeyset:
         assert self.seed, "seed not set"
         assert self.derivation_path, "derivation path not set"
 
+        # we compute the keyset id from the public keys only if it is not
+        # loaded from the database. This is to allow for backwards compatibility
+        # with old keysets with new id's and vice versa. This code can be removed
+        # if there are only new keysets in the mint (> 0.15.0)
+        id_in_db = self.id
+
         if self.version_tuple < (0, 12):
             # WARNING: Broken key derivation for backwards compatibility with < 0.12
             self.private_keys = derive_keys_backwards_compatible_insecure_pre_0_12(
@@ -830,7 +824,8 @@ class MintKeyset:
                 f"WARNING: Using weak key derivation for keyset {self.id} (backwards"
                 " compatibility < 0.12)"
             )
-            self.id = derive_keyset_id_deprecated(self.public_keys)  # type: ignore
+            # load from db or derive
+            self.id = id_in_db or derive_keyset_id_deprecated(self.public_keys)  # type: ignore
         elif self.version_tuple < (0, 15):
             self.private_keys = derive_keys_sha256(self.seed, self.derivation_path)
             logger.trace(
@@ -838,11 +833,13 @@ class MintKeyset:
                 " compatibility < 0.15)"
             )
             self.public_keys = derive_pubkeys(self.private_keys)  # type: ignore
-            self.id = derive_keyset_id_deprecated(self.public_keys)  # type: ignore
+            # load from db or derive
+            self.id = id_in_db or derive_keyset_id_deprecated(self.public_keys)  # type: ignore
         else:
             self.private_keys = derive_keys(self.seed, self.derivation_path)
             self.public_keys = derive_pubkeys(self.private_keys)  # type: ignore
-            self.id = derive_keyset_id(self.public_keys)  # type: ignore
+            # load from db or derive
+            self.id = id_in_db or derive_keyset_id(self.public_keys)  # type: ignore
 
 
 # ------- TOKEN -------

cashu/core/settings.py

@@ -52,20 +52,12 @@ class MintSettings(CashuSettings):
     mint_private_key: str = Field(default=None)
     mint_seed_decryption_key: Optional[str] = Field(default=None)
     mint_derivation_path: str = Field(default="m/0'/0'/0'")
-    mint_derivation_path_list: List[str] = Field(default=[])
+    mint_derivation_path_list: List[str] = Field(default=[""])
     mint_listen_host: str = Field(default="127.0.0.1")
     mint_listen_port: int = Field(default=3338)
 
     mint_database: str = Field(default="data/mint")
     mint_test_database: str = Field(default="test_data/test_mint")
-    mint_duplicate_old_keysets: bool = Field(
-        default=True,
-        title="Duplicate keysets",
-        description=(
-            "Whether to duplicate keysets for backwards compatibility before v1 API"
-            " (Nutshell 0.15.0)."
-        ),
-    )
 
 
 class MintBackends(MintSettings):
@@ -134,7 +126,6 @@ class MintInformation(CashuSettings):
     mint_info_description: str = Field(default=None)
     mint_info_description_long: str = Field(default=None)
     mint_info_contact: List[List[str]] = Field(default=[["", ""]])
-    mint_info_nuts: List[str] = Field(default=["NUT-07", "NUT-08", "NUT-09"])
     mint_info_motd: str = Field(default=None)
 
 

cashu/mint/ledger.py

@@ -1,5 +1,4 @@
 import asyncio
-import copy
 import time
 from typing import Dict, List, Mapping, Optional, Tuple
 
@@ -26,8 +25,6 @@ from ..core.base import (
 from ..core.crypto import b_dhke
 from ..core.crypto.aes import AESCipher
 from ..core.crypto.keys import (
-    derive_keyset_id,
-    derive_keyset_id_deprecated,
     derive_pubkey,
     random_hash,
 )
@@ -231,19 +228,10 @@ class Ledger(LedgerVerification, LedgerSpendingConditions):
         # load the new keyset in self.keysets
         self.keysets[keyset.id] = keyset
 
-        # BEGIN BACKWARDS COMPATIBILITY < 0.15.0
-        # set the deprecated id
-        if not keyset.public_keys:
-            raise KeysetError("no public keys for this keyset")
-        keyset.duplicate_keyset_id = derive_keyset_id_deprecated(keyset.public_keys)
-        # END BACKWARDS COMPATIBILITY < 0.15.0
-
         logger.debug(f"Loaded keyset {keyset.id}")
         return keyset
 
-    async def init_keysets(
+    async def init_keysets(self, autosave: bool = True) -> None:
-        self, autosave: bool = True, duplicate_keysets: Optional[bool] = None
-    ) -> None:
         """Initializes all keysets of the mint from the db. Loads all past keysets from db
         and generate their keys. Then activate the current keyset set by self.derivation_path.
 
@@ -251,9 +239,6 @@ class Ledger(LedgerVerification, LedgerSpendingConditions):
             autosave (bool, optional): Whether the current keyset should be saved if it is
                 not in the database yet. Will be passed to `self.activate_keyset` where it is
                 generated from `self.derivation_path`. Defaults to True.
-            duplicate_keysets (bool, optional): Whether to duplicate new keysets and compute
-                their old keyset id, and duplicate old keysets and compute their new keyset id.
-                Defaults to False.
         """
         # load all past keysets from db, the keys will be generated at instantiation
         tmp_keysets: List[MintKeyset] = await self.crud.get_keyset(db=self.db)
@@ -275,31 +260,6 @@ class Ledger(LedgerVerification, LedgerSpendingConditions):
         if not any([k.active for k in self.keysets.values()]):
             raise KeysetError("No active keyset found.")
 
-        # BEGIN BACKWARDS COMPATIBILITY < 0.15.0
-        # we duplicate new keysets and compute their old keyset id, and
-        # we duplicate old keysets and compute their new keyset id
-        if duplicate_keysets is not False and (
-            settings.mint_duplicate_old_keysets or duplicate_keysets
-        ):
-            for _, keyset in copy.copy(self.keysets).items():
-                # if keyset.version_tuple >= (0, 15, 3) and not duplicate_keysets:
-                #     # we do not duplicate keysets from version 0.15.3 and above if not forced by duplicate_keysets
-                #     continue
-                keyset_copy = copy.copy(keyset)
-                if not keyset_copy.public_keys:
-                    raise KeysetError("no public keys for this keyset")
-                if keyset.version_tuple >= (0, 15):
-                    keyset_copy.id = derive_keyset_id_deprecated(
-                        keyset_copy.public_keys
-                    )
-                else:
-                    keyset_copy.id = derive_keyset_id(keyset_copy.public_keys)
-                keyset_copy.duplicate_keyset_id = keyset.id
-                self.keysets[keyset_copy.id] = keyset_copy
-                # remember which keyset this keyset was duplicated from
-                logger.debug(f"Duplicated keyset id {keyset.id} -> {keyset_copy.id}")
-        # END BACKWARDS COMPATIBILITY < 0.15.0
-
     def get_keyset(self, keyset_id: Optional[str] = None) -> Dict[int, str]:
         """Returns a dictionary of hex public keys of a specific keyset for each supported amount"""
         if keyset_id and keyset_id not in self.keysets:
@@ -982,10 +942,7 @@ class Ledger(LedgerVerification, LedgerSpendingConditions):
             keyset = keyset or self.keysets[output.id]
             if output.id not in self.keysets:
                 raise TransactionError(f"keyset {output.id} not found")
-            if output.id not in [
+            if output.id != keyset.id:
-                keyset.id,
-                keyset.duplicate_keyset_id,
-            ]:
                 raise TransactionError("keyset id does not match output id")
             if not keyset.active:
                 raise TransactionError("keyset is not active")

cashu/mint/migrations.py

@@ -1,4 +1,7 @@
-from ..core.base import Proof
+import copy
+
+from ..core.base import MintKeyset, Proof
+from ..core.crypto.keys import derive_keyset_id, derive_keyset_id_deprecated
 from ..core.db import Connection, Database, table_with_schema, timestamp_now
 from ..core.settings import settings
 
@@ -718,3 +721,45 @@ async def m017_foreign_keys_proof_tables(db: Database):
 
     # recreate indices
     await m015_add_index_Y_to_proofs_used_and_pending(db)
+
+
+async def m018_duplicate_deprecated_keyset_ids(db: Database):
+    async with db.connect() as conn:
+        rows = await conn.fetchall(  # type: ignore
+            f"""
+                SELECT * from {table_with_schema(db, 'keysets')}
+                """,
+        )
+        keysets = [MintKeyset(**row) for row in rows]
+        duplicated_keysets: list[MintKeyset] = []
+        for keyset in keysets:
+            keyset_copy = copy.copy(keyset)
+            if not keyset_copy.public_keys:
+                raise Exception(f"keyset {keyset_copy.id} has no public keys")
+            if keyset.version_tuple < (0, 15):
+                keyset_copy.id = derive_keyset_id(keyset_copy.public_keys)
+            else:
+                keyset_copy.id = derive_keyset_id_deprecated(keyset_copy.public_keys)
+            duplicated_keysets.append(keyset_copy)
+
+        for keyset in duplicated_keysets:
+            await conn.execute(
+                f"""
+                INSERT INTO {table_with_schema(db, 'keysets')}
+                (id, derivation_path, valid_from, valid_to, first_seen, active, version, seed, unit, encrypted_seed, seed_encryption_method)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    keyset.id,
+                    keyset.derivation_path,
+                    keyset.valid_from,
+                    keyset.valid_to,
+                    keyset.first_seen,
+                    keyset.active,
+                    keyset.version,
+                    keyset.seed,
+                    keyset.unit.name,
+                    keyset.encrypted_seed,
+                    keyset.seed_encryption_method,
+                ),
+            )

cashu/mint/router_deprecated.py

@@ -52,7 +52,7 @@ async def info() -> GetInfoResponse_deprecated:
         description=settings.mint_info_description,
         description_long=settings.mint_info_description_long,
         contact=settings.mint_info_contact,
-        nuts=settings.mint_info_nuts,
+        nuts=["NUT-07", "NUT-08", "NUT-09"],
         motd=settings.mint_info_motd,
         parameter={
             "max_peg_in": settings.mint_max_peg_in,
@@ -178,12 +178,8 @@ async def mint_deprecated(
 
     # BEGIN BACKWARDS COMPATIBILITY < 0.15
     # Mint expects "id" in outputs to know which keyset to use to sign them.
-    # use the deprecated version of the current keyset
-    assert ledger.keyset.duplicate_keyset_id
     outputs: list[BlindedMessage] = [
-        BlindedMessage(
+        BlindedMessage(id=o.id or ledger.keyset.id, **o.dict(exclude={"id"}))
-            id=o.id or ledger.keyset.duplicate_keyset_id, **o.dict(exclude={"id"})
-        )
         for o in payload.outputs
     ]
     # END BACKWARDS COMPATIBILITY < 0.15

cashu/wallet/wallet_deprecated.py

@@ -160,9 +160,7 @@ class LedgerAPIDeprecated(SupportsHttpxClient, SupportsMintURL):
             int(amt): PublicKey(bytes.fromhex(val), raw=True)
             for amt, val in keys.items()
         }
-        keyset = WalletKeyset(
+        keyset = WalletKeyset(unit="sat", public_keys=keyset_keys, mint_url=url)
-            unit="sat", public_keys=keyset_keys, mint_url=url, use_deprecated_id=True
-        )
         return keyset
 
     @async_set_httpx_client
@@ -199,7 +197,6 @@ class LedgerAPIDeprecated(SupportsHttpxClient, SupportsMintURL):
             id=keyset_id,
             public_keys=keyset_keys,
             mint_url=url,
-            use_deprecated_id=True,
         )
         return keyset
 

tests/conftest.py

@@ -73,28 +73,25 @@ class UvicornServer(multiprocessing.Process):
         self.server.run()
 
 
-# # This fixture is used for tests that require API access to the mint
-@pytest.fixture(autouse=True, scope="session")
-def mint():
-    config = uvicorn.Config(
-        "cashu.mint.app:app",
-        port=settings.mint_listen_port,
-        host=settings.mint_listen_host,
-    )
-
-    server = UvicornServer(config=config)
-    server.start()
-    time.sleep(1)
-    yield server
-    server.stop()
-
-
 # This fixture is used for all other tests
 @pytest_asyncio.fixture(scope="function")
 async def ledger():
-    async def start_mint_init(ledger: Ledger):
+    async def start_mint_init(ledger: Ledger) -> Ledger:
         await migrate_databases(ledger.db, migrations_mint)
+        # add a new keyset (with a new ID) which will be duplicated with a keyset with an
+        # old ID by mint migration m018_duplicate_deprecated_keyset_ids
+        # await ledger.activate_keyset(derivation_path=settings.mint_derivation_path, version="0.15.0")
+        # await migrations_mint.m018_duplicate_deprecated_keyset_ids(ledger.db)
+
+        ledger = Ledger(
+            db=Database("mint", settings.mint_database),
+            seed=settings.mint_private_key,
+            derivation_path=settings.mint_derivation_path,
+            backends=backends,
+            crud=LedgerCrudSqlite(),
+        )
         await ledger.startup_ledger()
+        return ledger
 
     if not settings.mint_database.startswith("postgres"):
         # clear sqlite database
@@ -120,6 +117,22 @@ async def ledger():
         backends=backends,
         crud=LedgerCrudSqlite(),
     )
-    await start_mint_init(ledger)
+    ledger = await start_mint_init(ledger)
     yield ledger
     print("teardown")
+
+
+# # This fixture is used for tests that require API access to the mint
+@pytest.fixture(autouse=True, scope="session")
+def mint():
+    config = uvicorn.Config(
+        "cashu.mint.app:app",
+        port=settings.mint_listen_port,
+        host=settings.mint_listen_host,
+    )
+
+    server = UvicornServer(config=config)
+    server.start()
+    time.sleep(1)
+    yield server
+    server.stop()

tests/test_mint.py

@@ -58,17 +58,6 @@ async def test_keysets(ledger: Ledger):
     assert ledger.keyset.id == "009a1f293253e41e"
 
 
-@pytest.mark.asyncio
-async def test_keysets_backwards_compatibility_pre_v0_15(ledger: Ledger):
-    """Backwards compatibility test for keysets pre v0.15.0
-    We expect two instances of the same keyset but with different IDs.
-    First one is the new hex ID, second one is the old base64 ID.
-    """
-    assert len(ledger.keysets) == 2
-    assert list(ledger.keysets.keys()) == ["009a1f293253e41e", "eGnEWtdJ0PIM"]
-    assert ledger.keyset.id == "009a1f293253e41e"
-
-
 @pytest.mark.asyncio
 async def test_get_keyset(ledger: Ledger):
     keyset = ledger.get_keyset()
@@ -136,55 +125,6 @@ async def test_generate_promises(ledger: Ledger):
     assert promises[0].dleq.e
 
 
-@pytest.mark.asyncio
-async def test_generate_promises_deprecated_keyset_id(ledger: Ledger):
-    blinded_messages_mock = [
-        BlindedMessage(
-            amount=8,
-            B_="02634a2c2b34bec9e8a4aba4361f6bf202d7fa2365379b0840afe249a7a9d71239",
-            id="eGnEWtdJ0PIM",
-        )
-    ]
-    promises = await ledger._generate_promises(blinded_messages_mock)
-    assert (
-        promises[0].C_
-        == "031422eeffb25319e519c68de000effb294cb362ef713a7cf4832cea7b0452ba6e"
-    )
-    assert promises[0].amount == 8
-    assert promises[0].id == "eGnEWtdJ0PIM"
-
-    # DLEQ proof present
-    assert promises[0].dleq
-    assert promises[0].dleq.s
-    assert promises[0].dleq.e
-
-
-@pytest.mark.asyncio
-async def test_generate_promises_keyset_backwards_compatibility_pre_v0_15(
-    ledger: Ledger,
-):
-    """Backwards compatibility test for keysets pre v0.15.0
-    We want to generate promises using the old keyset ID.
-    We expect the promise to have the old base64 ID.
-    """
-    blinded_messages_mock = [
-        BlindedMessage(
-            amount=8,
-            B_="02634a2c2b34bec9e8a4aba4361f6bf202d7fa2365379b0840afe249a7a9d71239",
-            id="eGnEWtdJ0PIM",
-        )
-    ]
-    promises = await ledger._generate_promises(
-        blinded_messages_mock, keyset=ledger.keysets["eGnEWtdJ0PIM"]
-    )
-    assert (
-        promises[0].C_
-        == "031422eeffb25319e519c68de000effb294cb362ef713a7cf4832cea7b0452ba6e"
-    )
-    assert promises[0].amount == 8
-    assert promises[0].id == "eGnEWtdJ0PIM"
-
-
 @pytest.mark.asyncio
 async def test_generate_change_promises(ledger: Ledger):
     # Example slightly adapted from NUT-08 because we want to ensure the dynamic change

tests/test_mint_api.py

@@ -90,13 +90,6 @@ async def test_api_keysets(ledger: Ledger):
                 "unit": "sat",
                 "active": True,
             },
-            # for backwards compatibility of the new keyset ID format,
-            # we also return the same keyset with the old base64 ID
-            {
-                "id": "eGnEWtdJ0PIM",
-                "unit": "sat",
-                "active": True,
-            },
         ]
     }
     assert response.json() == expected
@@ -132,17 +125,17 @@ async def test_api_keyset_keys(ledger: Ledger):
     reason="settings.debug_mint_only_deprecated is set",
 )
 async def test_api_keyset_keys_old_keyset_id(ledger: Ledger):
-    response = httpx.get(f"{BASE_URL}/v1/keys/eGnEWtdJ0PIM")
+    response = httpx.get(f"{BASE_URL}/v1/keys/009a1f293253e41e")
     assert response.status_code == 200, f"{response.url} {response.status_code}"
     assert ledger.keyset.public_keys
     expected = {
         "keysets": [
             {
-                "id": "eGnEWtdJ0PIM",
+                "id": "009a1f293253e41e",
                 "unit": "sat",
                 "keys": {
                     str(k): v.serialize().hex()
-                    for k, v in ledger.keysets["eGnEWtdJ0PIM"].public_keys.items()  # type: ignore
+                    for k, v in ledger.keysets["009a1f293253e41e"].public_keys.items()  # type: ignore
                 },
             }
         ]

tests/test_mint_api_deprecated.py

@@ -9,7 +9,6 @@ from cashu.core.base import (
     PostRestoreResponse,
     Proof,
 )
-from cashu.core.settings import settings
 from cashu.mint.ledger import Ledger
 from cashu.wallet.crud import bump_secret_derivation
 from cashu.wallet.wallet import Wallet
@@ -141,9 +140,6 @@ async def test_mint(ledger: Ledger, wallet: Wallet):
     assert len(result["promises"]) == 2
     assert result["promises"][0]["amount"] == 32
     assert result["promises"][1]["amount"] == 32
-    if settings.debug_mint_only_deprecated:
-        assert result["promises"][0]["id"] == "eGnEWtdJ0PIM"
-    else:
     assert result["promises"][0]["id"] == "009a1f293253e41e"
     assert result["promises"][0]["dleq"]
     assert "e" in result["promises"][0]["dleq"]

tests/test_mint_init.py

@@ -56,31 +56,8 @@ async def wallet(ledger: Ledger):
 
 
 @pytest.mark.asyncio
-async def test_init_keysets_with_duplicates(ledger: Ledger):
+async def test_init_keysets(ledger: Ledger):
     ledger.keysets = {}
-    await ledger.init_keysets(duplicate_keysets=True)
-    assert len(ledger.keysets) == 2
-
-
-@pytest.mark.asyncio
-async def test_init_keysets_with_duplicates_via_settings(ledger: Ledger):
-    ledger.keysets = {}
-    settings.mint_duplicate_old_keysets = True
-    await ledger.init_keysets()
-    assert len(ledger.keysets) == 2
-
-
-@pytest.mark.asyncio
-async def test_init_keysets_without_duplicates(ledger: Ledger):
-    ledger.keysets = {}
-    await ledger.init_keysets(duplicate_keysets=False)
-    assert len(ledger.keysets) == 1
-
-
-@pytest.mark.asyncio
-async def test_init_keysets_without_duplicates_via_settings(ledger: Ledger):
-    ledger.keysets = {}
-    settings.mint_duplicate_old_keysets = False
     await ledger.init_keysets()
     assert len(ledger.keysets) == 1
 

tests/test_wallet.py

@@ -82,9 +82,6 @@ async def test_get_keys(wallet1: Wallet):
     keyset = keysets[0]
     assert keyset.id is not None
     # assert keyset.id_deprecated == "eGnEWtdJ0PIM"
-    if settings.debug_mint_only_deprecated:
-        assert keyset.id == "eGnEWtdJ0PIM"
-    else:
     assert keyset.id == "009a1f293253e41e"
     assert isinstance(keyset.id, str)
     assert len(keyset.id) > 0
@@ -441,14 +438,10 @@ async def test_token_state(wallet1: Wallet):
 @pytest.mark.asyncio
 async def test_load_mint_keys_specific_keyset(wallet1: Wallet):
     await wallet1._load_mint_keys()
-    if settings.debug_mint_only_deprecated:
+    assert list(wallet1.keysets.keys()) == ["009a1f293253e41e"]
-        assert list(wallet1.keysets.keys()) == ["eGnEWtdJ0PIM"]
-    else:
-        assert list(wallet1.keysets.keys()) == ["009a1f293253e41e", "eGnEWtdJ0PIM"]
     await wallet1._load_mint_keys(keyset_id=wallet1.keyset_id)
     await wallet1._load_mint_keys(keyset_id="009a1f293253e41e")
     # expect deprecated keyset id to be present
-    await wallet1._load_mint_keys(keyset_id="eGnEWtdJ0PIM")
     await assert_err(
         wallet1._load_mint_keys(keyset_id="nonexistent"),
         KeysetNotFoundError(),

tests/test_wallet_restore.py

@@ -8,7 +8,6 @@ import pytest_asyncio
 from cashu.core.base import Proof
 from cashu.core.crypto.secp import PrivateKey
 from cashu.core.errors import CashuError
-from cashu.core.settings import settings
 from cashu.wallet.wallet import Wallet
 from cashu.wallet.wallet import Wallet as Wallet1
 from cashu.wallet.wallet import Wallet as Wallet2
@@ -86,10 +85,6 @@ async def wallet3():
 
 
 @pytest.mark.asyncio
-@pytest.mark.skipif(
-    settings.debug_mint_only_deprecated,
-    reason="settings.debug_mint_only_deprecated is set",
-)
 async def test_bump_secret_derivation(wallet3: Wallet):
     await wallet3._init_private_key(
         "half depart obvious quality work element tank gorilla view sugar picture"