Fix loading b64 keysets and add option to set b64 inactive in WalletSettings (#579)

* Mint: fix loading b64 keysets and Wallet: option to set b64 inactive

* typo

* readd include fees flag (unused)

* fix test to respect new default False flag

* fix default flag for regtest

cashu/core/base.py

@@ -728,6 +728,13 @@ class MintKeyset:
         assert self.seed, "seed not set"
         assert self.derivation_path, "derivation path not set"
 
+        # BEGIN: BACKWARDS COMPATIBILITY < 0.15.0
+        # we overwrite keyset id only if it isn't already set in the database
+        # loaded from the database. This is to allow for backwards compatibility
+        # with old keysets with new id's and vice versa. This code and successive
+        # `id_in_db or` parts can be removed if there are only new keysets in the mint (> 0.15.0)
+        id_in_db = self.id
+
         if self.version_tuple < (0, 12):
             # WARNING: Broken key derivation for backwards compatibility with < 0.12
             self.private_keys = derive_keys_backwards_compatible_insecure_pre_0_12(
@@ -738,7 +745,7 @@ class MintKeyset:
                 f"WARNING: Using weak key derivation for keyset {self.id} (backwards"
                 " compatibility < 0.12)"
             )
-            self.id = derive_keyset_id_deprecated(self.public_keys)  # type: ignore
+            self.id = id_in_db or derive_keyset_id_deprecated(self.public_keys)  # type: ignore
         elif self.version_tuple < (0, 15):
             self.private_keys = derive_keys_sha256(self.seed, self.derivation_path)
             logger.trace(
@@ -746,11 +753,11 @@ class MintKeyset:
                 " compatibility < 0.15)"
             )
             self.public_keys = derive_pubkeys(self.private_keys)  # type: ignore
-            self.id = derive_keyset_id_deprecated(self.public_keys)  # type: ignore
+            self.id = id_in_db or derive_keyset_id_deprecated(self.public_keys)  # type: ignore
         else:
             self.private_keys = derive_keys(self.seed, self.derivation_path)
             self.public_keys = derive_pubkeys(self.private_keys)  # type: ignore
-            self.id = derive_keyset_id(self.public_keys)  # type: ignore
+            self.id = id_in_db or derive_keyset_id(self.public_keys)  # type: ignore
 
 
 # ------- TOKEN -------

cashu/core/settings.py

@@ -184,6 +184,14 @@ class WalletSettings(CashuSettings):
     wallet_target_amount_count: int = Field(default=3)
 
 
+class WalletFeatures(CashuSettings):
+    wallet_inactivate_legacy_keysets: bool = Field(
+        default=False,
+        title="Inactivate legacy base64 keysets",
+        description="If you turn on this flag, old bas64 keysets will be ignored and the wallet will ony use new keyset versions.",
+    )
+
+
 class LndRestFundingSource(MintSettings):
     mint_lnd_rest_endpoint: Optional[str] = Field(default=None)
     mint_lnd_rest_cert: Optional[str] = Field(default=None)
@@ -218,6 +226,7 @@ class Settings(
     MintSettings,
     MintInformation,
     WalletSettings,
+    WalletFeatures,
     CashuSettings,
 ):
     version: str = Field(default=VERSION)

cashu/wallet/wallet.py

@@ -1,3 +1,4 @@
+import base64
 import copy
 import threading
 import time
@@ -7,6 +8,7 @@ import bolt11
 from bip32 import BIP32
 from loguru import logger
 
+from cashu.core.crypto.keys import derive_keyset_id
 from cashu.core.json_rpc.base import JSONRPCSubscriptionKinds
 
 from ..core.base import (
@@ -210,10 +212,14 @@ class Wallet(
                 await store_keyset(keyset=wallet_keyset, db=self.db)
 
         for mint_keyset in mint_keysets_dict.values():
-            # if the active or the fee attributes have changed, update them in the database
+            # if the active flag changes from active to inactive
+            # or the fee attributes have changed, update them in the database
             if mint_keyset.id in keysets_in_db_dict:
                 changed = False
-                if mint_keyset.active != keysets_in_db_dict[mint_keyset.id].active:
+                if (
+                    not mint_keyset.active
+                    and mint_keyset.active != keysets_in_db_dict[mint_keyset.id].active
+                ):
                     keysets_in_db_dict[mint_keyset.id].active = mint_keyset.active
                     changed = True
                 if (
@@ -230,6 +236,37 @@ class Wallet(
                         keyset=keysets_in_db_dict[mint_keyset.id], db=self.db
                     )
 
+        # BEGIN backwards compatibility: phase out keysets with base64 ID by treating them as inactive
+        if settings.wallet_inactivate_legacy_keysets:
+            keysets_in_db = await get_keysets(mint_url=self.url, db=self.db)
+            for keyset in keysets_in_db:
+                if not keyset.active:
+                    continue
+                # test if the keyset id is a hex string, if not it's base64
+                try:
+                    int(keyset.id, 16)
+                except ValueError:
+                    # verify that it's base64
+                    try:
+                        _ = base64.b64decode(keyset.id)
+                    except ValueError:
+                        logger.error("Unexpected: keyset id is neither hex nor base64.")
+                        continue
+
+                    # verify that we have a hex version of the same keyset by comparing public keys
+                    hex_keyset_id = derive_keyset_id(keys=keyset.public_keys)
+                    if hex_keyset_id not in [k.id for k in keysets_in_db]:
+                        logger.warning(
+                            f"Keyset {keyset.id} is base64 but we don't have a hex version. Ignoring."
+                        )
+                        continue
+
+                    logger.warning(
+                        f"Keyset {keyset.id} is base64 and has a hex counterpart, setting inactive."
+                    )
+                    keyset.active = False
+                    await update_keyset(keyset=keyset, db=self.db)
+
         await self.load_keysets_from_db()
 
     async def activate_keyset(self, keyset_id: Optional[str] = None) -> None:
@@ -973,7 +1010,7 @@ class Wallet(
         *,
         set_reserved: bool = False,
         offline: bool = False,
-        include_fees: bool = True,
+        include_fees: bool = False,
     ) -> Tuple[List[Proof], int]:
         """
         Selects proofs such that a desired `amount` can be sent. If the offline coin selection is unsuccessful,
@@ -986,7 +1023,9 @@ class Wallet(
         Args:
             proofs (List[Proof]): Proofs to split
             amount (int): Amount to split to
-            set_reserved (bool, optional): If set, the proofs are marked as reserved.
+            set_reserved (bool, optional): If set, the proofs are marked as reserved. Defaults to False.
+            offline (bool, optional): If set, the coin selection is done offline. Defaults to False.
+            include_fees (bool, optional): If set, the fees are included in the amount to be selected. Defaults to False.
 
         Returns:
             List[Proof]: Proofs to send

tests/test_mint_fees.py

@@ -192,7 +192,7 @@ async def test_melt_internal(wallet1: Wallet, ledger: Ledger):
     assert not melt_quote_pre_payment.paid, "melt quote should not be paid"
 
     # let's first try to melt without enough funds
-    send_proofs, fees = await wallet1.select_to_send(wallet1.proofs, 63)
+    send_proofs, fees = await wallet1.select_to_send(wallet1.proofs, 64)
     # this should fail because we need 64 + 1 sat fees
     assert sum_proofs(send_proofs) == 64
     await assert_err(
@@ -201,7 +201,9 @@ async def test_melt_internal(wallet1: Wallet, ledger: Ledger):
     )
 
     # the wallet respects the fees for coin selection
-    send_proofs, fees = await wallet1.select_to_send(wallet1.proofs, 64)
+    send_proofs, fees = await wallet1.select_to_send(
+        wallet1.proofs, 64, include_fees=True
+    )
     # includes 1 sat fees
     assert sum_proofs(send_proofs) == 65
     await ledger.melt(proofs=send_proofs, quote=melt_quote.quote)
@@ -227,7 +229,9 @@ async def test_melt_external_with_fees(wallet1: Wallet, ledger: Ledger):
 
     mint_quote = await wallet1.melt_quote(invoice_payment_request)
     total_amount = mint_quote.amount + mint_quote.fee_reserve
-    send_proofs, fee = await wallet1.select_to_send(wallet1.proofs, total_amount)
+    send_proofs, fee = await wallet1.select_to_send(
+        wallet1.proofs, total_amount, include_fees=True
+    )
     melt_quote = await ledger.melt_quote(
         PostMeltQuoteRequest(request=invoice_payment_request, unit="sat")
     )