Nutshell cleanup wishlist (#332)

* fix keys

* fix tests

* backwards compatible api upgrade

* upgrade seems to work

* fix tests

* add deprecated api functions

* add more tests of backwards compat

* add test serialization for nut00

* remove a redundant test

* move mint and melt to new api

* mypy works

* CI: mypy --check-untyped-defs

* add deprecated router

* add hints and remove logs

* fix tests

* cleanup

* use new mint and melt endpoints

* tests passing?

* fix mypy

* make format

* make format

* make format

* commit

* errors gone

* save

* adjust the API

* store quotes in db

* make mypy happy

* add fakewallet settings

* remove LIGHTNING=True and pass quote id for melt

* format

* tests passing

* add CoreLightningRestWallet

* add macaroon loader

* add correct config

* preimage -> proof

* move wallet.status() to cli.helpers.print_status()

* remove statuses from tests

* remove

* make format

* Use httpx in deprecated wallet

* fix cln interface

* create invoice before quote

* internal transactions and deprecated api testing

* fix tests

* add deprecated API tests

* fastapi type hints break things

* fix duplicate wallet error

* make format

* update poetry in CI to 1.7.1

* precommit restore

* remove bolt11

* oops

* default poetry

* store fee reserve for melt quotes and refactor melt()

* works?

* make format

* test

* finally

* fix deprecated models

* rename v1 endpoints to bolt11

* raise restore and check to v1, bump version to 0.15.0

* add version byte to keyset id

* remove redundant fields in json

* checks

* generate bip32 keyset wip

* migrate old keysets

* load duplicate keys

* duplicate old keysets

* revert router changes

* add deprecated /check and /restore endpoints

* try except invalidate

* parse unit from derivation path, adjust keyset id calculation with bytes

* remove keyest id from functions again and rely on self.keyset_id

* mosts tests work

* mint loads multiple derivation paths

* make format

* properly print units

* fix tests

* wallet works with multiple units

* add strike wallet and choose backend dynamically

* fix mypy

* add get_payment_quote to lightning backends

* make format

* fix startup

* fix lnbitswallet

* fix tests

* LightningWallet -> LightningBackend

* remove comments

* make format

* remove msat conversion

* add Amount type

* fix regtest

* use melt_quote as argument for pay_invoice

* test old api

* fees in sats

* fix deprecated fees

* fixes

* print balance correctly

* internally index keyset response by int

* add pydantic validation to input models

* add timestamps to mint db

* store timestamps for invoices, promises, proofs_used

* fix wallet migration

* rotate keys correctly for testing

* remove print

* update latest keyset

* fix tests

* fix test

* make format

* make format with correct black version

* remove nsat and cheese

* test against deprecated mint

* fix tests?

* actually use env var

* mint run with env vars

* moar test

* cleanup

* simplify tests, load all keys

* try out testing with internal invoices

* fix internal melt test

* fix test

* deprecated checkfees expects appropriate fees

* adjust comment

* drop lightning table

* split migration for testing for now, remove it later

* remove unused lightning table

* skip_private_key -> skip_db_read

* throw error on migration error

* reorder

* fix migrations

* fix lnbits fee return value negative

* fix typo

* comments

* add type

* make format

* split must use correct amount

* fix tests

* test deprecated api with internal/external melts

* do not split if not necessary

* refactor

* fix test

* make format with new black

* cleanup and add comments

* add quote state check endpoints

* fix deprecated wallet response

* split -> swap endpoint

* make format

* add expiry to quotes, get quote endpoints, and adjust to nut review comments

* allow overpayment of melt

* add lightning wallet tests

* commiting to save

* fix tests a bit

* make format

* remove comments

* get mint info

* check_spendable default False, and return payment quote checking id

* make format

* bump version in pyproject

* update to /v1/checkstate

* make format

* fix mint api checks

* return witness on /v1/checkstate

* no failfast

* try fail-fast: false in ci.yaml

* fix db lookup

* clean up literals

cashu/mint/verification.py

@@ -1,4 +1,4 @@
-from typing import List, Literal, Optional, Set, Union
+from typing import Dict, List, Literal, Optional, Union
 
 from loguru import logger
 
@@ -6,7 +6,6 @@ from ..core.base import (
     BlindedMessage,
     BlindedSignature,
     MintKeyset,
-    MintKeysets,
     Proof,
 )
 from ..core.crypto import b_dhke
@@ -29,20 +28,20 @@ class LedgerVerification(LedgerSpendingConditions, SupportsKeysets, SupportsDb):
     """Verification functions for the ledger."""
 
     keyset: MintKeyset
-    keysets: MintKeysets
-    secrets_used: Set[str] = set()
+    keysets: Dict[str, MintKeyset]
+    spent_proofs: Dict[str, Proof]
     crud: LedgerCrud
     db: Database
 
     async def verify_inputs_and_outputs(
-        self, proofs: List[Proof], outputs: Optional[List[BlindedMessage]] = None
+        self, *, proofs: List[Proof], outputs: Optional[List[BlindedMessage]] = None
     ):
         """Checks all proofs and outputs for validity.
 
         Args:
             proofs (List[Proof]): List of proofs to check.
             outputs (Optional[List[BlindedMessage]], optional): List of outputs to check.
-            Must be provided for /split but not for /melt. Defaults to None.
+            Must be provided for a swap but not for a melt. Defaults to None.
 
         Raises:
             Exception: Scripts did not validate.
@@ -52,8 +51,8 @@ class LedgerVerification(LedgerSpendingConditions, SupportsKeysets, SupportsDb):
         """
         # Verify inputs
         # Verify proofs are spendable
-        spendable = await self._check_proofs_spendable(proofs)
-        if not all(spendable):
+        spent_proofs = await self._get_proofs_spent([p.secret for p in proofs])
+        if not len(spent_proofs) == 0:
             raise TokenAlreadySpentError()
         # Verify amounts of inputs
         if not all([self._verify_amount(p.amount) for p in proofs]):
@@ -83,12 +82,31 @@ class LedgerVerification(LedgerSpendingConditions, SupportsKeysets, SupportsDb):
         # Verify inputs and outputs together
         if not self._verify_input_output_amounts(proofs, outputs):
             raise TransactionError("input amounts less than output.")
+        # Verify that input keyset units are the same as output keyset unit
+        # We have previously verified that all outputs have the same keyset id in `_verify_outputs`
+        assert outputs[0].id, "output id not set"
+        if not all([
+            self.keysets[p.id].unit == self.keysets[outputs[0].id].unit
+            for p in proofs
+            if p.id
+        ]):
+            raise TransactionError("input and output keysets have different units.")
+
         # Verify output spending conditions
         if outputs and not self._verify_output_spending_conditions(proofs, outputs):
             raise TransactionError("validation of output spending conditions failed.")
 
     async def _verify_outputs(self, outputs: List[BlindedMessage]):
         """Verify that the outputs are valid."""
+        logger.trace(f"Verifying {len(outputs)} outputs.")
+        # Verify all outputs have the same keyset id
+        if not all([o.id == outputs[0].id for o in outputs]):
+            raise TransactionError("outputs have different keyset ids.")
+        # Verify that the keyset id is known and active
+        if outputs[0].id not in self.keysets:
+            raise TransactionError("keyset id unknown.")
+        if not self.keysets[outputs[0].id].active:
+            raise TransactionError("keyset id inactive.")
         # Verify amounts of outputs
         if not all([self._verify_amount(o.amount) for o in outputs]):
             raise TransactionError("invalid amount.")
@@ -98,6 +116,7 @@ class LedgerVerification(LedgerSpendingConditions, SupportsKeysets, SupportsDb):
         # verify that outputs have not been signed previously
         if any(await self._check_outputs_issued_before(outputs)):
             raise TransactionError("outputs have already been signed before.")
+        logger.trace(f"Verified {len(outputs)} outputs.")
 
     async def _check_outputs_issued_before(self, outputs: List[BlindedMessage]):
         """Checks whether the provided outputs have previously been signed by the mint
@@ -118,24 +137,32 @@ class LedgerVerification(LedgerSpendingConditions, SupportsKeysets, SupportsDb):
                 result.append(False if promise is None else True)
         return result
 
-    async def _check_proofs_spendable(self, proofs: List[Proof]) -> List[bool]:
-        """Checks whether the proof was already spent."""
-        spendable_states = []
+    async def _get_proofs_pending(self, secrets: List[str]) -> Dict[str, Proof]:
+        """Returns only those proofs that are pending."""
+        all_proofs_pending = await self.crud.get_proofs_pending(db=self.db)
+        proofs_pending = list(filter(lambda p: p.secret in secrets, all_proofs_pending))
+        proofs_pending_dict = {p.secret: p for p in proofs_pending}
+        return proofs_pending_dict
+
+    async def _get_proofs_spent(self, secrets: List[str]) -> Dict[str, Proof]:
+        """Returns all proofs that are spent."""
+        proofs_spent: List[Proof] = []
         if settings.mint_cache_secrets:
             # check used secrets in memory
-            for p in proofs:
-                spendable_state = p.secret not in self.secrets_used
-                spendable_states.append(spendable_state)
+            for secret in secrets:
+                if secret in self.spent_proofs:
+                    proofs_spent.append(self.spent_proofs[secret])
         else:
             # check used secrets in database
             async with self.db.connect() as conn:
-                for p in proofs:
-                    spendable_state = (
-                        await self.crud.get_proof_used(db=self.db, proof=p, conn=conn)
-                        is None
+                for secret in secrets:
+                    spent_proof = await self.crud.get_proof_used(
+                        db=self.db, secret=secret, conn=conn
                     )
-                    spendable_states.append(spendable_state)
-        return spendable_states
+                    if spent_proof:
+                        proofs_spent.append(spent_proof)
+        proofs_spent_dict = {p.secret: p for p in proofs_spent}
+        return proofs_spent_dict
 
     def _verify_secret_criteria(self, proof: Proof) -> Literal[True]:
         """Verifies that a secret is present and is not too long (DOS prevention)."""
@@ -145,23 +172,27 @@ class LedgerVerification(LedgerSpendingConditions, SupportsKeysets, SupportsDb):
             raise SecretTooLongError()
         return True
 
-    def _verify_proof_bdhke(self, proof: Proof):
+    def _verify_proof_bdhke(self, proof: Proof) -> bool:
         """Verifies that the proof of promise was issued by this ledger."""
         # if no keyset id is given in proof, assume the current one
         if not proof.id:
             private_key_amount = self.keyset.private_keys[proof.amount]
         else:
-            assert proof.id in self.keysets.keysets, f"keyset {proof.id} unknown"
+            assert proof.id in self.keysets, f"keyset {proof.id} unknown"
             logger.trace(
-                f"Validating proof with keyset {self.keysets.keysets[proof.id].id}."
+                f"Validating proof {proof.secret} with keyset"
+                f" {self.keysets[proof.id].id}."
             )
             # use the appropriate active keyset for this proof.id
-            private_key_amount = self.keysets.keysets[proof.id].private_keys[
-                proof.amount
-            ]
+            private_key_amount = self.keysets[proof.id].private_keys[proof.amount]
 
         C = PublicKey(bytes.fromhex(proof.C), raw=True)
-        return b_dhke.verify(private_key_amount, C, proof.secret)
+        valid = b_dhke.verify(private_key_amount, C, proof.secret)
+        if valid:
+            logger.trace("Proof verified.")
+        else:
+            logger.trace(f"Proof verification failed for {proof.secret} – {proof.C}.")
+        return valid
 
     def _verify_input_output_amounts(
         self, inputs: List[Proof], outputs: List[BlindedMessage]