index on db and read spent proofs from db (#370)

* index on db and read spent proofs from db

* add benchmark for testing

* remove benchmark

* add option to disable cached secrets

* disable python 3.9 tests

cashu/mint/ledger.py

@@ -1,6 +1,6 @@
 import asyncio
 import math
-from typing import Dict, List, Optional, Set, Tuple
+from typing import Dict, List, Optional, Tuple
 
 import bolt11
 from loguru import logger
@@ -49,7 +49,6 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerLightning):
         crud: LedgerCrud,
         derivation_path="",
     ):
-        self.secrets_used: Set[str] = set()
         self.master_key = seed
         self.derivation_path = derivation_path
 
@@ -162,9 +161,10 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerLightning):
         # Mark proofs as used and prepare new promises
         secrets = set([p.secret for p in proofs])
         self.secrets_used |= secrets
-        # store in db
-        for p in proofs:
-            await self.crud.invalidate_proof(proof=p, db=self.db)
+        async with self.db.connect() as conn:
+            # store in db
+            for p in proofs:
+                await self.crud.invalidate_proof(proof=p, db=self.db, conn=conn)
 
     async def _generate_change_promises(
         self,
@@ -538,70 +538,59 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerLightning):
     async def _generate_promises(
         self, B_s: List[BlindedMessage], keyset: Optional[MintKeyset] = None
     ) -> list[BlindedSignature]:
-        """Generates promises that sum to the given amount.
+        """Generates a promises (Blind signatures) for given amount and returns a pair (amount, C').
 
         Args:
-            B_s (List[BlindedMessage]): _description_
-            keyset (Optional[MintKeyset], optional): _description_. Defaults to None.
-
-        Returns:
-            list[BlindedSignature]: _description_
-        """
-        return [
-            await self._generate_promise(
-                b.amount, PublicKey(bytes.fromhex(b.B_), raw=True), keyset
-            )
-            for b in B_s
-        ]
-
-    async def _generate_promise(
-        self, amount: int, B_: PublicKey, keyset: Optional[MintKeyset] = None
-    ) -> BlindedSignature:
-        """Generates a promise (Blind signature) for given amount and returns a pair (amount, C').
-
-        Args:
-            amount (int): Amount of the promise.
-            B_ (PublicKey): Blinded secret (point on curve)
+            B_s (List[BlindedMessage]): Blinded secret (point on curve)
             keyset (Optional[MintKeyset], optional): Which keyset to use. Private keys will be taken from this keyset. Defaults to None.
 
         Returns:
-            BlindedSignature: Generated promise.
+            list[BlindedSignature]: Generated BlindedSignatures.
         """
         keyset = keyset if keyset else self.keyset
-        logger.trace(f"Generating promise with keyset {keyset.id}.")
-        private_key_amount = keyset.private_keys[amount]
-        C_, e, s = b_dhke.step2_bob(B_, private_key_amount)
-        logger.trace(f"crud: _generate_promise storing promise for {amount}")
-        await self.crud.store_promise(
-            amount=amount,
-            B_=B_.serialize().hex(),
-            C_=C_.serialize().hex(),
-            e=e.serialize(),
-            s=s.serialize(),
-            db=self.db,
-            id=keyset.id,
-        )
-        logger.trace(f"crud: _generate_promise stored promise for {amount}")
-        return BlindedSignature(
-            id=keyset.id,
-            amount=amount,
-            C_=C_.serialize().hex(),
-            dleq=DLEQ(e=e.serialize(), s=s.serialize()),
-        )
+        promises = []
+        for b in B_s:
+            amount = b.amount
+            B_ = PublicKey(bytes.fromhex(b.B_), raw=True)
+            logger.trace(f"Generating promise with keyset {keyset.id}.")
+            private_key_amount = keyset.private_keys[amount]
+            C_, e, s = b_dhke.step2_bob(B_, private_key_amount)
+            promises.append((B_, amount, C_, e, s))
+
+        signatures = []
+        async with self.db.connect() as conn:
+            for promise in promises:
+                B_, amount, C_, e, s = promise
+                logger.trace(f"crud: _generate_promise storing promise for {amount}")
+                await self.crud.store_promise(
+                    amount=amount,
+                    id=keyset.id,
+                    B_=B_.serialize().hex(),
+                    C_=C_.serialize().hex(),
+                    e=e.serialize(),
+                    s=s.serialize(),
+                    db=self.db,
+                    conn=conn,
+                )
+                logger.trace(f"crud: _generate_promise stored promise for {amount}")
+                signature = BlindedSignature(
+                    id=keyset.id,
+                    amount=amount,
+                    C_=C_.serialize().hex(),
+                    dleq=DLEQ(e=e.serialize(), s=s.serialize()),
+                )
+                signatures.append(signature)
+            return signatures
 
     # ------- PROOFS -------
 
     async def load_used_proofs(self) -> None:
         """Load all used proofs from database."""
-        logger.trace("crud: loading used proofs")
+        logger.debug("Loading used proofs into memory")
         secrets_used = await self.crud.get_secrets_used(db=self.db)
-        logger.trace(f"crud: loaded {len(secrets_used)} used proofs")
+        logger.debug(f"Loaded {len(secrets_used)} used proofs")
         self.secrets_used = set(secrets_used)
 
-    def _check_spendable(self, proof: Proof) -> bool:
-        """Checks whether the proof was already spent."""
-        return proof.secret not in self.secrets_used
-
     async def _check_pending(self, proofs: List[Proof]) -> List[bool]:
         """Checks whether the proof is still pending."""
         proofs_pending = await self.crud.get_proofs_pending(db=self.db)
@@ -628,13 +617,12 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerLightning):
             List[bool]: List of which proof is still spendable (True if still spendable, else False)
             List[bool]: List of which proof are pending (True if pending, else False)
         """
-        spendable = [self._check_spendable(p) for p in proofs]
+
+        spendable = await self._check_proofs_spendable(proofs)
         pending = await self._check_pending(proofs)
         return spendable, pending
 
-    async def _set_proofs_pending(
-        self, proofs: List[Proof], conn: Optional[Connection] = None
-    ) -> None:
+    async def _set_proofs_pending(self, proofs: List[Proof]) -> None:
         """If none of the proofs is in the pending table (_validate_proofs_pending), adds proofs to
         the list of pending proofs or removes them. Used as a mutex for proofs.
 
@@ -646,24 +634,26 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerLightning):
         """
         # first we check whether these proofs are pending already
         async with self.proofs_pending_lock:
-            await self._validate_proofs_pending(proofs, conn)
-            for p in proofs:
-                try:
-                    await self.crud.set_proof_pending(proof=p, db=self.db, conn=conn)
-                except Exception:
-                    raise TransactionError("proofs already pending.")
+            async with self.db.connect() as conn:
+                await self._validate_proofs_pending(proofs, conn)
+                for p in proofs:
+                    try:
+                        await self.crud.set_proof_pending(
+                            proof=p, db=self.db, conn=conn
+                        )
+                    except Exception:
+                        raise TransactionError("proofs already pending.")
 
-    async def _unset_proofs_pending(
-        self, proofs: List[Proof], conn: Optional[Connection] = None
-    ) -> None:
+    async def _unset_proofs_pending(self, proofs: List[Proof]) -> None:
         """Deletes proofs from pending table.
 
         Args:
             proofs (List[Proof]): Proofs to delete.
         """
         async with self.proofs_pending_lock:
-            for p in proofs:
-                await self.crud.unset_proof_pending(proof=p, db=self.db, conn=conn)
+            async with self.db.connect() as conn:
+                for p in proofs:
+                    await self.crud.unset_proof_pending(proof=p, db=self.db, conn=conn)
 
     async def _validate_proofs_pending(
         self, proofs: List[Proof], conn: Optional[Connection] = None