Issue NUT-08 overpaid Lightning fees for melt quote checks on startup (#688)

* startup: do not rollback unknown melt quote states

* fix: provide overpaid fees on startup

* fix: check if outputs in db

* fix test: expect melt quote pending if payment state is unknown

* fix up comment

cashu/core/base.py

@@ -287,6 +287,7 @@ class MeltQuote(LedgerEvent):
     fee_paid: int = 0
     payment_preimage: Optional[str] = None
     expiry: Optional[int] = None
+    outputs: Optional[List[BlindedMessage]] = None
     change: Optional[List[BlindedSignature]] = None
     mint: Optional[str] = None
 
@@ -307,9 +308,13 @@ class MeltQuote(LedgerEvent):
 
         # parse change from row as json
         change = None
-        if row["change"]:
+        if "change" in row.keys() and row["change"]:
             change = json.loads(row["change"])
 
+        outputs = None
+        if "outputs" in row.keys() and row["outputs"]:
+            outputs = json.loads(row["outputs"])
+
         return cls(
             quote=row["quote"],
             method=row["method"],
@@ -322,6 +327,7 @@ class MeltQuote(LedgerEvent):
             created_time=created_time,
             paid_time=paid_time,
             fee_paid=row["fee_paid"],
+            outputs=outputs,
             change=change,
             expiry=expiry,
             payment_preimage=payment_preimage,

cashu/mint/crud.py

@@ -440,7 +440,7 @@ class LedgerCrudSqlite(LedgerCrud):
                 "paid_time": db.to_timestamp(
                     db.timestamp_from_seconds(quote.paid_time) or ""
                 ),
-                "pubkey": quote.pubkey or ""
+                "pubkey": quote.pubkey or "",
             },
         )
 
@@ -522,8 +522,8 @@ class LedgerCrudSqlite(LedgerCrud):
         await (conn or db).execute(
             f"""
             INSERT INTO {db.table_with_schema('melt_quotes')}
-            (quote, method, request, checking_id, unit, amount, fee_reserve, state, paid, created_time, paid_time, fee_paid, proof, change, expiry)
+            (quote, method, request, checking_id, unit, amount, fee_reserve, state, paid, created_time, paid_time, fee_paid, proof, outputs, change, expiry)
-            VALUES (:quote, :method, :request, :checking_id, :unit, :amount, :fee_reserve, :state, :paid, :created_time, :paid_time, :fee_paid, :proof, :change, :expiry)
+            VALUES (:quote, :method, :request, :checking_id, :unit, :amount, :fee_reserve, :state, :paid, :created_time, :paid_time, :fee_paid, :proof, :outputs, :change, :expiry)
             """,
             {
                 "quote": quote.quote,
@@ -543,6 +543,7 @@ class LedgerCrudSqlite(LedgerCrud):
                 ),
                 "fee_paid": quote.fee_paid,
                 "proof": quote.payment_preimage,
+                "outputs": json.dumps(quote.outputs) if quote.outputs else None,
                 "change": json.dumps(quote.change) if quote.change else None,
                 "expiry": db.to_timestamp(
                     db.timestamp_from_seconds(quote.expiry) or ""
@@ -607,7 +608,7 @@ class LedgerCrudSqlite(LedgerCrud):
     ) -> None:
         await (conn or db).execute(
             f"""
-            UPDATE {db.table_with_schema('melt_quotes')} SET state = :state, fee_paid = :fee_paid, paid_time = :paid_time, proof = :proof, change = :change, checking_id = :checking_id WHERE quote = :quote
+            UPDATE {db.table_with_schema('melt_quotes')} SET state = :state, fee_paid = :fee_paid, paid_time = :paid_time, proof = :proof, outputs = :outputs, change = :change, checking_id = :checking_id WHERE quote = :quote
             """,
             {
                 "state": quote.state.value,
@@ -616,6 +617,11 @@ class LedgerCrudSqlite(LedgerCrud):
                     db.timestamp_from_seconds(quote.paid_time) or ""
                 ),
                 "proof": quote.payment_preimage,
+                "outputs": (
+                    json.dumps([s.dict() for s in quote.outputs])
+                    if quote.outputs
+                    else None
+                ),
                 "change": (
                     json.dumps([s.dict() for s in quote.change])
                     if quote.change

cashu/mint/db/write.py

@@ -3,6 +3,7 @@ from typing import List, Optional, Union
 from loguru import logger
 
 from ...core.base import (
+    BlindedMessage,
     MeltQuote,
     MeltQuoteState,
     MintQuote,
@@ -162,7 +163,7 @@ class DbWriteHelper:
                 raise TransactionError(
                     f"Mint quote not pending: {quote.state.value}. Cannot set as {state.value}."
                 )
-            # set the quote as pending
+            # set the quote to previous state
             quote.state = state
             logger.trace(f"crud: setting quote {quote_id} as {state.value}")
             await self.crud.update_mint_quote(quote=quote, db=self.db, conn=conn)
@@ -172,7 +173,9 @@ class DbWriteHelper:
         await self.events.submit(quote)
         return quote
 
-    async def _set_melt_quote_pending(self, quote: MeltQuote) -> MeltQuote:
+    async def _set_melt_quote_pending(
+        self, quote: MeltQuote, outputs: Optional[List[BlindedMessage]] = None
+    ) -> MeltQuote:
         """Sets the melt quote as pending.
 
         Args:
@@ -193,6 +196,8 @@ class DbWriteHelper:
                 raise TransactionError("Melt quote already pending.")
             # set the quote as pending
             quote_copy.state = MeltQuoteState.pending
+            if outputs:
+                quote_copy.outputs = outputs
             await self.crud.update_melt_quote(quote=quote_copy, db=self.db, conn=conn)
 
         await self.events.submit(quote_copy)
@@ -217,8 +222,11 @@ class DbWriteHelper:
                 raise TransactionError("Melt quote not found.")
             if quote_db.state != MeltQuoteState.pending:
                 raise TransactionError("Melt quote not pending.")
-            # set the quote as pending
+            # set the quote to previous state
             quote_copy.state = state
+
+            # unset outputs
+            quote_copy.outputs = None
             await self.crud.update_melt_quote(quote=quote_copy, db=self.db, conn=conn)
 
         await self.events.submit(quote_copy)

cashu/mint/ledger.py

@@ -144,18 +144,18 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
             task.cancel()
 
     async def _check_pending_proofs_and_melt_quotes(self):
-        """Startup routine that checks all pending proofs for their melt state and either invalidates
+        """Startup routine that checks all pending melt quotes and either invalidates
-        them for a successful melt or deletes them if the melt failed.
+        their pending proofs for a successful melt or deletes them if the melt failed.
         """
         # get all pending melt quotes
-        melt_quotes = await self.crud.get_all_melt_quotes_from_pending_proofs(
+        pending_melt_quotes = await self.crud.get_all_melt_quotes_from_pending_proofs(
             db=self.db
         )
-        if not melt_quotes:
+        if not pending_melt_quotes:
             return
-        logger.info("Checking pending melt quotes")
+        logger.info(f"Checking {len(pending_melt_quotes)} pending melt quotes")
-        for quote in melt_quotes:
+        for quote in pending_melt_quotes:
-            quote = await self.get_melt_quote(quote_id=quote.quote, purge_unknown=True)
+            quote = await self.get_melt_quote(quote_id=quote.quote)
             logger.info(f"Melt quote {quote.quote} state: {quote.state}")
 
     # ------- KEYS -------
@@ -723,17 +723,17 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
             expiry=quote.expiry,
         )
 
-    async def get_melt_quote(self, quote_id: str, purge_unknown=False) -> MeltQuote:
+    async def get_melt_quote(self, quote_id: str, rollback_unknown=False) -> MeltQuote:
         """Returns a melt quote.
 
         If the melt quote is pending, checks status of the payment with the backend.
             - If settled, sets the quote as paid and invalidates pending proofs (commit).
             - If failed, sets the quote as unpaid and unsets pending proofs (rollback).
-            - If purge_unknown is set, do the same for unknown states as for failed states.
+            - If rollback_unknown is set, do the same for unknown states as for failed states.
 
         Args:
             quote_id (str): ID of the melt quote.
-            purge_unknown (bool, optional): Rollback unknown payment states to unpaid. Defaults to False.
+            rollback_unknown (bool, optional): Rollback unknown payment states to unpaid. Defaults to False.
 
         Raises:
             Exception: Quote not found.
@@ -772,14 +772,28 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
                 if status.preimage:
                     melt_quote.payment_preimage = status.preimage
                 melt_quote.paid_time = int(time.time())
-                await self.crud.update_melt_quote(quote=melt_quote, db=self.db)
-                await self.events.submit(melt_quote)
                 pending_proofs = await self.crud.get_pending_proofs_for_quote(
                     quote_id=quote_id, db=self.db
                 )
                 await self._invalidate_proofs(proofs=pending_proofs, quote_id=quote_id)
                 await self.db_write._unset_proofs_pending(pending_proofs)
-            if status.failed or (purge_unknown and status.unknown):
+                # change to compensate wallet for overpaid fees
+                if melt_quote.outputs:
+                    total_provided = sum_proofs(pending_proofs)
+                    input_fees = self.get_fees_for_proofs(pending_proofs)
+                    fee_reserve_provided = (
+                        total_provided - melt_quote.amount - input_fees
+                    )
+                    return_promises = await self._generate_change_promises(
+                        fee_provided=fee_reserve_provided,
+                        fee_paid=melt_quote.fee_paid,
+                        outputs=melt_quote.outputs,
+                        keyset=self.keysets[melt_quote.outputs[0].id],
+                    )
+                    melt_quote.change = return_promises
+                await self.crud.update_melt_quote(quote=melt_quote, db=self.db)
+                await self.events.submit(melt_quote)
+            if status.failed or (rollback_unknown and status.unknown):
                 logger.debug(f"Setting quote {quote_id} as unpaid")
                 melt_quote.state = MeltQuoteState.unpaid
                 await self.crud.update_melt_quote(quote=melt_quote, db=self.db)
@@ -909,6 +923,8 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
                 raise TransactionError(
                     f"output unit {outputs_unit.name} does not match quote unit {melt_quote.unit}"
                 )
+            # we don't need to set it here, _set_melt_quote_pending will set it in the db
+            melt_quote.outputs = outputs
 
         # verify that the amount of the input proofs is equal to the amount of the quote
         total_provided = sum_proofs(proofs)
@@ -939,7 +955,7 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
             proofs, quote_id=melt_quote.quote
         )
         previous_state = melt_quote.state
-        melt_quote = await self.db_write._set_melt_quote_pending(melt_quote)
+        melt_quote = await self.db_write._set_melt_quote_pending(melt_quote, outputs)
 
         # if the melt corresponds to an internal mint, mark both as paid
         melt_quote = await self.melt_mint_settle_internally(melt_quote, proofs)

cashu/mint/migrations.py

@@ -839,6 +839,7 @@ async def m022_quote_set_states_to_values(db: Database):
                 f"UPDATE {db.table_with_schema('mint_quotes')} SET state = '{mint_quote_states.value}' WHERE state = '{mint_quote_states.name}'"
             )
 
+
 async def m023_add_key_to_mint_quote_table(db: Database):
     async with db.connect() as conn:
         await conn.execute(
@@ -847,3 +848,13 @@ async def m023_add_key_to_mint_quote_table(db: Database):
                 ADD COLUMN pubkey TEXT DEFAULT NULL
             """
         )
+
+
+async def m024_add_melt_quote_outputs(db: Database):
+    async with db.connect() as conn:
+        await conn.execute(
+            f"""
+                ALTER TABLE {db.table_with_schema('melt_quotes')}
+                ADD COLUMN outputs TEXT DEFAULT NULL
+            """
+        )

cashu/wallet/transactions.py

@@ -86,9 +86,9 @@ class WalletTransactions(SupportsDb, SupportsKeysets):
         remainder = amount_to_send
         selected_proofs = [smaller_proofs[0]]
         fee_ppk = self.get_fees_for_proofs_ppk(selected_proofs) if include_fees else 0
-        logger.debug(f"adding proof: {smaller_proofs[0].amount} – fee: {fee_ppk} ppk")
+        logger.trace(f"adding proof: {smaller_proofs[0].amount} – fee: {fee_ppk} ppk")
         remainder -= smaller_proofs[0].amount - fee_ppk / 1000
-        logger.debug(f"remainder: {remainder}")
+        logger.trace(f"remainder: {remainder}")
         if remainder > 0:
             logger.trace(
                 f"> selecting more proofs from {amount_summary(smaller_proofs[1:], self.unit)} sum: {sum_proofs(smaller_proofs[1:])} to reach {remainder}"

tests/test_mint_init.py

@@ -239,6 +239,7 @@ async def test_startup_fakewallet_pending_quote_pending(ledger: Ledger):
 @pytest.mark.asyncio
 @pytest.mark.skipif(is_regtest, reason="only for fake wallet")
 async def test_startup_fakewallet_pending_quote_unknown(ledger: Ledger):
+    # unknown state simulates a failure th check the lightning backend
     pending_proof, quote = await create_pending_melts(ledger)
     states = await ledger.db_read.get_proofs_states([pending_proof.Y])
     assert states[0].pending
@@ -250,11 +251,12 @@ async def test_startup_fakewallet_pending_quote_unknown(ledger: Ledger):
     melt_quotes = await ledger.crud.get_all_melt_quotes_from_pending_proofs(
         db=ledger.db
     )
-    assert not melt_quotes
+    assert melt_quotes
+    assert melt_quotes[0].state == MeltQuoteState.pending
 
     # expect that proofs are still pending
     states = await ledger.db_read.get_proofs_states([pending_proof.Y])
-    assert states[0].unspent
+    assert states[0].pending
 
 
 @pytest.mark.asyncio
@@ -450,18 +452,19 @@ async def test_startup_regtest_pending_quote_unknown(wallet: Wallet, ledger: Led
 
     await asyncio.sleep(SLEEP_TIME)
 
-    # run startup routinge
+    # run startup routine
     await ledger.startup_ledger()
 
-    # expect that no melt quote is pending
+    # expect that melt quote is still pending
     melt_quotes = await ledger.crud.get_all_melt_quotes_from_pending_proofs(
         db=ledger.db
     )
-    assert not melt_quotes
+    assert melt_quotes
+    assert melt_quotes[0].state == MeltQuoteState.pending
 
-    # expect that proofs are unspent
+    # expect that proofs are pending
     states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs])
-    assert all([s.unspent for s in states])
+    assert all([s.pending for s in states])
 
     # clean up
     cancel_invoice(preimage_hash=preimage_hash)

tests/test_mint_melt.py

@@ -167,8 +167,8 @@ async def test_fakewallet_pending_quote_get_melt_quote_unknown(ledger: Ledger):
     assert states[0].pending
     settings.fakewallet_payment_state = PaymentResult.UNKNOWN.name
 
-    # get_melt_quote(..., purge_unknown=True) should check the payment status and update the db
+    # get_melt_quote(..., rollback_unknown=True) should check the payment status and update the db
-    quote2 = await ledger.get_melt_quote(quote_id=quote.quote, purge_unknown=True)
+    quote2 = await ledger.get_melt_quote(quote_id=quote.quote, rollback_unknown=True)
     assert quote2.state == MeltQuoteState.unpaid
 
     # expect that pending tokens are still in db