From ad7c6b8e0b71e0d8ad4132aad8bb9165f87fe232 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:28:41 -0600 Subject: [PATCH] Issue NUT-08 overpaid Lightning fees for melt quote checks on startup (#688) * startup: do not rollback unknown melt quote states * fix: provide overpaid fees on startup * fix: check if outputs in db * fix test: expect melt quote pending if payment state is unknown * fix up comment --- cashu/core/base.py | 8 ++++++- cashu/mint/crud.py | 14 ++++++++---- cashu/mint/db/write.py | 14 +++++++++--- cashu/mint/ledger.py | 44 ++++++++++++++++++++++++------------ cashu/mint/migrations.py | 13 ++++++++++- cashu/wallet/transactions.py | 4 ++-- tests/test_mint_init.py | 17 ++++++++------ tests/test_mint_melt.py | 4 ++-- 8 files changed, 84 insertions(+), 34 deletions(-) diff --git a/cashu/core/base.py b/cashu/core/base.py index 1d04709..d08680f 100644 --- a/cashu/core/base.py +++ b/cashu/core/base.py @@ -287,6 +287,7 @@ class MeltQuote(LedgerEvent): fee_paid: int = 0 payment_preimage: Optional[str] = None expiry: Optional[int] = None + outputs: Optional[List[BlindedMessage]] = None change: Optional[List[BlindedSignature]] = None mint: Optional[str] = None @@ -307,9 +308,13 @@ class MeltQuote(LedgerEvent): # parse change from row as json change = None - if row["change"]: + if "change" in row.keys() and row["change"]: change = json.loads(row["change"]) + outputs = None + if "outputs" in row.keys() and row["outputs"]: + outputs = json.loads(row["outputs"]) + return cls( quote=row["quote"], method=row["method"], @@ -322,6 +327,7 @@ class MeltQuote(LedgerEvent): created_time=created_time, paid_time=paid_time, fee_paid=row["fee_paid"], + outputs=outputs, change=change, expiry=expiry, payment_preimage=payment_preimage, diff --git a/cashu/mint/crud.py b/cashu/mint/crud.py index 89feb3c..d7b17bd 100644 --- a/cashu/mint/crud.py +++ b/cashu/mint/crud.py @@ -440,7 +440,7 @@ class LedgerCrudSqlite(LedgerCrud): "paid_time": db.to_timestamp( db.timestamp_from_seconds(quote.paid_time) or "" ), - "pubkey": quote.pubkey or "" + "pubkey": quote.pubkey or "", }, ) @@ -522,8 +522,8 @@ class LedgerCrudSqlite(LedgerCrud): await (conn or db).execute( f""" INSERT INTO {db.table_with_schema('melt_quotes')} - (quote, method, request, checking_id, unit, amount, fee_reserve, state, paid, created_time, paid_time, fee_paid, proof, change, expiry) - VALUES (:quote, :method, :request, :checking_id, :unit, :amount, :fee_reserve, :state, :paid, :created_time, :paid_time, :fee_paid, :proof, :change, :expiry) + (quote, method, request, checking_id, unit, amount, fee_reserve, state, paid, created_time, paid_time, fee_paid, proof, outputs, change, expiry) + VALUES (:quote, :method, :request, :checking_id, :unit, :amount, :fee_reserve, :state, :paid, :created_time, :paid_time, :fee_paid, :proof, :outputs, :change, :expiry) """, { "quote": quote.quote, @@ -543,6 +543,7 @@ class LedgerCrudSqlite(LedgerCrud): ), "fee_paid": quote.fee_paid, "proof": quote.payment_preimage, + "outputs": json.dumps(quote.outputs) if quote.outputs else None, "change": json.dumps(quote.change) if quote.change else None, "expiry": db.to_timestamp( db.timestamp_from_seconds(quote.expiry) or "" @@ -607,7 +608,7 @@ class LedgerCrudSqlite(LedgerCrud): ) -> None: await (conn or db).execute( f""" - UPDATE {db.table_with_schema('melt_quotes')} SET state = :state, fee_paid = :fee_paid, paid_time = :paid_time, proof = :proof, change = :change, checking_id = :checking_id WHERE quote = :quote + UPDATE {db.table_with_schema('melt_quotes')} SET state = :state, fee_paid = :fee_paid, paid_time = :paid_time, proof = :proof, outputs = :outputs, change = :change, checking_id = :checking_id WHERE quote = :quote """, { "state": quote.state.value, @@ -616,6 +617,11 @@ class LedgerCrudSqlite(LedgerCrud): db.timestamp_from_seconds(quote.paid_time) or "" ), "proof": quote.payment_preimage, + "outputs": ( + json.dumps([s.dict() for s in quote.outputs]) + if quote.outputs + else None + ), "change": ( json.dumps([s.dict() for s in quote.change]) if quote.change diff --git a/cashu/mint/db/write.py b/cashu/mint/db/write.py index 239b529..1ad13d1 100644 --- a/cashu/mint/db/write.py +++ b/cashu/mint/db/write.py @@ -3,6 +3,7 @@ from typing import List, Optional, Union from loguru import logger from ...core.base import ( + BlindedMessage, MeltQuote, MeltQuoteState, MintQuote, @@ -162,7 +163,7 @@ class DbWriteHelper: raise TransactionError( f"Mint quote not pending: {quote.state.value}. Cannot set as {state.value}." ) - # set the quote as pending + # set the quote to previous state quote.state = state logger.trace(f"crud: setting quote {quote_id} as {state.value}") await self.crud.update_mint_quote(quote=quote, db=self.db, conn=conn) @@ -172,7 +173,9 @@ class DbWriteHelper: await self.events.submit(quote) return quote - async def _set_melt_quote_pending(self, quote: MeltQuote) -> MeltQuote: + async def _set_melt_quote_pending( + self, quote: MeltQuote, outputs: Optional[List[BlindedMessage]] = None + ) -> MeltQuote: """Sets the melt quote as pending. Args: @@ -193,6 +196,8 @@ class DbWriteHelper: raise TransactionError("Melt quote already pending.") # set the quote as pending quote_copy.state = MeltQuoteState.pending + if outputs: + quote_copy.outputs = outputs await self.crud.update_melt_quote(quote=quote_copy, db=self.db, conn=conn) await self.events.submit(quote_copy) @@ -217,8 +222,11 @@ class DbWriteHelper: raise TransactionError("Melt quote not found.") if quote_db.state != MeltQuoteState.pending: raise TransactionError("Melt quote not pending.") - # set the quote as pending + # set the quote to previous state quote_copy.state = state + + # unset outputs + quote_copy.outputs = None await self.crud.update_melt_quote(quote=quote_copy, db=self.db, conn=conn) await self.events.submit(quote_copy) diff --git a/cashu/mint/ledger.py b/cashu/mint/ledger.py index 4bac51e..a070f8d 100644 --- a/cashu/mint/ledger.py +++ b/cashu/mint/ledger.py @@ -144,18 +144,18 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe task.cancel() async def _check_pending_proofs_and_melt_quotes(self): - """Startup routine that checks all pending proofs for their melt state and either invalidates - them for a successful melt or deletes them if the melt failed. + """Startup routine that checks all pending melt quotes and either invalidates + their pending proofs for a successful melt or deletes them if the melt failed. """ # get all pending melt quotes - melt_quotes = await self.crud.get_all_melt_quotes_from_pending_proofs( + pending_melt_quotes = await self.crud.get_all_melt_quotes_from_pending_proofs( db=self.db ) - if not melt_quotes: + if not pending_melt_quotes: return - logger.info("Checking pending melt quotes") - for quote in melt_quotes: - quote = await self.get_melt_quote(quote_id=quote.quote, purge_unknown=True) + logger.info(f"Checking {len(pending_melt_quotes)} pending melt quotes") + for quote in pending_melt_quotes: + quote = await self.get_melt_quote(quote_id=quote.quote) logger.info(f"Melt quote {quote.quote} state: {quote.state}") # ------- KEYS ------- @@ -723,17 +723,17 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe expiry=quote.expiry, ) - async def get_melt_quote(self, quote_id: str, purge_unknown=False) -> MeltQuote: + async def get_melt_quote(self, quote_id: str, rollback_unknown=False) -> MeltQuote: """Returns a melt quote. If the melt quote is pending, checks status of the payment with the backend. - If settled, sets the quote as paid and invalidates pending proofs (commit). - If failed, sets the quote as unpaid and unsets pending proofs (rollback). - - If purge_unknown is set, do the same for unknown states as for failed states. + - If rollback_unknown is set, do the same for unknown states as for failed states. Args: quote_id (str): ID of the melt quote. - purge_unknown (bool, optional): Rollback unknown payment states to unpaid. Defaults to False. + rollback_unknown (bool, optional): Rollback unknown payment states to unpaid. Defaults to False. Raises: Exception: Quote not found. @@ -772,14 +772,28 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe if status.preimage: melt_quote.payment_preimage = status.preimage melt_quote.paid_time = int(time.time()) - await self.crud.update_melt_quote(quote=melt_quote, db=self.db) - await self.events.submit(melt_quote) pending_proofs = await self.crud.get_pending_proofs_for_quote( quote_id=quote_id, db=self.db ) await self._invalidate_proofs(proofs=pending_proofs, quote_id=quote_id) await self.db_write._unset_proofs_pending(pending_proofs) - if status.failed or (purge_unknown and status.unknown): + # change to compensate wallet for overpaid fees + if melt_quote.outputs: + total_provided = sum_proofs(pending_proofs) + input_fees = self.get_fees_for_proofs(pending_proofs) + fee_reserve_provided = ( + total_provided - melt_quote.amount - input_fees + ) + return_promises = await self._generate_change_promises( + fee_provided=fee_reserve_provided, + fee_paid=melt_quote.fee_paid, + outputs=melt_quote.outputs, + keyset=self.keysets[melt_quote.outputs[0].id], + ) + melt_quote.change = return_promises + await self.crud.update_melt_quote(quote=melt_quote, db=self.db) + await self.events.submit(melt_quote) + if status.failed or (rollback_unknown and status.unknown): logger.debug(f"Setting quote {quote_id} as unpaid") melt_quote.state = MeltQuoteState.unpaid await self.crud.update_melt_quote(quote=melt_quote, db=self.db) @@ -909,6 +923,8 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe raise TransactionError( f"output unit {outputs_unit.name} does not match quote unit {melt_quote.unit}" ) + # we don't need to set it here, _set_melt_quote_pending will set it in the db + melt_quote.outputs = outputs # verify that the amount of the input proofs is equal to the amount of the quote total_provided = sum_proofs(proofs) @@ -939,7 +955,7 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe proofs, quote_id=melt_quote.quote ) previous_state = melt_quote.state - melt_quote = await self.db_write._set_melt_quote_pending(melt_quote) + melt_quote = await self.db_write._set_melt_quote_pending(melt_quote, outputs) # if the melt corresponds to an internal mint, mark both as paid melt_quote = await self.melt_mint_settle_internally(melt_quote, proofs) diff --git a/cashu/mint/migrations.py b/cashu/mint/migrations.py index 39c2948..61c7b12 100644 --- a/cashu/mint/migrations.py +++ b/cashu/mint/migrations.py @@ -839,6 +839,7 @@ async def m022_quote_set_states_to_values(db: Database): f"UPDATE {db.table_with_schema('mint_quotes')} SET state = '{mint_quote_states.value}' WHERE state = '{mint_quote_states.name}'" ) + async def m023_add_key_to_mint_quote_table(db: Database): async with db.connect() as conn: await conn.execute( @@ -846,4 +847,14 @@ async def m023_add_key_to_mint_quote_table(db: Database): ALTER TABLE {db.table_with_schema('mint_quotes')} ADD COLUMN pubkey TEXT DEFAULT NULL """ - ) \ No newline at end of file + ) + + +async def m024_add_melt_quote_outputs(db: Database): + async with db.connect() as conn: + await conn.execute( + f""" + ALTER TABLE {db.table_with_schema('melt_quotes')} + ADD COLUMN outputs TEXT DEFAULT NULL + """ + ) diff --git a/cashu/wallet/transactions.py b/cashu/wallet/transactions.py index 0fb83d6..bb7464b 100644 --- a/cashu/wallet/transactions.py +++ b/cashu/wallet/transactions.py @@ -86,9 +86,9 @@ class WalletTransactions(SupportsDb, SupportsKeysets): remainder = amount_to_send selected_proofs = [smaller_proofs[0]] fee_ppk = self.get_fees_for_proofs_ppk(selected_proofs) if include_fees else 0 - logger.debug(f"adding proof: {smaller_proofs[0].amount} – fee: {fee_ppk} ppk") + logger.trace(f"adding proof: {smaller_proofs[0].amount} – fee: {fee_ppk} ppk") remainder -= smaller_proofs[0].amount - fee_ppk / 1000 - logger.debug(f"remainder: {remainder}") + logger.trace(f"remainder: {remainder}") if remainder > 0: logger.trace( f"> selecting more proofs from {amount_summary(smaller_proofs[1:], self.unit)} sum: {sum_proofs(smaller_proofs[1:])} to reach {remainder}" diff --git a/tests/test_mint_init.py b/tests/test_mint_init.py index 3568dcb..514e928 100644 --- a/tests/test_mint_init.py +++ b/tests/test_mint_init.py @@ -239,6 +239,7 @@ async def test_startup_fakewallet_pending_quote_pending(ledger: Ledger): @pytest.mark.asyncio @pytest.mark.skipif(is_regtest, reason="only for fake wallet") async def test_startup_fakewallet_pending_quote_unknown(ledger: Ledger): + # unknown state simulates a failure th check the lightning backend pending_proof, quote = await create_pending_melts(ledger) states = await ledger.db_read.get_proofs_states([pending_proof.Y]) assert states[0].pending @@ -250,11 +251,12 @@ async def test_startup_fakewallet_pending_quote_unknown(ledger: Ledger): melt_quotes = await ledger.crud.get_all_melt_quotes_from_pending_proofs( db=ledger.db ) - assert not melt_quotes + assert melt_quotes + assert melt_quotes[0].state == MeltQuoteState.pending # expect that proofs are still pending states = await ledger.db_read.get_proofs_states([pending_proof.Y]) - assert states[0].unspent + assert states[0].pending @pytest.mark.asyncio @@ -450,18 +452,19 @@ async def test_startup_regtest_pending_quote_unknown(wallet: Wallet, ledger: Led await asyncio.sleep(SLEEP_TIME) - # run startup routinge + # run startup routine await ledger.startup_ledger() - # expect that no melt quote is pending + # expect that melt quote is still pending melt_quotes = await ledger.crud.get_all_melt_quotes_from_pending_proofs( db=ledger.db ) - assert not melt_quotes + assert melt_quotes + assert melt_quotes[0].state == MeltQuoteState.pending - # expect that proofs are unspent + # expect that proofs are pending states = await ledger.db_read.get_proofs_states([p.Y for p in send_proofs]) - assert all([s.unspent for s in states]) + assert all([s.pending for s in states]) # clean up cancel_invoice(preimage_hash=preimage_hash) diff --git a/tests/test_mint_melt.py b/tests/test_mint_melt.py index 562062e..672d2ae 100644 --- a/tests/test_mint_melt.py +++ b/tests/test_mint_melt.py @@ -167,8 +167,8 @@ async def test_fakewallet_pending_quote_get_melt_quote_unknown(ledger: Ledger): assert states[0].pending settings.fakewallet_payment_state = PaymentResult.UNKNOWN.name - # get_melt_quote(..., purge_unknown=True) should check the payment status and update the db - quote2 = await ledger.get_melt_quote(quote_id=quote.quote, purge_unknown=True) + # get_melt_quote(..., rollback_unknown=True) should check the payment status and update the db + quote2 = await ledger.get_melt_quote(quote_id=quote.quote, rollback_unknown=True) assert quote2.state == MeltQuoteState.unpaid # expect that pending tokens are still in db