Halt melt on exception (#635)

* halt melt for payment status exception

* refactor

* less log

* add fakewallet test throwing exceptions
This commit is contained in:
callebtc
2024-10-06 17:51:54 +02:00
committed by GitHub
parent c5ccf65e4d
commit 307d2d7a98
4 changed files with 153 additions and 107 deletions

View File

@@ -140,7 +140,9 @@ class FakeWalletSettings(MintSettings):
fakewallet_delay_incoming_payment: Optional[float] = Field(default=3.0)
fakewallet_stochastic_invoice: bool = Field(default=False)
fakewallet_payment_state: Optional[str] = Field(default="SETTLED")
fakewallet_payment_state_exception: Optional[bool] = Field(default=False)
fakewallet_pay_invoice_state: Optional[str] = Field(default="SETTLED")
fakewallet_pay_invoice_state_exception: Optional[bool] = Field(default=False)
class MintInformation(CashuSettings):

View File

@@ -147,6 +147,9 @@ class FakeWallet(LightningBackend):
)
async def pay_invoice(self, quote: MeltQuote, fee_limit: int) -> PaymentResponse:
if settings.fakewallet_pay_invoice_state_exception:
raise Exception("FakeWallet pay_invoice exception")
invoice = decode(quote.request)
if settings.fakewallet_delay_outgoing_payment:
@@ -189,6 +192,8 @@ class FakeWallet(LightningBackend):
)
async def get_payment_status(self, checking_id: str) -> PaymentStatus:
if settings.fakewallet_payment_state_exception:
raise Exception("FakeWallet get_payment_status exception")
if settings.fakewallet_payment_state:
return PaymentStatus(
result=PaymentResult[settings.fakewallet_payment_state]

View File

@@ -68,11 +68,11 @@ from .verification import LedgerVerification
class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFeatures):
backends: Mapping[Method, Mapping[Unit, LightningBackend]] = {}
locks: Dict[str, asyncio.Lock] = {} # holds multiprocessing locks
keysets: Dict[str, MintKeyset] = {}
events = LedgerEventManager()
db_read: DbReadHelper
invoice_listener_tasks: List[asyncio.Task] = []
disable_melt: bool = False
def __init__(
self,
@@ -130,8 +130,7 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
logger.error(
"The backend for"
f" {self.backends[method][unit].__class__.__name__} isn't"
f" working properly: '{status.error_message}'",
RuntimeWarning,
f" working properly: '{status.error_message}'"
)
exit(1)
logger.info(f"Backend balance: {status.balance} {unit.name}")
@@ -880,6 +879,10 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
Returns:
Tuple[str, List[BlindedMessage]]: Proof of payment and signed outputs for returning overpaid fees to wallet.
"""
# make sure we're allowed to melt
if self.disable_melt:
raise NotAllowedError("Melt is disabled. Please contact the operator.")
# get melt quote and check if it was already paid
melt_quote = await self.get_melt_quote(quote_id=quote)
if not melt_quote.unpaid:
@@ -913,16 +916,14 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
raise TransactionError(
f"not enough fee reserve provided for melt. Provided fee reserve: {fee_reserve_provided}, needed: {melt_quote.fee_reserve}"
)
# verify that the amount of the proofs is not larger than the maximum allowed
if settings.mint_max_peg_out and total_provided > settings.mint_max_peg_out:
raise NotAllowedError(
f"Maximum melt amount is {settings.mint_max_peg_out} sat."
)
# verify inputs and their spending conditions
# note, we do not verify outputs here, as they are only used for returning overpaid fees
# we should have used _verify_outputs here already (see above)
# We must have called _verify_outputs here already! (see above)
await self.verify_inputs_and_outputs(proofs=proofs)
# set proofs to pending to avoid race conditions
@@ -931,7 +932,7 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
)
previous_state = melt_quote.state
melt_quote = await self.db_write._set_melt_quote_pending(melt_quote)
try:
# if the melt corresponds to an internal mint, mark both as paid
melt_quote = await self.melt_mint_settle_internally(melt_quote, proofs)
# quote not paid yet (not internal), pay it with the backend
@@ -942,7 +943,7 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
melt_quote, melt_quote.fee_reserve * 1000
)
logger.debug(
f"Melt Result: {payment.result}: preimage: {payment.preimage},"
f"Melt Result: {payment.result.name}: preimage: {payment.preimage},"
f" fee: {payment.fee.str() if payment.fee is not None else 'None'}"
)
if (
@@ -954,7 +955,6 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
)
melt_quote.checking_id = payment.checking_id
await self.crud.update_melt_quote(quote=melt_quote, db=self.db)
except Exception as e:
logger.error(f"Exception during pay_invoice: {e}")
payment = PaymentResponse(
@@ -967,29 +967,40 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
# explicitly check payment status for failed or unknown payment states
checking_id = payment.checking_id or melt_quote.checking_id
logger.debug(
f"Payment state is {payment.result}. Checking status for {checking_id}"
f"Payment state is {payment.result.name}.{' Error: ' + payment.error_message + '.' if payment.error_message else ''} Checking status for {checking_id}."
)
try:
status = await self.backends[method][
unit
].get_payment_status(checking_id)
except Exception as e:
# Something went wrong, better to keep the proofs in pending state
logger.error(
f"Lightning backend error: could not check payment status. Proofs for melt quote {melt_quote.quote} are stuck as PENDING. Error: {e}"
status = await self.backends[method][unit].get_payment_status(
checking_id
)
except Exception as e:
# Something went wrong. We might have lost connection to the backend. Keep transaction pending and return.
logger.error(
f"Lightning backend error: could not check payment status. Proofs for melt quote {melt_quote.quote} are stuck as PENDING. Disabling melt. Fix your Lightning backend and restart the mint.\nError: {e}"
)
self.disable_melt = True
return PostMeltQuoteResponse.from_melt_quote(melt_quote)
match status.result:
case PaymentResult.FAILED | PaymentResult.UNKNOWN:
# NOTE: We only throw a payment error if the payment AND a subsequent status check failed
# Everything as expected. Payment AND a status check both agree on a failure. We roll back the transaction.
await self.db_write._unset_proofs_pending(proofs)
await self.db_write._unset_melt_quote_pending(
quote=melt_quote, state=previous_state
)
if status.error_message:
logger.error(
f"Status check error: {status.error_message}"
)
raise LightningError(
f"Lightning payment failed: {payment.error_message}. Error: {status.error_message}"
f"Lightning payment failed{': ' + payment.error_message if payment.error_message else ''}."
)
case _:
# Something went wrong with our implementation or the backend. Status check returned different result than payment. Keep transaction pending and return.
logger.error(
f"Payment state is {status.result} and payment was {payment.result}. Proofs for melt quote {melt_quote.quote} are stuck as PENDING."
f"Payment state is {status.result.name} and payment was {payment.result}. Proofs for melt quote {melt_quote.quote} are stuck as PENDING. Disabling melt. Fix your Lightning backend and restart the mint."
)
self.disable_melt = True
return PostMeltQuoteResponse.from_melt_quote(melt_quote)
case PaymentResult.SETTLED:
@@ -1003,15 +1014,15 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
# set quote as paid
melt_quote.state = MeltQuoteState.paid
melt_quote.paid_time = int(time.time())
# NOTE: This is the only return point for a successful payment
# NOTE: This is the only branch for a successful payment
case PaymentResult.PENDING | _:
logger.debug(
f"Lightning payment is pending: {payment.checking_id}"
f"Lightning payment is {payment.result.name}: {payment.checking_id}"
)
return PostMeltQuoteResponse.from_melt_quote(melt_quote)
# melt successful, invalidate proofs
# melt was successful (either internal or via backend), invalidate proofs
await self._invalidate_proofs(proofs=proofs, quote_id=melt_quote.quote)
await self.db_write._unset_proofs_pending(proofs)
@@ -1032,14 +1043,6 @@ class Ledger(LedgerVerification, LedgerSpendingConditions, LedgerTasks, LedgerFe
return PostMeltQuoteResponse.from_melt_quote(melt_quote)
except Exception as e:
logger.trace(f"Payment has failed: {e}")
await self.db_write._unset_proofs_pending(proofs)
await self.db_write._unset_melt_quote_pending(
quote=melt_quote, state=previous_state
)
raise e
async def swap(
self,
*,

View File

@@ -295,3 +295,39 @@ async def test_melt_lightning_pay_invoice_failed_pending(
# expect that proofs are pending
states = await ledger.db_read.get_proofs_states([p.Y for p in wallet.proofs])
assert all([s.pending for s in states])
@pytest.mark.asyncio
@pytest.mark.skipif(is_regtest, reason="only fake wallet")
async def test_melt_lightning_pay_invoice_exception_exception(
ledger: Ledger, wallet: Wallet
):
"""Simulates the case where pay_invoice and get_payment_status raise an exception (due to network issues for example)."""
invoice = await wallet.request_mint(64)
await ledger.get_mint_quote(invoice.id) # fakewallet: set the quote to paid
await wallet.mint(64, id=invoice.id)
# invoice_64_sat = "lnbcrt640n1pn0r3tfpp5e30xac756gvd26cn3tgsh8ug6ct555zrvl7vsnma5cwp4g7auq5qdqqcqzzsxqyz5vqsp5xfhtzg0y3mekv6nsdnj43c346smh036t4f8gcfa2zwpxzwcryqvs9qxpqysgqw5juev8y3zxpdu0mvdrced5c6a852f9x7uh57g6fgjgcg5muqzd5474d7xgh770frazel67eejfwelnyr507q46hxqehala880rhlqspw07ta0"
invoice_62_sat = "lnbcrt620n1pn0r3vepp5zljn7g09fsyeahl4rnhuy0xax2puhua5r3gspt7ttlfrley6valqdqqcqzzsxqyz5vqsp577h763sel3q06tfnfe75kvwn5pxn344sd5vnays65f9wfgx4fpzq9qxpqysgqg3re9afz9rwwalytec04pdhf9mvh3e2k4r877tw7dr4g0fvzf9sny5nlfggdy6nduy2dytn06w50ls34qfldgsj37x0ymxam0a687mspp0ytr8"
quote_id = (
await ledger.melt_quote(
PostMeltQuoteRequest(unit="sat", request=invoice_62_sat)
)
).quote
# quote = await ledger.get_melt_quote(quote_id)
settings.fakewallet_payment_state_exception = True
settings.fakewallet_pay_invoice_state_exception = True
# we expect a pending melt quote because something has gone wrong (for example has lost connection to backend)
resp = await ledger.melt(proofs=wallet.proofs, quote=quote_id)
assert resp.state == MeltQuoteState.pending.value
# the mint should be locked now and not allow any other melts until it is restarted
quote_id = (
await ledger.melt_quote(
PostMeltQuoteRequest(unit="sat", request=invoice_62_sat)
)
).quote
await assert_err(
ledger.melt(proofs=wallet.proofs, quote=quote_id),
"Melt is disabled. Please contact the operator.",
)