[PATCH] LND use_mission_control + exclude failing channels (#738)

* lnd_grpc multinut patch

* lndrest multinut patch

* mypy fixes

* fixes non escaped double quotes in error messages formats

* fix

* fix debug log with correct hops number

* correctly escape "hops"

* remove `ignored_pairs` constraint

* Apply suggestions from code review

change some error logs to debug

* add tests and some cleanup

---------

Co-authored-by: callebtc <93376500+callebtc@users.noreply.github.com>

cashu/lightning/lnd_grpc/lnd_grpc.py

@@ -46,6 +46,8 @@ INVOICE_RESULT_MAP = {
     lnrpc.Invoice.InvoiceState.ACCEPTED: PaymentResult.PENDING,
 }
 
+MAX_ROUTE_RETRIES = 50
+
 
 class LndRPCWallet(LightningBackend):
     supports_mpp = settings.mint_lnd_enable_mpp
@@ -206,6 +208,8 @@ class LndRPCWallet(LightningBackend):
     async def pay_partial_invoice(
         self, quote: MeltQuote, amount: Amount, fee_limit_msat: int
     ) -> PaymentResponse:
+        total_attempts = 0
+
         # set the fee limit for the payment
         feelimit = lnrpc.FeeLimit(fixed_msat=fee_limit_msat)
         invoice = bolt11.decode(quote.request)
@@ -223,46 +227,58 @@ class LndRPCWallet(LightningBackend):
         payer_addr = str(payer_addr_tag.data)
 
         # get the route
-        r = None
+        response: Optional[lnrpc.HTLCAttempt] = None  # type: ignore
+        route: Optional[lnrpc.Rou] = None  # type: ignore
         try:
             async with grpc.aio.secure_channel(
                 self.endpoint, self.combined_creds
             ) as channel:
                 lnstub = lightningstub.LightningStub(channel)
                 router_stub = routerstub.RouterStub(channel)
-                r = await lnstub.QueryRoutes(
-                    lnrpc.QueryRoutesRequest(
-                        pub_key=pubkey,
-                        amt=amount.to(Unit.sat).amount,
-                        fee_limit=feelimit,
+
+                for attempt in range(MAX_ROUTE_RETRIES):
+                    total_attempts += 1
+                    route = await lnstub.QueryRoutes(
+                        lnrpc.QueryRoutesRequest(
+                            pub_key=pubkey,
+                            amt=amount.to(Unit.sat).amount,
+                            fee_limit=feelimit,
+                            use_mission_control=True,
+                        )
                     )
-                )
-                """
-                # We need to set the mpp_record for a partial payment
-                mpp_record = lnrpc.MPPRecord(
-                    payment_addr=bytes.fromhex(payer_addr),
-                    total_amt_msat=total_amount_msat,
-                )
-                """
-                # modify the mpp_record in the last hop
-                for route_nr in range(len(r.routes)):
-                    r.routes[route_nr].hops[-1].mpp_record.payment_addr = bytes.fromhex(  # type: ignore
+                    assert route
+                    logger.debug(
+                        f"QueryRoutes returned {len(route.routes)} available routes."
+                    )
+                    route.routes[0].hops[-1].mpp_record.payment_addr = bytes.fromhex(  # type: ignore
                         payer_addr
                     )
-                    r.routes[route_nr].hops[  # type: ignore
+                    route.routes[0].hops[  # type: ignore
                         -1
                     ].mpp_record.total_amt_msat = total_amount_msat
 
                     # Send to route request
-                    r = await router_stub.SendToRouteV2(
+                    response = await router_stub.SendToRouteV2(
                         routerrpc.SendToRouteRequest(
                             payment_hash=bytes.fromhex(invoice.payment_hash),
-                            route=r.routes[route_nr],  # type: ignore
+                            route=route.routes[0],  # type: ignore
                         )
                     )
-                    if r.status == lnrpc.HTLCAttempt.HTLCStatus.FAILED:
-                        if r.failure.code == lnrpc.Failure.FailureCode.TEMPORARY_CHANNEL_FAILURE:
-                            # Try a different route
+                    assert route and response
+                    if response.status == lnrpc.HTLCAttempt.HTLCStatus.FAILED:
+                        if (
+                            response.failure.code
+                            == lnrpc.Failure.FailureCode.TEMPORARY_CHANNEL_FAILURE
+                        ):
+                            # Add the channels that failed to the excluded channels
+                            failure_index = response.failure.failure_source_index
+                            failed_source = (
+                                route.routes[0].hops[failure_index - 1].pub_key
+                            )
+                            failed_dest = route.routes[0].hops[failure_index].pub_key
+                            logger.debug(
+                                f"Partial payment failed from {failed_source} to {failed_dest} at index {failure_index-1} of the route"
+                            )
                             continue
                     break
         except AioRpcError as e:
@@ -272,25 +288,29 @@ class LndRPCWallet(LightningBackend):
                 error_message=str(e),
             )
 
-        if r.status == lnrpc.HTLCAttempt.HTLCStatus.FAILED:
-            error_message = f"Sending to route failed with code {r.failure.code}"
-            logger.error(error_message)
+        assert route and response
+        if response.status == lnrpc.HTLCAttempt.HTLCStatus.FAILED:
+            error_message = f"Sending to route failed with code {response.failure.code} after {total_attempts} different tries."
             return PaymentResponse(
                 result=PaymentResult.FAILED,
                 error_message=error_message,
             )
 
-        result = PaymentResult.UNKNOWN
-        if r.status == lnrpc.HTLCAttempt.HTLCStatus.SUCCEEDED:
+        if response.status == lnrpc.HTLCAttempt.HTLCStatus.SUCCEEDED:
             result = PaymentResult.SETTLED
-        elif r.status == lnrpc.HTLCAttempt.HTLCStatus.IN_FLIGHT:
+        elif response.status == lnrpc.HTLCAttempt.HTLCStatus.IN_FLIGHT:
             result = PaymentResult.PENDING
         else:
-            result = PaymentResult.FAILED
+            result = PaymentResult.UNKNOWN
 
         checking_id = invoice.payment_hash
-        fee_msat = r.route.total_fees_msat
-        preimage = r.preimage.hex()
+        fee_msat = response.route.total_fees_msat
+        preimage = response.preimage.hex()
+
+        logger.debug(f"Partial payment succeeded after {total_attempts} tries!")
+        logger.debug(
+            f"Partial payment route length was {len(route.routes[0].hops)} hops."
+        )
         return PaymentResponse(
             result=result,
             checking_id=checking_id,
@@ -376,11 +396,7 @@ class LndRPCWallet(LightningBackend):
         self, melt_quote: PostMeltQuoteRequest
     ) -> PaymentQuoteResponse:
         # get amount from melt_quote or from bolt11
-        amount_msat = (
-            melt_quote.mpp_amount
-            if melt_quote.is_mpp
-            else None
-        )
+        amount_msat = melt_quote.mpp_amount if melt_quote.is_mpp else None
 
         invoice_obj = bolt11.decode(melt_quote.request)
         assert invoice_obj.amount_msat, "invoice has no amount."

cashu/lightning/lndrest.py

@@ -41,6 +41,9 @@ INVOICE_RESULT_MAP = {
     "ACCEPTED": PaymentResult.PENDING,
 }
 
+MAX_ROUTE_RETRIES = 50
+TEMPORARY_CHANNEL_FAILURE_ERROR = "TEMPORARY_CHANNEL_FAILURE"
+
 
 class LndRestWallet(LightningBackend):
     """https://api.lightning.community/rest/index.html#lnd-rest-api-reference"""
@@ -219,6 +222,8 @@ class LndRestWallet(LightningBackend):
     async def pay_partial_invoice(
         self, quote: MeltQuote, amount: Amount, fee_limit_msat: int
     ) -> PaymentResponse:
+        attempts = 0
+
         # set the fee limit for the payment
         lnrpcFeeLimit = dict()
         lnrpcFeeLimit["fixed_msat"] = f"{fee_limit_msat}"
@@ -236,56 +241,78 @@ class LndRestWallet(LightningBackend):
         assert payer_addr_tag
         payer_addr = str(payer_addr_tag.data)
 
-        # get the route
-        r = await self.client.post(
-            url=f"/v1/graph/routes/{pubkey}/{amount.to(Unit.sat).amount}",
-            json={"fee_limit": lnrpcFeeLimit},
-            timeout=None,
-        )
+        # add the mpp_record to the last hop
+        response: Optional[httpx.Response] = None
+        route: Optional[httpx.Response] = None
 
-        data = r.json()
-        if r.is_error or data.get("message"):
-            error_message = data.get("message") or r.text
-            return PaymentResponse(
-                result=PaymentResult.FAILED, error_message=error_message
+        for attempt in range(MAX_ROUTE_RETRIES):
+            attempts += 1
+            # get the route
+            route = await self.client.post(
+                url=f"/v1/graph/routes/{pubkey}/{amount.to(Unit.sat).amount}",
+                json={
+                    "fee_limit": lnrpcFeeLimit,
+                    "use_mission_control": True,
+                },
+                timeout=None,
             )
 
-        # We need to set the mpp_record for a partial payment
-        mpp_record = {
-            "mpp_record": {
-                "payment_addr": base64.b64encode(bytes.fromhex(payer_addr)).decode(),
-                "total_amt_msat": total_amount_msat,
-            }
-        }
+            assert route
 
-        # add the mpp_record to the last hop
-        r = None    # type: ignore
-        for route_nr in range(len(data["routes"])):
-            logger.debug(f"Trying to pay partial amount with route number {route_nr+1}")
-            data["routes"][route_nr]["hops"][-1].update(mpp_record)
+            route_data = route.json()
+            if route.is_error or route_data.get("message"):
+                error_message = route_data.get("message") or route.text
+                return PaymentResponse(
+                    result=PaymentResult.FAILED, error_message=error_message
+                )
+
+            # We need to set the mpp_record for a partial payment
+            mpp_record = {
+                "mpp_record": {
+                    "payment_addr": base64.b64encode(
+                        bytes.fromhex(payer_addr)
+                    ).decode(),
+                    "total_amt_msat": total_amount_msat,
+                }
+            }
+            route_data["routes"][0]["hops"][-1].update(mpp_record)
 
             # send to route
-            r = await self.client.post(
+            response = await self.client.post(
                 url="/v2/router/route/send",
                 json={
                     "payment_hash": base64.b64encode(
                         bytes.fromhex(invoice.payment_hash)
                     ).decode(),
-                    "route": data["routes"][route_nr],
+                    "route": route_data["routes"][0],
                 },
                 timeout=None,
             )
 
-            data = r.json()
-            failure = data.get("failure")
-            if failure:
-                if failure["code"] == 15:
-                    # Try with a different route
+            assert response
+
+            response_data = response.json()
+            if response_data.get("status") == "FAILED":
+                if response_data["failure"]["code"] == TEMPORARY_CHANNEL_FAILURE_ERROR:
+                    # Add the channels that failed to the excluded channels
+                    failure_index = response_data["failure"]["failure_source_index"]
+                    failed_source = route_data["routes"][0]["hops"][failure_index - 1][
+                        "pub_key"
+                    ]
+                    failed_dest = route_data["routes"][0]["hops"][failure_index][
+                        "pub_key"
+                    ]
+                    logger.debug(
+                        f"Partial payment failed from {failed_source} to {failed_dest} at index {failure_index-1} of the route"
+                    )
                     continue
             break
 
-        if r.is_error or data.get("message"):
-            error_message = data.get("message") or r.text
+        assert response and route
+
+        data = response.json()
+        if response.is_error or data.get("message") or data.get("status") == "FAILED":
+            error_message = f"Sending to route failed with code {data.get('failure').get('code')} after {attempts} tries."
             return PaymentResponse(
                 result=PaymentResult.FAILED, error_message=error_message
             )
@@ -296,6 +323,11 @@ class LndRestWallet(LightningBackend):
         preimage = (
             base64.b64decode(data["preimage"]).hex() if data.get("preimage") else None
         )
+
+        logger.debug(f"Partial payment succeeded after {attempts} different tries!")
+        logger.debug(
+            f"Partial payment route length was {len(route.json().get('routes'))} hops."
+        )
         return PaymentResponse(
             result=result,
             checking_id=checking_id,

cashu/mint/ledger.py

@@ -545,6 +545,7 @@ class Ledger(
             melt_quote.is_mpp
             and melt_quote.mpp_amount != payment_quote.amount.to(Unit.msat).amount
         ):
+            logger.error(f"expected {payment_quote.amount.to(Unit.msat).amount} msat but got {melt_quote.mpp_amount}")
             raise TransactionError("quote amount not as requested")
         # make sure the backend returned the amount with a correct unit
         if not payment_quote.amount.unit == unit:

tests/test_wallet_regtest_mpp.py

@@ -2,17 +2,21 @@ import asyncio
 import threading
 from typing import List
 
+import bolt11
 import pytest
 import pytest_asyncio
 
-from cashu.core.base import Method, Proof
+from cashu.core.base import MeltQuote, MeltQuoteState, Method, Proof
+from cashu.lightning.base import PaymentResponse
 from cashu.lightning.clnrest import CLNRestWallet
 from cashu.mint.ledger import Ledger
 from cashu.wallet.wallet import Wallet
 from tests.conftest import SERVER_ENDPOINT
 from tests.helpers import (
+    SLEEP_TIME,
     assert_err,
-    get_real_invoice,
+    cancel_invoice,
+    get_hold_invoice,
     is_fake,
     partial_pay_real_invoice,
     pay_if_regtest,
@@ -47,12 +51,12 @@ async def test_regtest_pay_mpp(wallet: Wallet, ledger: Ledger):
     assert wallet.balance == 128
 
     # this is the invoice we want to pay in two parts
-    invoice_dict = get_real_invoice(64)
-    invoice_payment_request = invoice_dict["payment_request"]
+    preimage, invoice_dict = get_hold_invoice(64)
+    invoice_payment_request = str(invoice_dict["payment_request"])
 
     async def _mint_pay_mpp(invoice: str, amount: int, proofs: List[Proof]):
         # wallet pays 32 sat of the invoice
-        quote = await wallet.melt_quote(invoice, amount_msat=amount*1000)
+        quote = await wallet.melt_quote(invoice, amount_msat=amount * 1000)
         assert quote.amount == amount
         await wallet.melt(
             proofs,
@@ -112,13 +116,15 @@ async def test_regtest_pay_mpp_incomplete_payment(wallet: Wallet, ledger: Ledger
     assert wallet.balance == 384
 
     # this is the invoice we want to pay in two parts
-    invoice_dict = get_real_invoice(64)
-    invoice_payment_request = invoice_dict["payment_request"]
+    preimage, invoice_dict = get_hold_invoice(64)
+    invoice_payment_request = str(invoice_dict["payment_request"])
 
     async def pay_mpp(amount: int, proofs: List[Proof], delay: float = 0.0):
         await asyncio.sleep(delay)
         # wallet pays 32 sat of the invoice
-        quote = await wallet.melt_quote(invoice_payment_request, amount_msat=amount*1000)
+        quote = await wallet.melt_quote(
+            invoice_payment_request, amount_msat=amount * 1000
+        )
         assert quote.amount == amount
         await wallet.melt(
             proofs,
@@ -154,5 +160,112 @@ async def test_regtest_internal_mpp_melt_quotes(wallet: Wallet, ledger: Ledger):
 
     # try and create a multi-part melt quote
     await assert_err(
-        wallet.melt_quote(mint_quote.request, 100*1000), "internal mpp not allowed"
+        wallet.melt_quote(mint_quote.request, 100 * 1000), "internal mpp not allowed"
     )
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="only regtest")
+async def test_regtest_pay_mpp_cancel_payment(wallet: Wallet, ledger: Ledger):
+    # make sure that mpp is supported by the bolt11-sat backend
+    if not ledger.backends[Method["bolt11"]][wallet.unit].supports_mpp:
+        pytest.skip("backend does not support mpp")
+
+    # make sure wallet knows the backend supports mpp
+    assert wallet.mint_info.supports_mpp("bolt11", wallet.unit)
+
+    # top up wallet so we have enough for the payment
+    topup_mint_quote = await wallet.request_mint(128)
+    await pay_if_regtest(topup_mint_quote.request)
+    proofs1 = await wallet.mint(128, quote_id=topup_mint_quote.quote)
+    assert wallet.balance == 128
+
+    # create a hold invoice that we can cancel
+    preimage, invoice_dict = get_hold_invoice(64)
+    invoice_payment_request = str(invoice_dict.get("payment_request", ""))
+    invoice_obj = bolt11.decode(invoice_payment_request)
+    payment_hash = invoice_obj.payment_hash
+
+    async def _mint_pay_mpp(invoice: str, amount: int, proofs: List[Proof]):
+        # wallet pays 32 sat of the invoice
+        quote = await wallet.melt_quote(invoice, amount_msat=amount * 1000)
+        assert quote.amount == amount
+        await wallet.melt(
+            proofs,
+            invoice,
+            fee_reserve_sat=quote.fee_reserve,
+            quote_id=quote.quote,
+        )
+
+    def mint_pay_mpp(invoice: str, amount: int, proofs: List[Proof]):
+        asyncio.run(_mint_pay_mpp(invoice, amount, proofs))
+
+    # start the MPP payment
+    t1 = threading.Thread(
+        target=mint_pay_mpp, args=(invoice_payment_request, 32, proofs1)
+    )
+    t1.start()
+    await asyncio.sleep(SLEEP_TIME)
+
+    # cancel the invoice
+    cancel_invoice(payment_hash)
+    await asyncio.sleep(SLEEP_TIME)
+
+    # check the payment status
+    status = await ledger.backends[Method["bolt11"]][wallet.unit].get_payment_status(
+        payment_hash
+    )
+    assert status.failed  # some backends return unknown instead of failed
+    assert not status.preimage  # no preimage since payment failed
+
+    # check that the proofs are unspent since payment failed
+    states = await wallet.check_proof_state(proofs1)
+    assert all([s.unspent for s in states.states])
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="only regtest")
+async def test_regtest_pay_mpp_cancel_payment_pay_partial_invoice(
+    wallet: Wallet, ledger: Ledger
+):
+    # create a hold invoice that we can cancel
+    preimage, invoice_dict = get_hold_invoice(64)
+    invoice_payment_request = str(invoice_dict.get("payment_request", ""))
+    invoice_obj = bolt11.decode(invoice_payment_request)
+    payment_hash = invoice_obj.payment_hash
+
+    # Use a shared container to store the result
+    result_container = []
+
+    async def _mint_pay_mpp(invoice: str, amount: int) -> PaymentResponse:
+        ret = await ledger.backends[Method["bolt11"]][wallet.unit].pay_invoice(
+            MeltQuote(
+                request=invoice,
+                amount=amount,
+                fee_reserve=0,
+                quote="",
+                method="bolt11",
+                checking_id="",
+                unit=wallet.unit.name,
+                state=MeltQuoteState.pending,
+            ),
+            0,
+        )
+        return ret
+
+    # Create a wrapper function that will store the result
+    def thread_func():
+        result = asyncio.run(_mint_pay_mpp(invoice_payment_request, 32))
+        result_container.append(result)
+
+    t1 = threading.Thread(target=thread_func)
+    t1.start()
+    await asyncio.sleep(SLEEP_TIME)
+
+    # cancel the invoice
+    cancel_invoice(payment_hash)
+    await asyncio.sleep(SLEEP_TIME)
+
+    t1.join()
+    # Get the result from the container
+    assert result_container[0].failed