Nutshell cleanup wishlist (#332)

* fix keys

* fix tests

* backwards compatible api upgrade

* upgrade seems to work

* fix tests

* add deprecated api functions

* add more tests of backwards compat

* add test serialization for nut00

* remove a redundant test

* move mint and melt to new api

* mypy works

* CI: mypy --check-untyped-defs

* add deprecated router

* add hints and remove logs

* fix tests

* cleanup

* use new mint and melt endpoints

* tests passing?

* fix mypy

* make format

* make format

* make format

* commit

* errors gone

* save

* adjust the API

* store quotes in db

* make mypy happy

* add fakewallet settings

* remove LIGHTNING=True and pass quote id for melt

* format

* tests passing

* add CoreLightningRestWallet

* add macaroon loader

* add correct config

* preimage -> proof

* move wallet.status() to cli.helpers.print_status()

* remove statuses from tests

* remove

* make format

* Use httpx in deprecated wallet

* fix cln interface

* create invoice before quote

* internal transactions and deprecated api testing

* fix tests

* add deprecated API tests

* fastapi type hints break things

* fix duplicate wallet error

* make format

* update poetry in CI to 1.7.1

* precommit restore

* remove bolt11

* oops

* default poetry

* store fee reserve for melt quotes and refactor melt()

* works?

* make format

* test

* finally

* fix deprecated models

* rename v1 endpoints to bolt11

* raise restore and check to v1, bump version to 0.15.0

* add version byte to keyset id

* remove redundant fields in json

* checks

* generate bip32 keyset wip

* migrate old keysets

* load duplicate keys

* duplicate old keysets

* revert router changes

* add deprecated /check and /restore endpoints

* try except invalidate

* parse unit from derivation path, adjust keyset id calculation with bytes

* remove keyest id from functions again and rely on self.keyset_id

* mosts tests work

* mint loads multiple derivation paths

* make format

* properly print units

* fix tests

* wallet works with multiple units

* add strike wallet and choose backend dynamically

* fix mypy

* add get_payment_quote to lightning backends

* make format

* fix startup

* fix lnbitswallet

* fix tests

* LightningWallet -> LightningBackend

* remove comments

* make format

* remove msat conversion

* add Amount type

* fix regtest

* use melt_quote as argument for pay_invoice

* test old api

* fees in sats

* fix deprecated fees

* fixes

* print balance correctly

* internally index keyset response by int

* add pydantic validation to input models

* add timestamps to mint db

* store timestamps for invoices, promises, proofs_used

* fix wallet migration

* rotate keys correctly for testing

* remove print

* update latest keyset

* fix tests

* fix test

* make format

* make format with correct black version

* remove nsat and cheese

* test against deprecated mint

* fix tests?

* actually use env var

* mint run with env vars

* moar test

* cleanup

* simplify tests, load all keys

* try out testing with internal invoices

* fix internal melt test

* fix test

* deprecated checkfees expects appropriate fees

* adjust comment

* drop lightning table

* split migration for testing for now, remove it later

* remove unused lightning table

* skip_private_key -> skip_db_read

* throw error on migration error

* reorder

* fix migrations

* fix lnbits fee return value negative

* fix typo

* comments

* add type

* make format

* split must use correct amount

* fix tests

* test deprecated api with internal/external melts

* do not split if not necessary

* refactor

* fix test

* make format with new black

* cleanup and add comments

* add quote state check endpoints

* fix deprecated wallet response

* split -> swap endpoint

* make format

* add expiry to quotes, get quote endpoints, and adjust to nut review comments

* allow overpayment of melt

* add lightning wallet tests

* commiting to save

* fix tests a bit

* make format

* remove comments

* get mint info

* check_spendable default False, and return payment quote checking id

* make format

* bump version in pyproject

* update to /v1/checkstate

* make format

* fix mint api checks

* return witness on /v1/checkstate

* no failfast

* try fail-fast: false in ci.yaml

* fix db lookup

* clean up literals

tests/test_mint_operations.py

@@ -1,11 +1,13 @@
 import pytest
 import pytest_asyncio
 
+from cashu.core.base import PostMeltQuoteRequest, PostMintQuoteRequest
+from cashu.core.helpers import sum_proofs
 from cashu.mint.ledger import Ledger
 from cashu.wallet.wallet import Wallet
 from cashu.wallet.wallet import Wallet as Wallet1
 from tests.conftest import SERVER_ENDPOINT
-from tests.helpers import pay_if_regtest
+from tests.helpers import get_real_invoice, is_fake, is_regtest, pay_if_regtest
 
 
 async def assert_err(f, msg):
@@ -20,36 +22,120 @@ async def assert_err(f, msg):
 
 
 @pytest_asyncio.fixture(scope="function")
-async def wallet1(mint):
+async def wallet1(ledger: Ledger):
     wallet1 = await Wallet1.with_db(
         url=SERVER_ENDPOINT,
         db="test_data/wallet1",
         name="wallet1",
     )
     await wallet1.load_mint()
-    wallet1.status()
     yield wallet1
 
 
 @pytest.mark.asyncio
-async def test_melt(wallet1: Wallet, ledger: Ledger):
+@pytest.mark.skipif(is_regtest, reason="only works with FakeWallet")
+async def test_melt_internal(wallet1: Wallet, ledger: Ledger):
     # mint twice so we have enough to pay the second invoice back
-    invoice = await wallet1.request_mint(64)
-    pay_if_regtest(invoice.bolt11)
-    await wallet1.mint(64, id=invoice.id)
-    invoice2 = await wallet1.request_mint(64)
-    pay_if_regtest(invoice2.bolt11)
-    await wallet1.mint(64, id=invoice2.id)
+    invoice = await wallet1.request_mint(128)
+    await wallet1.mint(128, id=invoice.id)
     assert wallet1.balance == 128
-    total_amount, fee_reserve_sat = await wallet1.get_pay_amount_with_fees(
-        invoice2.bolt11
+
+    # create a mint quote so that we can melt to it internally
+    invoice_to_pay = await wallet1.request_mint(64)
+    invoice_payment_request = invoice_to_pay.bolt11
+
+    melt_quote = await ledger.melt_quote(
+        PostMeltQuoteRequest(request=invoice_payment_request, unit="sat")
     )
-    melt_fees = await ledger.get_melt_fees(invoice2.bolt11)
-    assert melt_fees == fee_reserve_sat
+    assert not melt_quote.paid
+    assert melt_quote.amount == 64
+    assert melt_quote.fee_reserve == 0
 
+    melt_quote_pre_payment = await ledger.get_melt_quote(melt_quote.quote)
+    assert not melt_quote_pre_payment.paid, "melt quote should not be paid"
+
+    keep_proofs, send_proofs = await wallet1.split_to_send(wallet1.proofs, 64)
+    await ledger.melt(proofs=send_proofs, quote=melt_quote.quote)
+
+    melt_quote_post_payment = await ledger.get_melt_quote(melt_quote.quote)
+    assert melt_quote_post_payment.paid, "melt quote should be paid"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="only works with Regtest")
+async def test_melt_external(wallet1: Wallet, ledger: Ledger):
+    # mint twice so we have enough to pay the second invoice back
+    invoice = await wallet1.request_mint(128)
+    pay_if_regtest(invoice.bolt11)
+    await wallet1.mint(128, id=invoice.id)
+    assert wallet1.balance == 128
+
+    invoice_dict = get_real_invoice(64)
+    invoice_payment_request = invoice_dict["payment_request"]
+
+    mint_quote = await wallet1.get_pay_amount_with_fees(invoice_payment_request)
+    total_amount = mint_quote.amount + mint_quote.fee_reserve
     keep_proofs, send_proofs = await wallet1.split_to_send(wallet1.proofs, total_amount)
+    melt_quote = await ledger.melt_quote(
+        PostMeltQuoteRequest(request=invoice_payment_request, unit="sat")
+    )
 
-    await ledger.melt(send_proofs, invoice2.bolt11, outputs=None)
+    melt_quote_pre_payment = await ledger.get_melt_quote(melt_quote.quote)
+    assert not melt_quote_pre_payment.paid, "melt quote should not be paid"
+
+    assert not melt_quote.paid, "melt quote should not be paid"
+    await ledger.melt(proofs=send_proofs, quote=melt_quote.quote)
+
+    melt_quote_post_payment = await ledger.get_melt_quote(melt_quote.quote)
+    assert melt_quote_post_payment.paid, "melt quote should be paid"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_regtest, reason="only works with FakeWallet")
+async def test_mint_internal(wallet1: Wallet, ledger: Ledger):
+    invoice = await wallet1.request_mint(128)
+
+    mint_quote = await ledger.get_mint_quote(invoice.id)
+
+    assert mint_quote.paid, "mint quote should be paid"
+
+    output_amounts = [128]
+    secrets, rs, derivation_paths = await wallet1.generate_n_secrets(
+        len(output_amounts)
+    )
+    outputs, rs = wallet1._construct_outputs(output_amounts, secrets, rs)
+    await ledger.mint(outputs=outputs, quote_id=invoice.id)
+
+    await assert_err(
+        ledger.mint(outputs=outputs, quote_id=invoice.id),
+        "outputs have already been signed before.",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="only works with Regtest")
+async def test_mint_external(wallet1: Wallet, ledger: Ledger):
+    quote = await ledger.mint_quote(PostMintQuoteRequest(amount=128, unit="sat"))
+
+    mint_quote = await ledger.get_mint_quote(quote.quote)
+    assert not mint_quote.paid, "mint quote not should be paid"
+
+    await assert_err(
+        wallet1.mint(128, id=quote.quote),
+        "quote not paid",
+    )
+
+    pay_if_regtest(quote.request)
+
+    mint_quote = await ledger.get_mint_quote(quote.quote)
+    assert mint_quote.paid, "mint quote should be paid"
+
+    output_amounts = [128]
+    secrets, rs, derivation_paths = await wallet1.generate_n_secrets(
+        len(output_amounts)
+    )
+    outputs, rs = wallet1._construct_outputs(output_amounts, secrets, rs)
+    await ledger.mint(outputs=outputs, quote_id=quote.quote)
 
 
 @pytest.mark.asyncio
@@ -120,14 +206,13 @@ async def test_split_with_input_more_than_outputs(wallet1: Wallet, ledger: Ledge
 
     # make sure we can still spend our tokens
     keep_proofs, send_proofs = await wallet1.split(inputs, 10)
-    print(keep_proofs, send_proofs)
 
 
 @pytest.mark.asyncio
 async def test_split_twice_with_same_outputs(wallet1: Wallet, ledger: Ledger):
     invoice = await wallet1.request_mint(128)
     pay_if_regtest(invoice.bolt11)
-    await wallet1.mint(128, [64, 64], id=invoice.id)
+    await wallet1.mint(128, split=[64, 64], id=invoice.id)
     inputs1 = wallet1.proofs[:1]
     inputs2 = wallet1.proofs[1:]
 
@@ -164,14 +249,14 @@ async def test_mint_with_same_outputs_twice(wallet1: Wallet, ledger: Ledger):
         len(output_amounts)
     )
     outputs, rs = wallet1._construct_outputs(output_amounts, secrets, rs)
-    await ledger.mint(outputs, id=invoice.id)
+    await ledger.mint(outputs=outputs, quote_id=invoice.id)
 
     # now try to mint with the same outputs again
     invoice2 = await wallet1.request_mint(128)
     pay_if_regtest(invoice2.bolt11)
 
     await assert_err(
-        ledger.mint(outputs, id=invoice2.id),
+        ledger.mint(outputs=outputs, quote_id=invoice2.id),
         "outputs have already been signed before.",
     )
 
@@ -191,16 +276,79 @@ async def test_melt_with_same_outputs_twice(wallet1: Wallet, ledger: Ledger):
     # we use the outputs once for minting
     invoice2 = await wallet1.request_mint(128)
     pay_if_regtest(invoice2.bolt11)
-    await ledger.mint(outputs, id=invoice2.id)
+    await ledger.mint(outputs=outputs, quote_id=invoice2.id)
 
     # use the same outputs for melting
-    invoice3 = await wallet1.request_mint(128)
+    mint_quote = await ledger.mint_quote(PostMintQuoteRequest(unit="sat", amount=128))
+    melt_quote = await ledger.melt_quote(
+        PostMeltQuoteRequest(unit="sat", request=mint_quote.request)
+    )
     await assert_err(
-        ledger.melt(wallet1.proofs, invoice3.bolt11, outputs=outputs),
+        ledger.melt(proofs=wallet1.proofs, quote=melt_quote.quote, outputs=outputs),
         "outputs have already been signed before.",
     )
 
 
+@pytest.mark.asyncio
+async def test_melt_with_less_inputs_than_invoice(wallet1: Wallet, ledger: Ledger):
+    invoice = await wallet1.request_mint(32)
+    pay_if_regtest(invoice.bolt11)
+    await wallet1.mint(32, id=invoice.id)
+
+    # outputs for fee return
+    output_amounts = [1, 1, 1, 1]
+    secrets, rs, derivation_paths = await wallet1.generate_n_secrets(
+        len(output_amounts)
+    )
+    outputs, rs = wallet1._construct_outputs(output_amounts, secrets, rs)
+
+    # create a mint quote to pay
+    mint_quote = await ledger.mint_quote(PostMintQuoteRequest(unit="sat", amount=128))
+    # prepare melt quote
+    melt_quote = await ledger.melt_quote(
+        PostMeltQuoteRequest(unit="sat", request=mint_quote.request)
+    )
+
+    assert melt_quote.amount + melt_quote.fee_reserve > sum_proofs(wallet1.proofs)
+
+    # try to pay with not enough inputs
+    await assert_err(
+        ledger.melt(proofs=wallet1.proofs, quote=melt_quote.quote, outputs=outputs),
+        "not enough inputs provided for melt",
+    )
+
+
+@pytest.mark.asyncio
+async def test_melt_with_more_inputs_than_invoice(wallet1: Wallet, ledger: Ledger):
+    invoice = await wallet1.request_mint(130)
+    pay_if_regtest(invoice.bolt11)
+    await wallet1.mint(130, split=[64, 64, 2], id=invoice.id)
+
+    # outputs for fee return
+    output_amounts = [1, 1, 1, 1]
+    secrets, rs, derivation_paths = await wallet1.generate_n_secrets(
+        len(output_amounts)
+    )
+    outputs, rs = wallet1._construct_outputs(output_amounts, secrets, rs)
+
+    # create a mint quote to pay
+    mint_quote = await ledger.mint_quote(PostMintQuoteRequest(unit="sat", amount=128))
+    # prepare melt quote
+    melt_quote = await ledger.melt_quote(
+        PostMeltQuoteRequest(unit="sat", request=mint_quote.request)
+    )
+    # fees are 0 because it's internal
+    assert melt_quote.fee_reserve == 0
+
+    # make sure we have more inputs than the melt quote needs
+    assert sum_proofs(wallet1.proofs) >= melt_quote.amount + melt_quote.fee_reserve
+    payment_proof, return_outputs = await ledger.melt(
+        proofs=wallet1.proofs, quote=melt_quote.quote, outputs=outputs
+    )
+    # we get 2 sats back because we overpaid
+    assert sum([o.amount for o in return_outputs]) == 2
+
+
 @pytest.mark.asyncio
 async def test_check_proof_state(wallet1: Wallet, ledger: Ledger):
     invoice = await wallet1.request_mint(64)
@@ -209,6 +357,7 @@ async def test_check_proof_state(wallet1: Wallet, ledger: Ledger):
 
     keep_proofs, send_proofs = await wallet1.split_to_send(wallet1.proofs, 10)
 
-    spendable, pending = await ledger.check_proof_state(proofs=send_proofs)
-    assert sum(spendable) == len(send_proofs)
-    assert sum(pending) == 0
+    proof_states = await ledger.check_proofs_state(
+        secrets=[p.secret for p in send_proofs]
+    )
+    assert all([p.state.value == "UNSPENT" for p in proof_states])