bkpr/listpeeers: add lease_fees back to funds; separate out in listpeers

First, how we record "our_funds" and then apply pushes vs lease_fees
(for liquidity ad buys/sales) was exactly opposite.

For pushes we were reporting the total funded into the channel, with the
push representing how much we'd later moved to the peer.

For lease_fees we were rerporting the total in the channel, with the
push representing how much was already moved to the peer.

We fix this (from a view perspective) by re-adding lease fees to what's
reported in the channel funding totals. Since this is now new behavior
(for leased channel values), we added new fields so we can take the old
field names thru a deprecation cycle.

We also make it possible to differentiate btw a push and a lease_fee
(before they were all the same), by adding to new fields to `listpeers`:
`fee_paid_msat` and `fee_rcvd_msat`.

This allows us to avoid math in the bookkeeper, instead we just pick
the numbers out directly and record them.

Fixes #5472

Changelog-Added: JSON-RPC: `listpeers` now has a few new fields for `funding` (`remote_funds_msat`, `local_funds_msat`, `fee_paid_msat`, `fee_rcvd_msat`).
Changelog-Deprecated: JSON-RPC: `listpeers`.`funded` fields `local_msat` and `remote_msat` are now deprecated.

tests/test_bookkeeper.py

@@ -1,11 +1,12 @@
 from fixtures import *  # noqa: F401,F403
 from decimal import Decimal
-from pyln.client import Millisatoshi, RpcError
+from pyln.client import Millisatoshi
 from fixtures import TEST_NETWORK
 from utils import (
-    sync_blockheight, wait_for, only_one
+    sync_blockheight, wait_for, only_one, first_channel_id
 )
 
+from pathlib import Path
 import os
 import pytest
 import unittest
@@ -324,6 +325,144 @@ def test_bookkeeping_rbf_withdraw(node_factory, bitcoind):
     assert len(fees) == 1
 
 
+@pytest.mark.openchannel('v2')
+@unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start")
+@unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded")
+def test_bookkeeping_missed_chans_leases(node_factory, bitcoind):
+    """
+    Test that a lease is correctly recorded if bookkeeper was off
+    """
+
+    coin_mvt_plugin = Path(__file__).parent / "plugins" / "coin_movements.py"
+    opts = {'funder-policy': 'match', 'funder-policy-mod': 100,
+            'lease-fee-base-sat': '100sat', 'lease-fee-basis': 100,
+            'plugin': str(coin_mvt_plugin),
+            'disable-plugin': 'bookkeeper'}
+    l1, l2 = node_factory.get_nodes(2, opts=opts)
+
+    open_amt = 500000
+    feerate = 2000
+    lease_fee = 6432000
+    invoice_msat = 11000000
+
+    l1.fundwallet(open_amt * 1000)
+    l2.fundwallet(open_amt * 1000)
+    l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+
+    # l1 leases a channel from l2
+    compact_lease = l2.rpc.funderupdate()['compact_lease']
+    txid = l1.rpc.fundchannel(l2.info['id'], open_amt, request_amt=open_amt,
+                              feerate='{}perkw'.format(feerate),
+                              compact_lease=compact_lease)['txid']
+    bitcoind.generate_block(1, wait_for_mempool=[txid])
+    wait_for(lambda: l1.channel_state(l2) == 'CHANNELD_NORMAL')
+    scid = l1.get_channel_scid(l2)
+    l1.wait_channel_active(scid)
+    channel_id = first_channel_id(l1, l2)
+
+    l1.pay(l2, invoice_msat)
+    l1.daemon.wait_for_log(r'coin movement:.*\'invoice\'')
+
+    # Now turn the bookkeeper on and restart
+    l1.stop()
+    l2.stop()
+    del l1.daemon.opts['disable-plugin']
+    del l2.daemon.opts['disable-plugin']
+    l1.start()
+    l2.start()
+
+    # Wait for the balance snapshot to fire/finish
+    l1.daemon.wait_for_log('Snapshot balances updated')
+    l2.daemon.wait_for_log('Snapshot balances updated')
+
+    def _check_events(node, channel_id, exp_events):
+        chan_events = [ev for ev in node.rpc.bkpr_listaccountevents()['events'] if ev['account'] == channel_id]
+        assert len(chan_events) == len(exp_events)
+        for ev, exp in zip(chan_events, exp_events):
+            assert ev['tag'] == exp[0]
+            assert ev['credit_msat'] == Millisatoshi(exp[1])
+            assert ev['debit_msat'] == Millisatoshi(exp[2])
+
+    # l1 events
+    exp_events = [('channel_open', open_amt * 1000 + lease_fee, 0),
+                  ('onchain_fee', 1408000, 0),
+                  ('lease_fee', 0, lease_fee),
+                  ('journal_entry', 0, invoice_msat)]
+    _check_events(l1, channel_id, exp_events)
+
+    exp_events = [('channel_open', open_amt * 1000, 0),
+                  ('onchain_fee', 980000, 0),
+                  ('lease_fee', lease_fee, 0),
+                  ('journal_entry', invoice_msat, 0)]
+    _check_events(l2, channel_id, exp_events)
+
+
+@unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start")
+@unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded")
+@pytest.mark.openchannel('v1', 'Uses push-msat')
+def test_bookkeeping_missed_chans_pushed(node_factory, bitcoind):
+    """
+    Test for a push_msat value in a missed channel open.
+    """
+    coin_mvt_plugin = Path(__file__).parent / "plugins" / "coin_movements.py"
+    l1, l2 = node_factory.get_nodes(2, opts={'disable-plugin': 'bookkeeper',
+                                             'plugin': str(coin_mvt_plugin)})
+
+    # Double check there's no bookkeeper plugin on
+    assert l1.daemon.opts['disable-plugin'] == 'bookkeeper'
+    assert l2.daemon.opts['disable-plugin'] == 'bookkeeper'
+
+    open_amt = 10**7
+    push_amt = 10**6 * 1000
+    invoice_msat = 11000000
+
+    l1.fundwallet(200000000)
+    l1.rpc.connect(l2.info['id'], 'localhost', l2.port)
+    txid = l1.rpc.fundchannel(l2.info['id'], open_amt, push_msat=push_amt)['txid']
+    bitcoind.generate_block(1, wait_for_mempool=[txid])
+    wait_for(lambda: l1.channel_state(l2) == 'CHANNELD_NORMAL')
+    scid = l1.get_channel_scid(l2)
+    l1.wait_channel_active(scid)
+    channel_id = first_channel_id(l1, l2)
+
+    # Send l2 funds via the channel
+    l1.pay(l2, invoice_msat)
+    l1.daemon.wait_for_log(r'coin movement:.*\'invoice\'')
+
+    # Now turn the bookkeeper on and restart
+    l1.stop()
+    l2.stop()
+    del l1.daemon.opts['disable-plugin']
+    del l2.daemon.opts['disable-plugin']
+    l1.start()
+    l2.start()
+
+    # Wait for the balance snapshot to fire/finish
+    l1.daemon.wait_for_log('Snapshot balances updated')
+    l2.daemon.wait_for_log('Snapshot balances updated')
+
+    def _check_events(node, channel_id, exp_events):
+        chan_events = [ev for ev in node.rpc.bkpr_listaccountevents()['events'] if ev['account'] == channel_id]
+        assert len(chan_events) == len(exp_events)
+        for ev, exp in zip(chan_events, exp_events):
+            assert ev['tag'] == exp[0]
+            assert ev['credit_msat'] == Millisatoshi(exp[1])
+            assert ev['debit_msat'] == Millisatoshi(exp[2])
+
+    # l1 events
+    exp_events = [('channel_open', open_amt * 1000, 0),
+                  ('onchain_fee', 5257000, 0),
+                  ('pushed', 0, push_amt),
+                  ('journal_entry', 0, invoice_msat)]
+    _check_events(l1, channel_id, exp_events)
+
+    # l2 events
+    exp_events = [('channel_open', 0, 0),
+                  ('pushed', push_amt, 0),
+                  ('journal_entry', invoice_msat, 0)]
+    _check_events(l2, channel_id, exp_events)
+
+
 @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start")
 def test_bookkeeping_onchaind_txs(node_factory, bitcoind):
     """
@@ -336,8 +475,6 @@ def test_bookkeeping_onchaind_txs(node_factory, bitcoind):
 
     # Double check there's no bookkeeper plugin on
     assert l1.daemon.opts['disable-plugin'] == 'bookkeeper'
-    with pytest.raises(RpcError):
-        l1.rpc.listincome()
 
     # Send l2 funds via the channel
     l1.pay(l2, 11000000)