Mint: watchdog balance log and killswitch (#705)

* wip store balance

* store balances in watchdog worker

* move mint_auth_database setting

* auth db

* balances returned as Amount (instead of int)

* add test for balance change on invoice receive

* fix 1 test

* cancel tasks on shutdown

* watchdog can now abort

* remove wallet api server

* fix lndgrpc

* fix lnbits balance

* disable watchdog

* balance lnbits msat

* test db watcher with its own database connection

* init superclass only once

* wip: log balance in keysets table

* check max balance using new keyset balance

* fix test

* fix another test

* store fees in keysets

* format

* cleanup

* shorter

* add keyset migration to auth server

* fix fakewallet

* fix db tests

* fix postgres problems during migration 26 (mint)

* fix cln

* ledger

* working with pending

* super fast watchdog, errors

* test new pipeline

* delete walletapi

* delete unneeded files

* revert workflows

cashu/mint/crud.py

@@ -1,15 +1,18 @@
 import json
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 from loguru import logger
 
 from ..core.base import (
+    Amount,
     BlindedSignature,
     MeltQuote,
+    MintBalanceLogEntry,
     MintKeyset,
     MintQuote,
     Proof,
+    Unit,
 )
 from ..core.db import (
     Connection,
@@ -31,6 +34,7 @@ class LedgerCrud(ABC):
         *,
         db: Database,
         id: str = "",
+        unit: str = "",
         derivation_path: str = "",
         seed: str = "",
         conn: Optional[Connection] = None,
@@ -118,13 +122,33 @@ class LedgerCrud(ABC):
         conn: Optional[Connection] = None,
     ) -> None: ...
 
+    @abstractmethod
+    async def bump_keyset_balance(
+        self,
+        *,
+        db: Database,
+        keyset: MintKeyset,
+        amount: int,
+        conn: Optional[Connection] = None,
+    ) -> None: ...
+
+    @abstractmethod
+    async def bump_keyset_fees_paid(
+        self,
+        *,
+        db: Database,
+        keyset: MintKeyset,
+        amount: int,
+        conn: Optional[Connection] = None,
+    ) -> None: ...
+
     @abstractmethod
     async def get_balance(
         self,
         keyset: MintKeyset,
         db: Database,
         conn: Optional[Connection] = None,
-    ) -> int: ...
+    ) -> Tuple[Amount, Amount]: ...
 
     @abstractmethod
     async def store_promise(
@@ -234,6 +258,25 @@ class LedgerCrud(ABC):
         conn: Optional[Connection] = None,
     ) -> None: ...
 
+    @abstractmethod
+    async def store_balance_log(
+        self,
+        backend_balance: Amount,
+        keyset_balance: Amount,
+        keyset_fees_paid: Amount,
+        db: Database,
+        conn: Optional[Connection] = None,
+    ) -> None: ...
+
+    @abstractmethod
+    async def get_last_balance_log_entry(
+        self,
+        *,
+        unit: Unit,
+        db: Database,
+        conn: Optional[Connection] = None,
+    ) -> MintBalanceLogEntry | None: ...
+
 
 class LedgerCrudSqlite(LedgerCrud):
     """Implementation of LedgerCrud for sqlite.
@@ -645,8 +688,8 @@ class LedgerCrudSqlite(LedgerCrud):
         await (conn or db).execute(
             f"""
             INSERT INTO {db.table_with_schema('keysets')}
-            (id, seed, encrypted_seed, seed_encryption_method, derivation_path, valid_from, valid_to, first_seen, active, version, unit, input_fee_ppk, amounts)
-            VALUES (:id, :seed, :encrypted_seed, :seed_encryption_method, :derivation_path, :valid_from, :valid_to, :first_seen, :active, :version, :unit, :input_fee_ppk, :amounts)
+            (id, seed, encrypted_seed, seed_encryption_method, derivation_path, valid_from, valid_to, first_seen, active, version, unit, input_fee_ppk, amounts, balance)
+            VALUES (:id, :seed, :encrypted_seed, :seed_encryption_method, :derivation_path, :valid_from, :valid_to, :first_seen, :active, :version, :unit, :input_fee_ppk, :amounts, :balance)
             """,
             {
                 "id": keyset.id,
@@ -666,31 +709,66 @@ class LedgerCrudSqlite(LedgerCrud):
                 "unit": keyset.unit.name,
                 "input_fee_ppk": keyset.input_fee_ppk,
                 "amounts": json.dumps(keyset.amounts),
+                "balance": keyset.balance,
             },
         )
 
+    async def bump_keyset_balance(
+        self,
+        *,
+        db: Database,
+        keyset: MintKeyset,
+        amount: int,
+        conn: Optional[Connection] = None,
+    ) -> None:
+        await (conn or db).execute(
+            f"""
+            UPDATE {db.table_with_schema('keysets')}
+            SET balance = balance + :amount
+            WHERE id = :id
+            """,
+            {"amount": amount, "id": keyset.id},
+        )
+
+    async def bump_keyset_fees_paid(
+        self,
+        *,
+        db: Database,
+        keyset: MintKeyset,
+        amount: int,
+        conn: Optional[Connection] = None,
+    ) -> None:
+        await (conn or db).execute(
+            f"""
+            UPDATE {db.table_with_schema('keysets')}
+            SET fees_paid = fees_paid + :amount
+            WHERE id = :id
+            """,
+            {"amount": amount, "id": keyset.id},
+        )
+
     async def get_balance(
         self,
         keyset: MintKeyset,
         db: Database,
         conn: Optional[Connection] = None,
-    ) -> int:
+    ) -> Tuple[Amount, Amount]:
         row = await (conn or db).fetchone(
             f"""
-            SELECT balance FROM {db.table_with_schema('balance')}
-            WHERE keyset = :keyset
+            SELECT balance, fees_paid FROM {db.table_with_schema('keysets')}
+            WHERE id = :id
             """,
             {
-                "keyset": keyset.id,
+                "id": keyset.id,
             },
         )
 
         if row is None:
-            return 0
+            return Amount(keyset.unit, 0), Amount(keyset.unit, 0)
 
-        # sqlalchemy index of first element
-        key = next(iter(row))
-        return int(row[key])
+        return Amount(keyset.unit, int(row["balance"])), Amount(
+            keyset.unit, int(row["fees_paid"])
+        )
 
     async def get_keyset(
         self,
@@ -764,6 +842,7 @@ class LedgerCrudSqlite(LedgerCrud):
                 "version": keyset.version,
                 "unit": keyset.unit.name,
                 "input_fee_ppk": keyset.input_fee_ppk,
+                "balance": keyset.balance,
             },
         )
 
@@ -781,3 +860,48 @@ class LedgerCrudSqlite(LedgerCrud):
         values = {f"y_{i}": Ys[i] for i in range(len(Ys))}
         rows = await (conn or db).fetchall(query, values)
         return [Proof(**r) for r in rows] if rows else []
+
+    async def store_balance_log(
+        self,
+        backend_balance: Amount,
+        keyset_balance: Amount,
+        keyset_fees_paid: Amount,
+        db: Database,
+        conn: Optional[Connection] = None,
+    ):
+        if backend_balance.unit != keyset_balance.unit:
+            raise ValueError("Units do not match")
+
+        await (conn or db).execute(
+            f"""
+            INSERT INTO {db.table_with_schema('balance_log')}
+            (unit, backend_balance, keyset_balance, keyset_fees_paid, time)
+            VALUES (:unit, :backend_balance, :keyset_balance, :keyset_fees_paid, :time)
+            """,
+            {
+                "unit": backend_balance.unit.name,
+                "backend_balance": backend_balance.amount,
+                "keyset_balance": keyset_balance.amount,
+                "keyset_fees_paid": keyset_fees_paid.amount,
+                "time": db.to_timestamp(db.timestamp_now_str()),
+            },
+        )
+
+    async def get_last_balance_log_entry(
+        self,
+        *,
+        unit: Unit,
+        db: Database,
+        conn: Optional[Connection] = None,
+    ) -> MintBalanceLogEntry | None:
+        row = await (conn or db).fetchone(
+            f"""
+            SELECT * from {db.table_with_schema('balance_log')}
+            WHERE unit = :unit
+            ORDER BY time DESC
+            LIMIT 1
+            """,
+            {"unit": unit.name},
+        )
+
+        return MintBalanceLogEntry.from_row(row) if row else None