listinvoices: add index and start params.

Now we have defined ordering, we can add a start param.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Changelog-Added: JSON-RPC: `listinvoices` has `index` and `start` parameters for listing control.

.msggen.json

@@ -102,6 +102,10 @@
             "spent": 2,
             "unconfirmed": 0
         },
+        "ListinvoicesIndex": {
+            "created": 0,
+            "updated": 1
+        },
         "ListinvoicesInvoicesStatus": {
             "expired": 2,
             "paid": 1,
@@ -935,10 +939,12 @@
             "ListInvoices.invoices[].status": 4
         },
         "ListinvoicesRequest": {
+            "ListInvoices.index": 5,
             "ListInvoices.invstring": 2,
             "ListInvoices.label": 1,
             "ListInvoices.offer_id": 4,
-            "ListInvoices.payment_hash": 3
+            "ListInvoices.payment_hash": 3,
+            "ListInvoices.start": 6
         },
         "ListinvoicesResponse": {
             "ListInvoices.invoices[]": 1
@@ -3548,6 +3554,10 @@
             "added": "pre-v0.10.1",
             "deprecated": null
         },
+        "ListInvoices.index": {
+            "added": "v23.08",
+            "deprecated": false
+        },
         "ListInvoices.invoices[]": {
             "added": "pre-v0.10.1",
             "deprecated": false
@@ -3624,6 +3634,10 @@
             "added": "pre-v0.10.1",
             "deprecated": false
         },
+        "ListInvoices.start": {
+            "added": "v23.08",
+            "deprecated": false
+        },
         "ListNodes": {
             "added": "pre-v0.10.1",
             "deprecated": null

cln-grpc/proto/node.proto

@@ -627,10 +627,17 @@ message ListdatastoreDatastore {
 }
 
 message ListinvoicesRequest {
+	// ListInvoices.index
+	enum ListinvoicesIndex {
+		CREATED = 0;
+		UPDATED = 1;
+	}
 	optional string label = 1;
 	optional string invstring = 2;
 	optional bytes payment_hash = 3;
 	optional string offer_id = 4;
+	optional ListinvoicesIndex index = 5;
+	optional uint64 start = 6;
 }
 
 message ListinvoicesResponse {

cln-grpc/src/convert.rs

@@ -1823,6 +1823,8 @@ impl From<requests::ListinvoicesRequest> for pb::ListinvoicesRequest {
             invstring: c.invstring, // Rule #2 for type string?
             payment_hash: c.payment_hash.map(|v| hex::decode(v).unwrap()), // Rule #2 for type hex?
             offer_id: c.offer_id, // Rule #2 for type string?
+            index: c.index.map(|v| v as i32),
+            start: c.start, // Rule #2 for type u64?
         }
     }
 }
@@ -2499,6 +2501,8 @@ impl From<pb::ListinvoicesRequest> for requests::ListinvoicesRequest {
             invstring: c.invstring, // Rule #1 for type string?
             payment_hash: c.payment_hash.map(|v| hex::encode(v)), // Rule #1 for type hex?
             offer_id: c.offer_id, // Rule #1 for type string?
+            index: c.index.map(|v| v.try_into().unwrap()),
+            start: c.start, // Rule #1 for type u64?
         }
     }
 }

cln-rpc/src/model.rs

@@ -583,6 +583,34 @@ pub mod requests {
 	    type Response = super::responses::ListdatastoreResponse;
 	}
 
+	#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
+	pub enum ListinvoicesIndex {
+	    #[serde(rename = "created")]
+	    CREATED,
+	    #[serde(rename = "updated")]
+	    UPDATED,
+	}
+
+	impl TryFrom<i32> for ListinvoicesIndex {
+	    type Error = anyhow::Error;
+	    fn try_from(c: i32) -> Result<ListinvoicesIndex, anyhow::Error> {
+	        match c {
+	    0 => Ok(ListinvoicesIndex::CREATED),
+	    1 => Ok(ListinvoicesIndex::UPDATED),
+	            o => Err(anyhow::anyhow!("Unknown variant {} for enum ListinvoicesIndex", o)),
+	        }
+	    }
+	}
+
+	impl ToString for ListinvoicesIndex {
+	    fn to_string(&self) -> String {
+	        match self {
+	            ListinvoicesIndex::CREATED => "CREATED",
+	            ListinvoicesIndex::UPDATED => "UPDATED",
+	        }.to_string()
+	    }
+	}
+
 	#[derive(Clone, Debug, Deserialize, Serialize)]
 	pub struct ListinvoicesRequest {
 	    #[serde(skip_serializing_if = "Option::is_none")]
@@ -593,6 +621,10 @@ pub mod requests {
 	    pub payment_hash: Option<String>,
 	    #[serde(skip_serializing_if = "Option::is_none")]
 	    pub offer_id: Option<String>,
+	    #[serde(skip_serializing_if = "Option::is_none")]
+	    pub index: Option<ListinvoicesIndex>,
+	    #[serde(skip_serializing_if = "Option::is_none")]
+	    pub start: Option<u64>,
 	}
 
 	impl From<ListinvoicesRequest> for Request {

contrib/pyln-client/pyln/client/lightning.py

@@ -1023,7 +1023,7 @@ class LightningRpc(UnixDomainSocketRpc):
         """
         return self.call("listtransactions")
 
-    def listinvoices(self, label=None, payment_hash=None, invstring=None, offer_id=None):
+    def listinvoices(self, label=None, payment_hash=None, invstring=None, offer_id=None, index=None, start=None):
         """Query invoices
 
         Show invoice matching {label}, {payment_hash}, {invstring} or {offer_id}
@@ -1035,6 +1035,8 @@ class LightningRpc(UnixDomainSocketRpc):
             "payment_hash": payment_hash,
             "invstring": invstring,
             "offer_id": offer_id,
+            "index": index,
+            "start": start,
         }
         return self.call("listinvoices", payload)
 

doc/lightning-listinvoices.7.md

@@ -4,7 +4,7 @@ lightning-listinvoices -- Command for querying invoice status
 SYNOPSIS
 --------
 
-**listinvoices** [*label*] [*invstring*] [*payment\_hash*] [*offer\_id*]
+**listinvoices** [*label*] [*invstring*] [*payment\_hash*] [*offer\_id*] [*index* [*start*]]
 
 DESCRIPTION
 -----------
@@ -17,6 +17,10 @@ provided when creating the invoice, the `invstring` string representing
 the invoice, the `payment_hash` of the invoice, or the local `offer_id`
 this invoice was issued for. Only one of the query parameters can be used at once.
 
+`index` controls ordering, by `created` (default) or `updated`.  If
+`index` is specified, `start` may be specified to start from that
+value, which is generally returned from lightning-wait(7).
+
 RETURN VALUE
 ------------
 

doc/schemas/listinvoices.request.json

@@ -27,6 +27,20 @@
     "offer_id": {
       "type": "string",
       "description": ""
+    },
+    "index": {
+      "type": "string",
+      "added": "v23.08",
+      "enum": [
+        "created",
+        "updated"
+      ],
+      "description": ""
+    },
+    "start": {
+      "type": "u64",
+      "added": "v23.08",
+      "description": ""
     }
   }
 }

lightningd/invoice.c

@@ -1222,7 +1222,9 @@ static void json_add_invoices(struct json_stream *response,
 			      struct wallet *wallet,
 			      const struct json_escape *label,
 			      const struct sha256 *payment_hash,
-			      const struct sha256 *local_offer_id)
+			      const struct sha256 *local_offer_id,
+			      const enum wait_index *listindex,
+			      u64 liststart)
 {
 	const struct invoice_details *details;
 	u64 inv_dbid;
@@ -1244,7 +1246,9 @@ static void json_add_invoices(struct json_stream *response,
 	} else {
 		struct db_stmt *stmt;
 
-		for (stmt = invoices_first(wallet->invoices, &inv_dbid);
+		for (stmt = invoices_first(wallet->invoices,
+					   listindex, liststart,
+					   &inv_dbid);
 		     stmt;
 		     stmt = invoices_next(wallet->invoices, stmt, &inv_dbid)) {
 			details = invoices_get_details(tmpctx,
@@ -1271,6 +1275,8 @@ static struct command_result *json_listinvoices(struct command *cmd,
 	struct wallet *wallet = cmd->ld->wallet;
 	const char *invstring;
 	struct sha256 *payment_hash, *offer_id;
+	enum wait_index *listindex;
+	u64 *liststart;
 	char *fail;
 
 	if (!param(cmd, buffer, params,
@@ -1278,6 +1284,8 @@ static struct command_result *json_listinvoices(struct command *cmd,
 		   p_opt("invstring", param_invstring, &invstring),
 		   p_opt("payment_hash", param_sha256, &payment_hash),
 		   p_opt("offer_id", param_sha256, &offer_id),
+		   p_opt("index", param_index, &listindex),
+		   p_opt_def("start", param_u64, &liststart, 0),
 		   NULL))
 		return command_param_failed();
 
@@ -1288,6 +1296,10 @@ static struct command_result *json_listinvoices(struct command *cmd,
 				    " {label}, {invstring}, {payment_hash}"
 				    " or {offer_id}");
 	}
+	if (*liststart != 0 && !listindex) {
+		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+				    "Can only specify {start} with {index}");
+	}
 
 	/* Extract the payment_hash from the invoice. */
 	if (invstring != NULL) {
@@ -1312,7 +1324,8 @@ static struct command_result *json_listinvoices(struct command *cmd,
 
 	response = json_stream_success(cmd);
 	json_array_start(response, "invoices");
-	json_add_invoices(response, wallet, label, payment_hash, offer_id);
+	json_add_invoices(response, wallet, label, payment_hash, offer_id,
+			  listindex, *liststart);
 	json_array_end(response);
 	return command_success(cmd, response);
 }

lightningd/test/run-invoice-select-inchan.c

@@ -399,6 +399,8 @@ bool invoices_find_unpaid(struct invoices *invoices UNNEEDED,
 { fprintf(stderr, "invoices_find_unpaid called!\n"); abort(); }
 /* Generated stub for invoices_first */
 struct db_stmt *invoices_first(struct invoices *invoices UNNEEDED,
+			       const enum wait_index *listindex UNNEEDED,
+			       u64 liststart UNNEEDED,
 			       u64 *inv_dbid UNNEEDED)
 { fprintf(stderr, "invoices_first called!\n"); abort(); }
 /* Generated stub for invoices_get_details */
@@ -720,6 +722,12 @@ struct command_result *param_escaped_string(struct command *cmd UNNEEDED,
 					    const jsmntok_t *tok UNNEEDED,
 					    const char **str UNNEEDED)
 { fprintf(stderr, "param_escaped_string called!\n"); abort(); }
+/* Generated stub for param_index */
+struct command_result *param_index(struct command *cmd UNNEEDED, const char *name UNNEEDED,
+				   const char *buffer UNNEEDED,
+				   const jsmntok_t *tok UNNEEDED,
+				   enum wait_index **index UNNEEDED)
+{ fprintf(stderr, "param_index called!\n"); abort(); }
 /* Generated stub for param_invstring */
 struct command_result *param_invstring(struct command *cmd UNNEEDED, const char *name UNNEEDED,
 				    const char * buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,

tests/test_invoices.py

@@ -858,3 +858,35 @@ def test_invoice_deschash(node_factory, chainparams):
     wait_for(lambda: len([ev for ev in l1.rpc.bkpr_listincome()['income_events'] if ev['tag'] == 'invoice']) == 1)
     inv = only_one([ev for ev in l1.rpc.bkpr_listincome()['income_events'] if ev['tag'] == 'invoice'])
     assert inv['description'] == b11['description_hash']
+
+
+def test_listinvoices_index(node_factory, executor):
+    l1, l2 = node_factory.line_graph(2)
+
+    invs = {}
+    for i in range(1, 100):
+        invs[i] = l2.rpc.invoice(i, str(i), "test_listinvoices_index")
+
+    assert [inv['label'] for inv in l2.rpc.listinvoices(index='created')['invoices']] == [str(i) for i in range(1, 100)]
+    assert [inv['label'] for inv in l2.rpc.listinvoices(index='created', start=1)['invoices']] == [str(i) for i in range(1, 100)]
+    assert [inv['label'] for inv in l2.rpc.listinvoices(index='created', start=2)['invoices']] == [str(i) for i in range(2, 100)]
+    assert [inv['label'] for inv in l2.rpc.listinvoices(index='created', start=99)['invoices']] == [str(i) for i in range(99, 100)]
+    assert l2.rpc.listinvoices(index='created', start=100) == {'invoices': []}
+    assert l2.rpc.listinvoices(index='created', start=2100) == {'invoices': []}
+
+    # Pay 10 of them, in reverse order.  These will be the last ones in the 'updated' index.
+    for i in range(70, 60, -1):
+        l1.rpc.pay(invs[i]['bolt11'])
+
+    # Make sure it's fully resolved!
+    wait_for(lambda: only_one(l2.rpc.listpeerchannels()['channels'])['htlcs'] == [])
+
+    # They're all still there!
+    assert set([inv['label'] for inv in l2.rpc.listinvoices(index='updated')['invoices']]) == set([str(i) for i in range(1, 100)])
+
+    # Last 10 are in a defined order:
+    assert [inv['label'] for inv in l2.rpc.listinvoices(index='updated', start=1)['invoices']] == [str(i) for i in range(70, 60, -1)]
+    assert [inv['label'] for inv in l2.rpc.listinvoices(index='updated', start=2)['invoices']] == [str(i) for i in range(69, 60, -1)]
+    assert [inv['label'] for inv in l2.rpc.listinvoices(index='updated', start=10)['invoices']] == [str(i) for i in range(61, 60, -1)]
+    assert l2.rpc.listinvoices(index='updated', start=11) == {'invoices': []}
+    assert l2.rpc.listinvoices(index='updated', start=2100) == {'invoices': []}

wallet/db.c

@@ -964,6 +964,7 @@ static struct migration dbmigrations[] = {
     {SQL("ALTER TABLE channels ADD ignore_fee_limits INTEGER DEFAULT 0;"), NULL},
     {NULL, migrate_initialize_wait_indexes},
     {SQL("ALTER TABLE invoices ADD updated_index BIGINT DEFAULT 0"), NULL},
+    {SQL("CREATE INDEX invoice_update_idx ON invoices (updated_index)"), NULL},
 };
 
 /**

wallet/invoices.c

@@ -470,11 +470,24 @@ void invoices_delete_expired(struct invoices *invoices,
 }
 
 struct db_stmt *invoices_first(struct invoices *invoices,
+			       const enum wait_index *listindex,
+			       u64 liststart,
 			       u64 *inv_dbid)
 {
 	struct db_stmt *stmt;
 
-	stmt = db_prepare_v2(invoices->wallet->db, SQL("SELECT id FROM invoices ORDER by id;"));
+	if (listindex && *listindex == WAIT_INDEX_UPDATED) {
+		stmt = db_prepare_v2(invoices->wallet->db,
+				     SQL("SELECT id FROM invoices"
+					 " WHERE updated_index >= ?"
+					 " ORDER BY updated_index;"));
+	} else {
+		stmt = db_prepare_v2(invoices->wallet->db,
+				     SQL("SELECT id FROM invoices"
+					 " WHERE id >= ?"
+					 " ORDER BY id;"));
+	}
+	db_bind_u64(stmt, liststart);
 	db_query_prepared(stmt);
 
 	return invoices_next(invoices, stmt, inv_dbid);

wallet/invoices.h

@@ -138,12 +138,16 @@ void invoices_delete_expired(struct invoices *invoices,
 /**
  * Iterate through all the invoices.
  * @invoices: the invoices
+ * @listindex: what index order to use (if you care)
+ * @liststart: first index to return (0 == all).
  * @inv_dbid: the first invoice dbid (if returns non-NULL)
  *
  * Returns pointer to hand as @stmt to invoices_next(), or NULL.
  * If you choose not to call invoices_next() you must free it!
  */
 struct db_stmt *invoices_first(struct invoices *invoices,
+			       const enum wait_index *listindex,
+			       u64 liststart,
 			       u64 *inv_dbid);
 
 /**

wallet/wallet.h

@@ -11,6 +11,7 @@
 #include <lightningd/bitcoind.h>
 #include <lightningd/log.h>
 #include <lightningd/peer_htlcs.h>
+#include <lightningd/wait.h>
 
 struct amount_msat;
 struct invoices;