datastore: turn keys into arrays

After some discussion with @shesek, and my own usage, we agreed that
a more comprehensive interface, which explicitly supports grouping,
is desirable.

Thus keys are now arrays, with the semantic that a key is either a
parent or has a value, never both.

For convenience in the JSON schema, we always return them as arrays,
though we accept simple strings as arguments.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/jsonrpc_errors.h

@@ -92,6 +92,8 @@ static const errcode_t DATASTORE_DEL_WRONG_GENERATION = 1201;
 static const errcode_t DATASTORE_UPDATE_ALREADY_EXISTS = 1202;
 static const errcode_t DATASTORE_UPDATE_DOES_NOT_EXIST = 1203;
 static const errcode_t DATASTORE_UPDATE_WRONG_GENERATION = 1204;
+static const errcode_t DATASTORE_UPDATE_HAS_CHILDREN = 1205;
+static const errcode_t DATASTORE_UPDATE_NO_CHILDREN = 1206;
 
 /* Errors from wait* commands */
 static const errcode_t WAIT_TIMEOUT = 2000;

doc/lightning-datastore.7

@@ -11,8 +11,11 @@ The \fBdatastore\fR RPC command allows plugins to store data in the
 c-lightning database, for later retrieval\.
 
 
-There can only be one entry for each \fIkey\fR, so prefixing with the
+\fIkey\fR is an array of values (though a single value is treated as a
-plugin name (e\.g\. \fBsummary.\fR) is recommended\.
+one-element array), to form a heirarchy\.  Using the first element of
+the key as the plugin name (e\.g\. \fB[ "summary" ]\fR) is recommended\.
+A key can either have children or a value, never both: parents are
+created and removed automatically\.
 
 
 \fImode\fR is one of "must-create" (default, fails it it already exists),
@@ -33,11 +36,17 @@ On success, an object is returned, containing:
 
 .RS
 .IP \[bu]
-\fBkey\fR (string): The key which has been added to the datastore
+\fBkey\fR (array of strings):
+.RS
 .IP \[bu]
-\fBgeneration\fR (u64): The number of times this has been updated
+Part of the key added to the datastore
+
+.RE
+
 .IP \[bu]
-\fBhex\fR (hex): The hex data which has been added to the datastore
+\fBgeneration\fR (u64, optional): The number of times this has been updated
+.IP \[bu]
+\fBhex\fR (hex, optional): The hex data which has been added to the datastore
 .IP \[bu]
 \fBstring\fR (string, optional): The data as a string, if it's valid utf-8
 
@@ -53,6 +62,10 @@ The following error codes may occur:
 .IP \[bu]
 1204: The generation was wrong (and generation was specified)
 .IP \[bu]
+1205: The key has children already\.
+.IP \[bu]
+1206: One of the parents already exists with a value\.
+.IP \[bu]
 -32602: invalid parameters
 
 .RE
@@ -68,4 +81,4 @@ Rusty Russell \fI<rusty@rustcorp.com.au\fR> is mainly responsible\.
 
 Main web site: \fIhttps://github.com/ElementsProject/lightning\fR
 
-\" SHA256STAMP:0ef09e6f98d7e34e7d8339351c29ffc70be71fbf9f05f581488e3c7f603d3721
+\" SHA256STAMP:cd66becc88b27328ebf6629d1ea607166b935fcd970c8a4a851e45fc1abe67d8

doc/lightning-datastore.7.md

@@ -12,8 +12,11 @@ DESCRIPTION
 The **datastore** RPC command allows plugins to store data in the
 c-lightning database, for later retrieval.
 
-There can only be one entry for each *key*, so prefixing with the
+*key* is an array of values (though a single value is treated as a
-plugin name (e.g. `summary.`) is recommended.
+one-element array), to form a heirarchy.  Using the first element of
+the key as the plugin name (e.g. `[ "summary" ]`) is recommended.
+A key can either have children or a value, never both: parents are
+created and removed automatically.
 
 *mode* is one of "must-create" (default, fails it it already exists),
 "must-replace" (fails it it doesn't already exist),
@@ -31,9 +34,10 @@ RETURN VALUE
 
 [comment]: # (GENERATE-FROM-SCHEMA-START)
 On success, an object is returned, containing:
-- **key** (string): The key which has been added to the datastore
+- **key** (array of strings):
-- **generation** (u64): The number of times this has been updated
+  - Part of the key added to the datastore
-- **hex** (hex): The hex data which has been added to the datastore
+- **generation** (u64, optional): The number of times this has been updated
+- **hex** (hex, optional): The hex data which has been added to the datastore
 - **string** (string, optional): The data as a string, if it's valid utf-8
 [comment]: # (GENERATE-FROM-SCHEMA-END)
 
@@ -41,6 +45,8 @@ The following error codes may occur:
 - 1202: The key already exists (and mode said it must not)
 - 1203: The key does not exist (and mode said it must)
 - 1204: The generation was wrong (and generation was specified)
+- 1205: The key has children already.
+- 1206: One of the parents already exists with a value.
 - -32602: invalid parameters
 
 AUTHOR
@@ -58,4 +64,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:0867f9910b75ef66e640a92aad55dbab7ce0b3278fd1fb200f91c2a1a6164409)
+[comment]: # ( SHA256STAMP:a66ad377a86e479704a3f5db06cffbd54bcd34fc9d36724649ace7b1f1e16bce)

doc/lightning-deldatastore.7

@@ -20,11 +20,17 @@ On success, an object is returned, containing:
 
 .RS
 .IP \[bu]
-\fBkey\fR (string): The key which has been removed from the datastore
+\fBkey\fR (array of strings):
+.RS
 .IP \[bu]
-\fBgeneration\fR (u64): The number of times this has been updated
+Part of the key added to the datastore
+
+.RE
+
 .IP \[bu]
-\fBhex\fR (hex): The hex data which has removed from the datastore
+\fBgeneration\fR (u64, optional): The number of times this has been updated
+.IP \[bu]
+\fBhex\fR (hex, optional): The hex data which has removed from the datastore
 .IP \[bu]
 \fBstring\fR (string, optional): The data as a string, if it's valid utf-8
 
@@ -53,4 +59,4 @@ Rusty Russell \fI<rusty@rustcorp.com.au\fR> is mainly responsible\.
 
 Main web site: \fIhttps://github.com/ElementsProject/lightning\fR
 
-\" SHA256STAMP:784f58fc76fc32b92d043b67b0b7efb88534dd29a7fabda2d705cdc0611e3c11
+\" SHA256STAMP:5450068ba0e62da8f46465b4f87cce4b4a59d064efbb13d03b24228790173dcc

doc/lightning-deldatastore.7.md

@@ -20,9 +20,10 @@ RETURN VALUE
 
 [comment]: # (GENERATE-FROM-SCHEMA-START)
 On success, an object is returned, containing:
-- **key** (string): The key which has been removed from the datastore
+- **key** (array of strings):
-- **generation** (u64): The number of times this has been updated
+  - Part of the key added to the datastore
-- **hex** (hex): The hex data which has removed from the datastore
+- **generation** (u64, optional): The number of times this has been updated
+- **hex** (hex, optional): The hex data which has removed from the datastore
 - **string** (string, optional): The data as a string, if it's valid utf-8
 [comment]: # (GENERATE-FROM-SCHEMA-END)
 
@@ -46,4 +47,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:ca2b7b8f45b3ecd6332978599c803e38c4f80945119a777cb8ae346cbf063b10)
+[comment]: # ( SHA256STAMP:cd6f944965165b0e276da493592f9decb15046150367e06ff3e5b5547517d4b9)

doc/lightning-listdatastore.7

@@ -11,8 +11,8 @@ The \fBlistdatastore\fR RPC command allows plugins to fetch data which was
 stored in the c-lightning database\.
 
 
-All entries are returned in \fIkey\fR isn't present; if \fIkey\fR is present,
+All immediate children of the \fIkey\fR (or root children) are returned:
-zero or one entries are returned\.
+a \fIkey\fR with children won't have a \fIhex\fR or \fIgeneration\fR entry\.
 
 .SH RETURN VALUE
 
@@ -20,11 +20,17 @@ On success, an object containing \fBdatastore\fR is returned\.  It is an array o
 
 .RS
 .IP \[bu]
-\fBkey\fR (string): The key which from the datastore
+\fBkey\fR (array of strings):
+.RS
 .IP \[bu]
-\fBgeneration\fR (u64): The number of times this has been updated
+Part of the key added to the datastore
+
+.RE
+
 .IP \[bu]
-\fBhex\fR (hex): The hex data from the datastore
+\fBgeneration\fR (u64, optional): The number of times this has been updated
+.IP \[bu]
+\fBhex\fR (hex, optional): The hex data from the datastore
 .IP \[bu]
 \fBstring\fR (string, optional): The data as a string, if it's valid utf-8
 
@@ -49,4 +55,4 @@ Rusty Russell \fI<rusty@rustcorp.com.au\fR> is mainly responsible\.
 
 Main web site: \fIhttps://github.com/ElementsProject/lightning\fR
 
-\" SHA256STAMP:b4128fc60690b3161eb76295e98f042c7be0142342bffa461c4f50f223c10684
+\" SHA256STAMP:bf75b8d32d0c95b4b5d5b9e8c220afc7c26cf8eef318facc9fb2923fdc3ab93b

doc/lightning-listdatastore.7.md

@@ -12,17 +12,18 @@ DESCRIPTION
 The **listdatastore** RPC command allows plugins to fetch data which was
 stored in the c-lightning database.
 
-All entries are returned in *key* isn't present; if *key* is present,
+All immediate children of the *key* (or root children) are returned:
-zero or one entries are returned.
+a *key* with children won't have a *hex* or *generation* entry.
 
 RETURN VALUE
 ------------
 
 [comment]: # (GENERATE-FROM-SCHEMA-START)
 On success, an object containing **datastore** is returned.  It is an array of objects, where each object contains:
-- **key** (string): The key which from the datastore
+- **key** (array of strings):
-- **generation** (u64): The number of times this has been updated
+  - Part of the key added to the datastore
-- **hex** (hex): The hex data from the datastore
+- **generation** (u64, optional): The number of times this has been updated
+- **hex** (hex, optional): The hex data from the datastore
 - **string** (string, optional): The data as a string, if it's valid utf-8
 [comment]: # (GENERATE-FROM-SCHEMA-END)
 
@@ -44,4 +45,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:a6503e3d2da8f9a35a0d461b5b93248f3fea306371ad62f98df613efea51959d)
+[comment]: # ( SHA256STAMP:ee29b53cad20c6dfe9e19a979816280cc9f778507e5638d1803418284125e4c1)

doc/schemas/datastore.schema.json

@@ -2,11 +2,14 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
     "additionalProperties": false,
-    "required": [ "key", "hex", "generation" ],
+    "required": [ "key" ],
     "properties": {
         "key": {
+	    "type": "array",
+	    "items": {
 		"type": "string",
-            "description": "The key which has been added to the datastore"
+		"description": "Part of the key added to the datastore"
+	    }
 	},
         "generation": {
             "type": "u64",

doc/schemas/deldatastore.schema.json

@@ -2,11 +2,14 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
     "additionalProperties": false,
-    "required": [ "key", "hex", "generation" ],
+    "required": [ "key" ],
     "properties": {
         "key": {
+	    "type": "array",
+	    "items": {
 		"type": "string",
-            "description": "The key which has been removed from the datastore"
+		"description": "Part of the key added to the datastore"
+	    }
 	},
         "generation": {
             "type": "u64",

doc/schemas/listdatastore.schema.json

@@ -9,11 +9,14 @@
 	    "items": {
 		"type": "object",
 		"additionalProperties": false,
-		"required": [ "key", "hex", "generation" ],
+		"required": [ "key" ],
 		"properties": {
 		    "key": {
+			"type": "array",
+			"items": {
 			    "type": "string",
-			"description": "The key which from the datastore"
+			    "description": "Part of the key added to the datastore"
+			}
 		    },
 		    "generation": {
 			"type": "u64",

lightningd/datastore.c

@@ -5,17 +5,24 @@
 #include <wallet/wallet.h>
 
 static void json_add_datastore(struct json_stream *response,
-			       const char *key, const u8 *data,
+			       const char **key, const u8 *data,
 			       u64 generation)
 {
+	json_array_start(response, "key");
+	for (size_t i = 0; i < tal_count(key); i++)
+		json_add_string(response, NULL, key[i]);
+	json_array_end(response);
+
+	if (data) {
 		const char *str;
-	json_add_string(response, "key", key);
+
 		json_add_u64(response, "generation", generation);
 		json_add_hex(response, "hex", data, tal_bytelen(data));
 		str = utf8_str(response, data, tal_bytelen(data));
 		if (str)
 			json_add_string(response, "string", str);
 	}
+}
 
 enum ds_mode {
 	DS_MUST_EXIST = 1,
@@ -51,19 +58,78 @@ static struct command_result *param_mode(struct command *cmd,
 	return NULL;
 }
 
+static struct command_result *param_list_or_string(struct command *cmd,
+						   const char *name,
+						   const char *buffer,
+						   const jsmntok_t *tok,
+						   const char ***str)
+{
+	if (tok->type == JSMN_ARRAY) {
+		size_t i;
+		const jsmntok_t *t;
+		*str = tal_arr(cmd, const char *, tok->size);
+		json_for_each_arr(i, t, tok) {
+			if (t->type != JSMN_STRING && t->type != JSMN_PRIMITIVE)
+				return command_fail_badparam(cmd, name,
+							     buffer, t,
+							     "should be string");
+			(*str)[i] = json_strdup(*str, buffer, t);
+		}
+	} else if (tok->type == JSMN_STRING || tok->type == JSMN_PRIMITIVE) {
+		*str = tal_arr(cmd, const char *, 1);
+		(*str)[0] = json_strdup(*str, buffer, tok);
+	} else
+		return command_fail_badparam(cmd, name,
+					     buffer, tok,
+					     "should be string or array");
+	return NULL;
+}
+
+/* Does k1 match k2 as far as k2 goes? */
+static bool datastore_key_startswith(const char **k1, const char **k2)
+{
+	size_t k1len = tal_count(k1), k2len = tal_count(k2);
+
+	if (k2len > k1len)
+		return false;
+
+	for (size_t i = 0; i < k2len; i++) {
+		if (!streq(k1[i], k2[i]))
+			return false;
+	}
+	return true;
+}
+
+static bool datastore_key_eq(const char **k1, const char **k2)
+{
+	return tal_count(k1) == tal_count(k2)
+		&& datastore_key_startswith(k1, k2);
+}
+
+static char *datastore_key_fmt(const tal_t *ctx, const char **key)
+{
+	char *ret = tal_strdup(ctx, "[");
+	for (size_t i = 0; i < tal_count(key); i++)
+		tal_append_fmt(&ret, "%s%s", i ? "," : "", key[i]);
+	tal_append_fmt(&ret, "]");
+	return ret;
+}
+
 static struct command_result *json_datastore(struct command *cmd,
 					     const char *buffer,
 					     const jsmntok_t *obj UNNEEDED,
 					     const jsmntok_t *params)
 {
 	struct json_stream *response;
-	const char *key, *strdata;
+	const char **key, *strdata, **k;
-	u8 *data, *prevdata;
+	u8 *data;
+	const u8 *prevdata;
 	enum ds_mode *mode;
 	u64 *generation, actual_gen;
+	struct db_stmt *stmt;
 
 	if (!param(cmd, buffer, params,
-		   p_req("key", param_string, &key),
+		   p_req("key", param_list_or_string, &key),
 		   p_opt("string", param_string, &strdata),
 		   p_opt("hex", param_bin_from_hex, &data),
 		   p_opt_def("mode", param_mode, &mode, DS_MUST_NOT_EXIST),
@@ -87,8 +153,40 @@ static struct command_result *json_datastore(struct command *cmd,
 				    "generation only valid with must-replace"
 				    " or must-append");
 
-	prevdata = wallet_datastore_fetch(cmd, cmd->ld->wallet, key,
+	/* Fetch, and make sure we don't have children! */
-					  &actual_gen);
+	stmt = wallet_datastore_first(cmd, cmd->ld->wallet, key,
+				      &k, &prevdata, &actual_gen);
+	tal_free(stmt);
+
+	/* We use prevdata as a "does it exist?" flag */
+	if (!stmt)
+		prevdata = NULL;
+	else if (!datastore_key_eq(k, key)) {
+		prevdata = tal_free(prevdata);
+		/* Make sure we don't have a child! */
+		if (datastore_key_startswith(k, key))
+			return command_fail(cmd, DATASTORE_UPDATE_HAS_CHILDREN,
+					    "Key has children already");
+	}
+
+	/* We have to make sure that parents don't exist. */
+	if (!prevdata) {
+		for (size_t i = 1; i < tal_count(key); i++) {
+			const char **parent;
+			parent = tal_dup_arr(cmd, const char *, key, i, 0);
+
+			stmt = wallet_datastore_first(cmd, cmd->ld->wallet,
+						      parent, &k, NULL, NULL);
+			tal_free(stmt);
+			if (stmt && datastore_key_eq(k, parent))
+				return command_fail(cmd,
+						    DATASTORE_UPDATE_NO_CHILDREN,
+						    "Parent key %s exists",
+						    datastore_key_fmt(tmpctx,
+								      parent));
+		}
+	}
+
 	if ((*mode & DS_MUST_NOT_EXIST) && prevdata)
 		return command_fail(cmd, DATASTORE_UPDATE_ALREADY_EXISTS,
 				    "Key already exists");
@@ -103,9 +201,10 @@ static struct command_result *json_datastore(struct command *cmd,
 
 	if ((*mode & DS_APPEND) && prevdata) {
 		size_t prevlen = tal_bytelen(prevdata);
-		tal_resize(&prevdata, prevlen + tal_bytelen(data));
+		u8 *newdata = tal_arr(cmd, u8, prevlen + tal_bytelen(data));
-		memcpy(prevdata + prevlen, data, tal_bytelen(data));
+		memcpy(newdata, prevdata, prevlen);
-		data = prevdata;
+		memcpy(newdata + prevlen, data, tal_bytelen(data));
+		data = newdata;
 	}
 
 	if (prevdata) {
@@ -127,36 +226,50 @@ static struct command_result *json_listdatastore(struct command *cmd,
 						 const jsmntok_t *params)
 {
 	struct json_stream *response;
-	const char *key;
+	const char **key, **k, **prev_k = NULL;
 	const u8 *data;
 	u64 generation;
+	struct db_stmt *stmt;
 
 	if (!param(cmd, buffer, params,
-		   p_opt("key", param_string, &key),
+		   p_opt("key", param_list_or_string, &key),
 		   NULL))
 		return command_param_failed();
 
+	if (key)
+		log_debug(cmd->ld->log, "Looking for %s",
+			  datastore_key_fmt(tmpctx, key));
+
 	response = json_stream_success(cmd);
 	json_array_start(response, "datastore");
-	if (key) {
-		data = wallet_datastore_fetch(cmd, cmd->ld->wallet, key,
-					      &generation);
-		if (data) {
-			json_object_start(response, NULL);
-			json_add_datastore(response, key, data, generation);
-			json_object_end(response);
-		}
-	} else {
-		struct db_stmt *stmt;
 
-		for (stmt = wallet_datastore_first(cmd, cmd->ld->wallet,
+	for (stmt = wallet_datastore_first(cmd, cmd->ld->wallet, key,
-						   &key, &data, &generation);
+					   &k, &data, &generation);
 	     stmt;
 	     stmt = wallet_datastore_next(cmd, cmd->ld->wallet,
-						  stmt, &key, &data,
+					  stmt, &k, &data,
 					  &generation)) {
+		log_debug(cmd->ld->log, "Got %s",
+			  datastore_key_fmt(tmpctx, k));
+
+		/* Don't list children, except implicitly */
+		if (tal_count(k) > tal_count(key) + 1) {
+			log_debug(cmd->ld->log, "Too long");
+			if (!prev_k || !datastore_key_startswith(k, prev_k)) {
+				prev_k = tal_dup_arr(cmd, const char *, k,
+						     tal_count(key) + 1, 0);
 				json_object_start(response, NULL);
-			json_add_datastore(response, key, data, generation);
+				json_add_datastore(response, prev_k, NULL, 0);
+				json_object_end(response);
+			}
+		} else if (key && !datastore_key_startswith(k, key)) {
+			log_debug(cmd->ld->log, "Not interested");
+			tal_free(stmt);
+			break;
+		} else {
+			log_debug(cmd->ld->log, "Printing");
+			json_object_start(response, NULL);
+			json_add_datastore(response, k, data, generation);
 			json_object_end(response);
 		}
 	}
@@ -170,28 +283,31 @@ static struct command_result *json_deldatastore(struct command *cmd,
 						 const jsmntok_t *params)
 {
 	struct json_stream *response;
-	const char *key;
+	const char **key, **k;
-	u8 *data;
+	const u8 *data;
 	u64 *generation;
 	u64 actual_gen;
+	struct db_stmt *stmt;
 
 	if (!param(cmd, buffer, params,
-		   p_req("key", param_string, &key),
+		   p_req("key", param_list_or_string, &key),
 		   p_opt("generation", param_u64, &generation),
 		   NULL))
 		return command_param_failed();
 
-	if (generation) {
+	stmt = wallet_datastore_first(cmd, cmd->ld->wallet, key,
-		data = wallet_datastore_fetch(cmd, cmd->ld->wallet, key,
+				      &k, &data, &actual_gen);
-					      &actual_gen);
+	tal_free(stmt);
-		if (data && actual_gen != *generation)
+
-			return command_fail(cmd, DATASTORE_DEL_WRONG_GENERATION,
+	if (!stmt || !datastore_key_eq(k, key)) {
-					    "generation is different");
-	}
-	data = wallet_datastore_remove(cmd, cmd->ld->wallet, key, &actual_gen);
-	if (!data)
 		return command_fail(cmd, DATASTORE_DEL_DOES_NOT_EXIST,
 				    "Key does not exist");
+	}
+	if (generation && actual_gen != *generation)
+		return command_fail(cmd, DATASTORE_DEL_WRONG_GENERATION,
+				    "generation is different");
+
+	wallet_datastore_remove(cmd->ld->wallet, key);
 
 	response = json_stream_success(cmd);
 	json_add_datastore(response, key, data, actual_gen);

tests/test_misc.py

@@ -2635,7 +2635,7 @@ def test_datastore(node_factory):
 
     # Add entries.
     somedata = b'somedata'.hex()
-    somedata_expect = {'key': 'somekey',
+    somedata_expect = {'key': ['somekey'],
                        'generation': 0,
                        'hex': somedata,
                        'string': 'somedata'}
@@ -2673,7 +2673,7 @@ def test_datastore(node_factory):
         l1.rpc.datastore(key='otherkey', hex=somedata, mode="must-append")
 
     otherdata = b'otherdata'.hex()
-    otherdata_expect = {'key': 'otherkey',
+    otherdata_expect = {'key': ['otherkey'],
                         'generation': 0,
                         'hex': otherdata,
                         'string': 'otherdata'}
@@ -2683,10 +2683,8 @@ def test_datastore(node_factory):
     assert l1.rpc.listdatastore('otherkey') == {'datastore': [otherdata_expect]}
     assert l1.rpc.listdatastore('badkey') == {'datastore': []}
 
-    ds = l1.rpc.listdatastore()
+    # Order is sorted!
-    # Order is undefined!
+    assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect, somedata_expect]}
-    assert (ds == {'datastore': [somedata_expect, otherdata_expect]}
-            or ds == {'datastore': [otherdata_expect, somedata_expect]})
 
     assert l1.rpc.deldatastore('somekey') == somedata_expect
     assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]}
@@ -2696,7 +2694,7 @@ def test_datastore(node_factory):
     assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]}
 
     # if it's not a string, won't print
-    badstring_expect = {'key': 'badstring',
+    badstring_expect = {'key': ['badstring'],
                         'generation': 0,
                         'hex': '00'}
     assert l1.rpc.datastore(key='badstring', hex='00') == badstring_expect
@@ -2731,3 +2729,65 @@ def test_datastore(node_factory):
                                 generation=otherdata_expect['generation'])
             == otherdata_expect)
     assert l1.rpc.listdatastore() == {'datastore': []}
+
+
+def test_datastore_keylist(node_factory):
+    l1 = node_factory.get_node()
+
+    # Starts empty
+    assert l1.rpc.listdatastore() == {'datastore': []}
+    assert l1.rpc.listdatastore(['a']) == {'datastore': []}
+    assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []}
+
+    # Cannot add child to existing!
+    l1.rpc.datastore(key='a', string='aval')
+    with pytest.raises(RpcError, match=r'1206.*Parent key \[a\] exists'):
+        l1.rpc.datastore(key=['a', 'b'], string='abval',
+                         mode='create-or-replace')
+    # Listing subkey gives DNE.
+    assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []}
+    l1.rpc.deldatastore(key=['a'])
+
+    # Create child key.
+    l1.rpc.datastore(key=['a', 'b'], string='abval')
+    assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
+    assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
+                                                              'generation': 0,
+                                                              'string': 'abval',
+                                                              'hex': b'abval'.hex()}]}
+
+    # Cannot create key over that
+    with pytest.raises(RpcError, match='has children'):
+        l1.rpc.datastore(key='a', string='aval', mode='create-or-replace')
+
+    # Can create another key.
+    l1.rpc.datastore(key=['a', 'b2'], string='ab2val')
+    assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
+    assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
+                                                              'string': 'abval',
+                                                              'generation': 0,
+                                                              'hex': b'abval'.hex()},
+                                                             {'key': ['a', 'b2'],
+                                                              'string': 'ab2val',
+                                                              'generation': 0,
+                                                              'hex': b'ab2val'.hex()}]}
+
+    # Can create subkey.
+    l1.rpc.datastore(key=['a', 'b3', 'c'], string='ab2val')
+    assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
+    assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
+                                                              'string': 'abval',
+                                                              'generation': 0,
+                                                              'hex': b'abval'.hex()},
+                                                             {'key': ['a', 'b2'],
+                                                              'string': 'ab2val',
+                                                              'generation': 0,
+                                                              'hex': b'ab2val'.hex()},
+                                                             {'key': ['a', 'b3']}]}
+
+    # Can update subkey
+    l1.rpc.datastore(key=['a', 'b3', 'c'], string='2', mode='must-append')
+    assert l1.rpc.listdatastore(key=['a', 'b3', 'c']) == {'datastore': [{'key': ['a', 'b3', 'c'],
+                                                                         'string': 'ab2val2',
+                                                                         'generation': 1,
+                                                                         'hex': b'ab2val2'.hex()}]}

wallet/db.c

@@ -750,7 +750,7 @@ static struct migration dbmigrations[] = {
      fillin_missing_channel_blockheights},
     {SQL("ALTER TABLE outputs ADD csv_lock INTEGER DEFAULT 1;"), NULL},
     {SQL("CREATE TABLE datastore ("
-	 "  key TEXT,"
+	 "  key BLOB,"
 	 "  data BLOB,"
 	 "  generation BIGINT,"
 	 "  PRIMARY KEY (key)"

wallet/db_postgres_sqlgen.c

@@ -1023,8 +1023,8 @@ struct db_query db_postgres_queries[] = {
          .readonly = false,
     },
     {
-         .name = "CREATE TABLE datastore (  key TEXT,  data BLOB,  generation BIGINT,  PRIMARY KEY (key));",
+         .name = "CREATE TABLE datastore (  key BLOB,  data BLOB,  generation BIGINT,  PRIMARY KEY (key));",
-         .query = "CREATE TABLE datastore (  key TEXT,  data BYTEA,  generation BIGINT,  PRIMARY KEY (key));",
+         .query = "CREATE TABLE datastore (  key BYTEA,  data BYTEA,  generation BIGINT,  PRIMARY KEY (key));",
          .placeholders = 0,
          .readonly = false,
     },
@@ -2025,14 +2025,14 @@ struct db_query db_postgres_queries[] = {
          .readonly = false,
     },
     {
-         .name = "SELECT data, generation  FROM datastore WHERE key = ?;",
+         .name = "SELECT key, data, generation FROM datastore WHERE key >= ? ORDER BY key;",
-         .query = "SELECT data, generation  FROM datastore WHERE key = $1;",
+         .query = "SELECT key, data, generation FROM datastore WHERE key >= $1 ORDER BY key;",
          .placeholders = 1,
          .readonly = true,
     },
     {
-         .name = "SELECT key, data, generation FROM datastore;",
+         .name = "SELECT key, data, generation FROM datastore ORDER BY key;",
-         .query = "SELECT key, data, generation FROM datastore;",
+         .query = "SELECT key, data, generation FROM datastore ORDER BY key;",
          .placeholders = 0,
          .readonly = true,
     },
@@ -2068,4 +2068,4 @@ struct db_query db_postgres_queries[] = {
 
 #endif /* LIGHTNINGD_WALLET_GEN_DB_POSTGRES */
 
-// SHA256STAMP:247577c68e8e536ee1736ddce425efb58ff609c1d3d4fbb1148b5c65b8922741
+// SHA256STAMP:1808964024bcccbd2787e723881f263b1a77ea33c302ac2b6d61dae20486a7e4

wallet/db_sqlite3_sqlgen.c

@@ -1023,8 +1023,8 @@ struct db_query db_sqlite3_queries[] = {
          .readonly = false,
     },
     {
-         .name = "CREATE TABLE datastore (  key TEXT,  data BLOB,  generation BIGINT,  PRIMARY KEY (key));",
+         .name = "CREATE TABLE datastore (  key BLOB,  data BLOB,  generation BIGINT,  PRIMARY KEY (key));",
-         .query = "CREATE TABLE datastore (  key TEXT,  data BLOB,  generation INTEGER,  PRIMARY KEY (key));",
+         .query = "CREATE TABLE datastore (  key BLOB,  data BLOB,  generation INTEGER,  PRIMARY KEY (key));",
          .placeholders = 0,
          .readonly = false,
     },
@@ -2025,14 +2025,14 @@ struct db_query db_sqlite3_queries[] = {
          .readonly = false,
     },
     {
-         .name = "SELECT data, generation  FROM datastore WHERE key = ?;",
+         .name = "SELECT key, data, generation FROM datastore WHERE key >= ? ORDER BY key;",
-         .query = "SELECT data, generation  FROM datastore WHERE key = ?;",
+         .query = "SELECT key, data, generation FROM datastore WHERE key >= ? ORDER BY key;",
          .placeholders = 1,
          .readonly = true,
     },
     {
-         .name = "SELECT key, data, generation FROM datastore;",
+         .name = "SELECT key, data, generation FROM datastore ORDER BY key;",
-         .query = "SELECT key, data, generation FROM datastore;",
+         .query = "SELECT key, data, generation FROM datastore ORDER BY key;",
          .placeholders = 0,
          .readonly = true,
     },
@@ -2068,4 +2068,4 @@ struct db_query db_sqlite3_queries[] = {
 
 #endif /* LIGHTNINGD_WALLET_GEN_DB_SQLITE3 */
 
-// SHA256STAMP:247577c68e8e536ee1736ddce425efb58ff609c1d3d4fbb1148b5c65b8922741
+// SHA256STAMP:1808964024bcccbd2787e723881f263b1a77ea33c302ac2b6d61dae20486a7e4

wallet/statements_gettextgen.po

@@ -675,7 +675,7 @@ msgid "ALTER TABLE outputs ADD csv_lock INTEGER DEFAULT 1;"
 msgstr ""
 
 #: wallet/db.c:752
-msgid "CREATE TABLE datastore (  key TEXT,  data BLOB,  generation BIGINT,  PRIMARY KEY (key));"
+msgid "CREATE TABLE datastore (  key BLOB,  data BLOB,  generation BIGINT,  PRIMARY KEY (key));"
 msgstr ""
 
 #: wallet/db.c:985
@@ -826,528 +826,528 @@ msgstr ""
 msgid "SELECT  state, payment_key, payment_hash, label, msatoshi, expiry_time, pay_index, msatoshi_received, paid_timestamp, bolt11, description, features, local_offer_id FROM invoices WHERE id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:66
+#: wallet/wallet.c:67
 msgid "SELECT txid, outnum FROM utxoset WHERE spendheight is NULL"
 msgstr ""
 
-#: wallet/wallet.c:107 wallet/wallet.c:611
+#: wallet/wallet.c:108 wallet/wallet.c:612
 msgid "SELECT * from outputs WHERE prev_out_tx=? AND prev_out_index=?"
 msgstr ""
 
-#: wallet/wallet.c:121
+#: wallet/wallet.c:122
 msgid "INSERT INTO outputs (  prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:239
+#: wallet/wallet.c:240
 msgid "UPDATE outputs SET status=? WHERE status=? AND prev_out_tx=? AND prev_out_index=?"
 msgstr ""
 
-#: wallet/wallet.c:247
+#: wallet/wallet.c:248
 msgid "UPDATE outputs SET status=? WHERE prev_out_tx=? AND prev_out_index=?"
 msgstr ""
 
-#: wallet/wallet.c:266
+#: wallet/wallet.c:267
 msgid "SELECT  prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey , reserved_til , csv_lock FROM outputs"
 msgstr ""
 
-#: wallet/wallet.c:284
+#: wallet/wallet.c:285
 msgid "SELECT  prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey , reserved_til , csv_lock FROM outputs WHERE status= ? "
 msgstr ""
 
-#: wallet/wallet.c:323
+#: wallet/wallet.c:324
 msgid "SELECT  prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey, reserved_til, csv_lock FROM outputs WHERE channel_id IS NOT NULL AND confirmation_height IS NULL"
 msgstr ""
 
-#: wallet/wallet.c:361
+#: wallet/wallet.c:362
 msgid "SELECT  prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey, reserved_til, csv_lock FROM outputs WHERE prev_out_tx = ? AND prev_out_index = ?"
 msgstr ""
 
-#: wallet/wallet.c:454
+#: wallet/wallet.c:455
 msgid "UPDATE outputs SET status=?, reserved_til=? WHERE prev_out_tx=? AND prev_out_index=?"
 msgstr ""
 
-#: wallet/wallet.c:557
+#: wallet/wallet.c:558
 msgid "SELECT  prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey , reserved_til, csv_lock FROM outputs WHERE status = ? OR (status = ? AND reserved_til <= ?)ORDER BY RANDOM();"
 msgstr ""
 
-#: wallet/wallet.c:625
+#: wallet/wallet.c:626
 msgid "INSERT INTO outputs (  prev_out_tx, prev_out_index, value, type, status, keyindex, channel_id, peer_id, commitment_point, option_anchor_outputs, confirmation_height, spend_height, scriptpubkey, csv_lock) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:732
+#: wallet/wallet.c:733
 msgid "INSERT INTO shachains (min_index, num_valid) VALUES (?, 0);"
 msgstr ""
 
-#: wallet/wallet.c:776
+#: wallet/wallet.c:777
 msgid "UPDATE shachains SET num_valid=?, min_index=? WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:783
+#: wallet/wallet.c:784
 msgid "UPDATE shachain_known SET idx=?, hash=? WHERE shachain_id=? AND pos=?"
 msgstr ""
 
-#: wallet/wallet.c:795
+#: wallet/wallet.c:796
 msgid "INSERT INTO shachain_known (shachain_id, pos, idx, hash) VALUES (?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:817
+#: wallet/wallet.c:818
 msgid "SELECT min_index, num_valid FROM shachains WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:832
+#: wallet/wallet.c:833
 msgid "SELECT idx, hash, pos FROM shachain_known WHERE shachain_id=?"
 msgstr ""
 
-#: wallet/wallet.c:855
+#: wallet/wallet.c:856
 msgid "SELECT id, node_id, address FROM peers WHERE id=?;"
 msgstr ""
 
-#: wallet/wallet.c:888
+#: wallet/wallet.c:889
 msgid "SELECT signature FROM htlc_sigs WHERE channelid = ?"
 msgstr ""
 
-#: wallet/wallet.c:922
+#: wallet/wallet.c:923
 msgid "SELECT remote_ann_node_sig, remote_ann_bitcoin_sig FROM channels WHERE id = ?"
 msgstr ""
 
-#: wallet/wallet.c:966
+#: wallet/wallet.c:967
 msgid "SELECT hstate, feerate_per_kw FROM channel_feerates WHERE channel_id = ?"
 msgstr ""
 
-#: wallet/wallet.c:1002
+#: wallet/wallet.c:1003
 msgid "SELECT hstate, blockheight FROM channel_blockheights WHERE channel_id = ?"
 msgstr ""
 
-#: wallet/wallet.c:1035
+#: wallet/wallet.c:1036
 msgid "INSERT INTO channel_funding_inflights (  channel_id, funding_tx_id, funding_tx_outnum, funding_feerate, funding_satoshi, our_funding_satoshi, funding_psbt, last_tx, last_sig, lease_commit_sig, lease_chan_max_msat, lease_chan_max_ppt, lease_expiry, lease_blockheight_start) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:1091
+#: wallet/wallet.c:1092
 msgid "UPDATE channel_funding_inflights SET  funding_psbt=?, funding_tx_remote_sigs_received=? WHERE  channel_id=? AND funding_tx_id=? AND funding_tx_outnum=?"
 msgstr ""
 
-#: wallet/wallet.c:1114
+#: wallet/wallet.c:1115
 msgid "DELETE FROM channel_funding_inflights WHERE channel_id = ?"
 msgstr ""
 
-#: wallet/wallet.c:1184
+#: wallet/wallet.c:1185
 msgid "SELECT  funding_tx_id, funding_tx_outnum, funding_feerate, funding_satoshi, our_funding_satoshi, funding_psbt, last_tx, last_sig, funding_tx_remote_sigs_received, lease_expiry, lease_commit_sig, lease_chan_max_msat, lease_chan_max_ppt, lease_blockheight_start FROM channel_funding_inflights WHERE channel_id = ? ORDER BY funding_feerate"
 msgstr ""
 
-#: wallet/wallet.c:1452
+#: wallet/wallet.c:1453
 msgid "SELECT id FROM channels ORDER BY id DESC LIMIT 1;"
 msgstr ""
 
-#: wallet/wallet.c:1469
+#: wallet/wallet.c:1470
 msgid "SELECT  id, peer_id, short_channel_id, full_channel_id, channel_config_local, channel_config_remote, state, funder, channel_flags, minimum_depth, next_index_local, next_index_remote, next_htlc_id, funding_tx_id, funding_tx_outnum, funding_satoshi, our_funding_satoshi, funding_locked_remote, push_msatoshi, msatoshi_local, fundingkey_remote, revocation_basepoint_remote, payment_basepoint_remote, htlc_basepoint_remote, delayed_payment_basepoint_remote, per_commit_remote, old_per_commit_remote, local_feerate_per_kw, remote_feerate_per_kw, shachain_remote_id, shutdown_scriptpubkey_remote, shutdown_keyidx_local, last_sent_commit_state, last_sent_commit_id, last_tx, last_sig, last_was_revoke, first_blocknum, min_possible_feerate, max_possible_feerate, msatoshi_to_us_min, msatoshi_to_us_max, future_per_commitment_point, last_sent_commit, feerate_base, feerate_ppm, remote_upfront_shutdown_script, local_static_remotekey_start, remote_static_remotekey_start, option_anchor_outputs, shutdown_scriptpubkey_local, closer, state_change_reason, revocation_basepoint_local, payment_basepoint_local, htlc_basepoint_local, delayed_payment_basepoint_local, funding_pubkey_local, shutdown_wrong_txid, shutdown_wrong_outnum, lease_expiry, lease_commit_sig, lease_chan_max_msat, lease_chan_max_ppt FROM channels WHERE state != ?;"
 msgstr ""
 
-#: wallet/wallet.c:1581
+#: wallet/wallet.c:1582
 msgid "UPDATE channels   SET in_payments_offered = COALESCE(in_payments_offered, 0) + 1     , in_msatoshi_offered = COALESCE(in_msatoshi_offered, 0) + ? WHERE id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:1587
+#: wallet/wallet.c:1588
 msgid "UPDATE channels   SET in_payments_fulfilled = COALESCE(in_payments_fulfilled, 0) + 1     , in_msatoshi_fulfilled = COALESCE(in_msatoshi_fulfilled, 0) + ? WHERE id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:1593
+#: wallet/wallet.c:1594
 msgid "UPDATE channels   SET out_payments_offered = COALESCE(out_payments_offered, 0) + 1     , out_msatoshi_offered = COALESCE(out_msatoshi_offered, 0) + ? WHERE id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:1599
+#: wallet/wallet.c:1600
 msgid "UPDATE channels   SET out_payments_fulfilled = COALESCE(out_payments_fulfilled, 0) + 1     , out_msatoshi_fulfilled = COALESCE(out_msatoshi_fulfilled, 0) + ? WHERE id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:1644
+#: wallet/wallet.c:1645
 msgid "SELECT   in_payments_offered,  in_payments_fulfilled,  in_msatoshi_offered,  in_msatoshi_fulfilled, out_payments_offered, out_payments_fulfilled, out_msatoshi_offered, out_msatoshi_fulfilled  FROM channels WHERE id = ?"
 msgstr ""
 
-#: wallet/wallet.c:1673
+#: wallet/wallet.c:1674
 msgid "SELECT MIN(height), MAX(height) FROM blocks;"
 msgstr ""
 
-#: wallet/wallet.c:1695
+#: wallet/wallet.c:1696
 msgid "INSERT INTO channel_configs DEFAULT VALUES;"
 msgstr ""
 
-#: wallet/wallet.c:1707
+#: wallet/wallet.c:1708
 msgid "UPDATE channel_configs SET  dust_limit_satoshis=?,  max_htlc_value_in_flight_msat=?,  channel_reserve_satoshis=?,  htlc_minimum_msat=?,  to_self_delay=?,  max_accepted_htlcs=? WHERE id=?;"
 msgstr ""
 
-#: wallet/wallet.c:1731
+#: wallet/wallet.c:1732
 msgid "SELECT id, dust_limit_satoshis, max_htlc_value_in_flight_msat, channel_reserve_satoshis, htlc_minimum_msat, to_self_delay, max_accepted_htlcs FROM channel_configs WHERE id= ? ;"
 msgstr ""
 
-#: wallet/wallet.c:1765
+#: wallet/wallet.c:1766
 msgid "UPDATE channels SET  remote_ann_node_sig=?,  remote_ann_bitcoin_sig=? WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:1784
+#: wallet/wallet.c:1785
 msgid "UPDATE channels SET  shachain_remote_id=?,  short_channel_id=?,  full_channel_id=?,  state=?,  funder=?,  channel_flags=?,  minimum_depth=?,  next_index_local=?,  next_index_remote=?,  next_htlc_id=?,  funding_tx_id=?,  funding_tx_outnum=?,  funding_satoshi=?,  our_funding_satoshi=?,  funding_locked_remote=?,  push_msatoshi=?,  msatoshi_local=?,  shutdown_scriptpubkey_remote=?,  shutdown_keyidx_local=?,  channel_config_local=?,  last_tx=?, last_sig=?,  last_was_revoke=?,  min_possible_feerate=?,  max_possible_feerate=?,  msatoshi_to_us_min=?,  msatoshi_to_us_max=?,  feerate_base=?,  feerate_ppm=?,  remote_upfront_shutdown_script=?,  local_static_remotekey_start=?,  remote_static_remotekey_start=?,  option_anchor_outputs=?,  shutdown_scriptpubkey_local=?,  closer=?,  state_change_reason=?,  shutdown_wrong_txid=?,  shutdown_wrong_outnum=?,  lease_expiry=?,  lease_commit_sig=?,  lease_chan_max_msat=?,  lease_chan_max_ppt=? WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:1893
+#: wallet/wallet.c:1894
 msgid "UPDATE channels SET  fundingkey_remote=?,  revocation_basepoint_remote=?,  payment_basepoint_remote=?,  htlc_basepoint_remote=?,  delayed_payment_basepoint_remote=?,  per_commit_remote=?,  old_per_commit_remote=?,  channel_config_remote=?,  future_per_commitment_point=? WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:1920
+#: wallet/wallet.c:1921
 msgid "DELETE FROM channel_feerates WHERE channel_id=?"
 msgstr ""
 
-#: wallet/wallet.c:1930
+#: wallet/wallet.c:1931
 msgid "INSERT INTO channel_feerates  VALUES(?, ?, ?)"
 msgstr ""
 
-#: wallet/wallet.c:1939
+#: wallet/wallet.c:1940
 msgid "DELETE FROM channel_blockheights WHERE channel_id=?"
 msgstr ""
 
-#: wallet/wallet.c:1949
+#: wallet/wallet.c:1950
 msgid "INSERT INTO channel_blockheights  VALUES(?, ?, ?)"
 msgstr ""
 
-#: wallet/wallet.c:1966
+#: wallet/wallet.c:1967
 msgid "UPDATE channels SET  last_sent_commit=? WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:1989
+#: wallet/wallet.c:1990
 msgid "INSERT INTO channel_state_changes (  channel_id, timestamp, old_state, new_state, cause, message) VALUES (?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:2017
+#: wallet/wallet.c:2018
 msgid "SELECT timestamp, old_state, new_state, cause, message FROM channel_state_changes WHERE channel_id = ? ORDER BY timestamp ASC;"
 msgstr ""
 
-#: wallet/wallet.c:2046
+#: wallet/wallet.c:2047
 msgid "SELECT id FROM peers WHERE node_id = ?"
 msgstr ""
 
-#: wallet/wallet.c:2058
+#: wallet/wallet.c:2059
 msgid "UPDATE peers SET address = ? WHERE id = ?"
 msgstr ""
 
-#: wallet/wallet.c:2067
+#: wallet/wallet.c:2068
 msgid "INSERT INTO peers (node_id, address) VALUES (?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:2088
+#: wallet/wallet.c:2089
 msgid "INSERT INTO channels (  peer_id, first_blocknum, id, revocation_basepoint_local, payment_basepoint_local, htlc_basepoint_local, delayed_payment_basepoint_local, funding_pubkey_local) VALUES (?, ?, ?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:2129
+#: wallet/wallet.c:2130
 msgid "DELETE FROM channel_htlcs WHERE channel_id=?"
 msgstr ""
 
-#: wallet/wallet.c:2135
+#: wallet/wallet.c:2136
 msgid "DELETE FROM htlc_sigs WHERE channelid=?"
 msgstr ""
 
-#: wallet/wallet.c:2141
+#: wallet/wallet.c:2142
 msgid "DELETE FROM channeltxs WHERE channel_id=?"
 msgstr ""
 
-#: wallet/wallet.c:2148
+#: wallet/wallet.c:2149
 msgid "DELETE FROM channel_funding_inflights  WHERE channel_id=?"
 msgstr ""
 
-#: wallet/wallet.c:2154
+#: wallet/wallet.c:2155
 msgid "DELETE FROM shachains WHERE id IN (  SELECT shachain_remote_id   FROM channels   WHERE channels.id=?)"
 msgstr ""
 
-#: wallet/wallet.c:2164
+#: wallet/wallet.c:2165
 msgid "UPDATE channels SET state=?, peer_id=? WHERE channels.id=?"
 msgstr ""
 
-#: wallet/wallet.c:2178
+#: wallet/wallet.c:2179
 msgid "SELECT * FROM channels WHERE peer_id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:2186
+#: wallet/wallet.c:2187
 msgid "DELETE FROM peers WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:2197
+#: wallet/wallet.c:2198
 msgid "UPDATE outputs SET confirmation_height = ? WHERE prev_out_tx = ?"
 msgstr ""
 
-#: wallet/wallet.c:2300
+#: wallet/wallet.c:2301
 msgid "INSERT INTO channel_htlcs ( channel_id, channel_htlc_id,  direction, msatoshi, cltv_expiry, payment_hash,  payment_key, hstate, shared_secret, routing_onion, received_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:2353
+#: wallet/wallet.c:2354
 msgid "INSERT INTO channel_htlcs ( channel_id, channel_htlc_id, direction, origin_htlc, msatoshi, cltv_expiry, payment_hash, payment_key, hstate, routing_onion, malformed_onion, partid) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?);"
 msgstr ""
 
-#: wallet/wallet.c:2414
+#: wallet/wallet.c:2415
 msgid "UPDATE channel_htlcs SET hstate=?, payment_key=?, malformed_onion=?, failuremsg=?, localfailmsg=?, we_filled=? WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:2631
+#: wallet/wallet.c:2632
 msgid "SELECT  id, channel_htlc_id, msatoshi, cltv_expiry, hstate, payment_hash, payment_key, routing_onion, failuremsg, malformed_onion, origin_htlc, shared_secret, received_time, we_filled FROM channel_htlcs WHERE direction= ? AND channel_id= ? AND hstate != ?"
 msgstr ""
 
-#: wallet/wallet.c:2678
+#: wallet/wallet.c:2679
 msgid "SELECT  id, channel_htlc_id, msatoshi, cltv_expiry, hstate, payment_hash, payment_key, routing_onion, failuremsg, malformed_onion, origin_htlc, shared_secret, received_time, partid, localfailmsg FROM channel_htlcs WHERE direction = ? AND channel_id = ? AND hstate != ?"
 msgstr ""
 
-#: wallet/wallet.c:2809
+#: wallet/wallet.c:2810
 msgid "SELECT channel_id, direction, cltv_expiry, channel_htlc_id, payment_hash FROM channel_htlcs WHERE channel_id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:2843
+#: wallet/wallet.c:2844
 msgid "DELETE FROM channel_htlcs WHERE direction = ? AND origin_htlc = ? AND payment_hash = ? AND partid = ?;"
 msgstr ""
 
-#: wallet/wallet.c:2896
+#: wallet/wallet.c:2897
 msgid "SELECT status FROM payments WHERE payment_hash=? AND partid = ?;"
 msgstr ""
 
-#: wallet/wallet.c:2914
+#: wallet/wallet.c:2915
 msgid "INSERT INTO payments (  status,  payment_hash,  destination,  msatoshi,  timestamp,  path_secrets,  route_nodes,  route_channels,  msatoshi_sent,  description,  bolt11,  total_msat,  partid,  local_offer_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:3003
+#: wallet/wallet.c:3004
 msgid "DELETE FROM payments WHERE payment_hash = ? AND partid = ?"
 msgstr ""
 
-#: wallet/wallet.c:3017
+#: wallet/wallet.c:3018
 msgid "DELETE FROM payments WHERE payment_hash = ?"
 msgstr ""
 
-#: wallet/wallet.c:3118
+#: wallet/wallet.c:3119
 msgid "SELECT  id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ? AND partid = ?"
 msgstr ""
 
-#: wallet/wallet.c:3168
+#: wallet/wallet.c:3169
 msgid "UPDATE payments SET status=? WHERE payment_hash=? AND partid=?"
 msgstr ""
 
-#: wallet/wallet.c:3178
+#: wallet/wallet.c:3179
 msgid "UPDATE payments SET payment_preimage=? WHERE payment_hash=? AND partid=?"
 msgstr ""
 
-#: wallet/wallet.c:3188
+#: wallet/wallet.c:3189
 msgid "UPDATE payments   SET path_secrets = NULL     , route_nodes = NULL     , route_channels = NULL WHERE payment_hash = ? AND partid = ?;"
 msgstr ""
 
-#: wallet/wallet.c:3220
+#: wallet/wallet.c:3221
 msgid "SELECT failonionreply, faildestperm, failindex, failcode, failnode, failchannel, failupdate, faildetail, faildirection  FROM payments WHERE payment_hash=? AND partid=?;"
 msgstr ""
 
-#: wallet/wallet.c:3287
+#: wallet/wallet.c:3288
 msgid "UPDATE payments   SET failonionreply=?     , faildestperm=?     , failindex=?     , failcode=?     , failnode=?     , failchannel=?     , failupdate=?     , faildetail=?     , faildirection=? WHERE payment_hash=? AND partid=?;"
 msgstr ""
 
-#: wallet/wallet.c:3346
+#: wallet/wallet.c:3347
 msgid "SELECT  id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ? ORDER BY id;"
 msgstr ""
 
-#: wallet/wallet.c:3369
+#: wallet/wallet.c:3370
 msgid "SELECT  id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments ORDER BY id;"
 msgstr ""
 
-#: wallet/wallet.c:3420
+#: wallet/wallet.c:3421
 msgid "SELECT  id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE local_offer_id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:3465
+#: wallet/wallet.c:3466
 msgid "DELETE FROM htlc_sigs WHERE channelid = ?"
 msgstr ""
 
-#: wallet/wallet.c:3472
+#: wallet/wallet.c:3473
 msgid "INSERT INTO htlc_sigs (channelid, signature) VALUES (?, ?)"
 msgstr ""
 
-#: wallet/wallet.c:3484
+#: wallet/wallet.c:3485
 msgid "SELECT blobval FROM vars WHERE name='genesis_hash'"
 msgstr ""
 
-#: wallet/wallet.c:3508
+#: wallet/wallet.c:3509
 msgid "INSERT INTO vars (name, blobval) VALUES ('genesis_hash', ?);"
 msgstr ""
 
-#: wallet/wallet.c:3526
+#: wallet/wallet.c:3527
 msgid "SELECT txid, outnum FROM utxoset WHERE spendheight < ?"
 msgstr ""
 
-#: wallet/wallet.c:3538
+#: wallet/wallet.c:3539
 msgid "DELETE FROM utxoset WHERE spendheight < ?"
 msgstr ""
 
-#: wallet/wallet.c:3546 wallet/wallet.c:3660
+#: wallet/wallet.c:3547 wallet/wallet.c:3661
 msgid "INSERT INTO blocks (height, hash, prev_hash) VALUES (?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:3565
+#: wallet/wallet.c:3566
 msgid "DELETE FROM blocks WHERE hash = ?"
 msgstr ""
 
-#: wallet/wallet.c:3571
+#: wallet/wallet.c:3572
 msgid "SELECT * FROM blocks WHERE height >= ?;"
 msgstr ""
 
-#: wallet/wallet.c:3580
+#: wallet/wallet.c:3581
 msgid "DELETE FROM blocks WHERE height > ?"
 msgstr ""
 
-#: wallet/wallet.c:3592
+#: wallet/wallet.c:3593
 msgid "UPDATE outputs SET spend_height = ?,  status = ? WHERE prev_out_tx = ? AND prev_out_index = ?"
 msgstr ""
 
-#: wallet/wallet.c:3610
+#: wallet/wallet.c:3611
 msgid "UPDATE utxoset SET spendheight = ? WHERE txid = ? AND outnum = ?"
 msgstr ""
 
-#: wallet/wallet.c:3633 wallet/wallet.c:3671
+#: wallet/wallet.c:3634 wallet/wallet.c:3672
 msgid "INSERT INTO utxoset ( txid, outnum, blockheight, spendheight, txindex, scriptpubkey, satoshis) VALUES(?, ?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:3697
+#: wallet/wallet.c:3698
 msgid "SELECT height FROM blocks WHERE height = ?"
 msgstr ""
 
-#: wallet/wallet.c:3710
+#: wallet/wallet.c:3711
 msgid "SELECT txid, spendheight, scriptpubkey, satoshis FROM utxoset WHERE blockheight = ? AND txindex = ? AND outnum = ? AND spendheight IS NULL"
 msgstr ""
 
-#: wallet/wallet.c:3774
+#: wallet/wallet.c:3775
 msgid "SELECT blockheight, txindex, outnum FROM utxoset WHERE spendheight = ?"
 msgstr ""
 
-#: wallet/wallet.c:3791
+#: wallet/wallet.c:3792
 msgid "SELECT blockheight, txindex, outnum FROM utxoset WHERE blockheight = ?"
 msgstr ""
 
-#: wallet/wallet.c:3808 wallet/wallet.c:3968
+#: wallet/wallet.c:3809 wallet/wallet.c:3969
 msgid "SELECT blockheight FROM transactions WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:3818
+#: wallet/wallet.c:3819
 msgid "INSERT INTO transactions (  id, blockheight, txindex, rawtx) VALUES (?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:3839
+#: wallet/wallet.c:3840
 msgid "UPDATE transactions SET blockheight = ?, txindex = ? WHERE id = ?"
 msgstr ""
 
-#: wallet/wallet.c:3856
+#: wallet/wallet.c:3857
 msgid "INSERT INTO transaction_annotations (txid, idx, location, type, channel) VALUES (?, ?, ?, ?, ?) ON CONFLICT(txid,idx) DO NOTHING;"
 msgstr ""
 
-#: wallet/wallet.c:3888
+#: wallet/wallet.c:3889
 msgid "SELECT type, channel_id FROM transactions WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:3904
+#: wallet/wallet.c:3905
 msgid "UPDATE transactions SET type = ?, channel_id = ? WHERE id = ?"
 msgstr ""
 
-#: wallet/wallet.c:3923
+#: wallet/wallet.c:3924
 msgid "SELECT type FROM transactions WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:3946
+#: wallet/wallet.c:3947
 msgid "SELECT rawtx FROM transactions WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:3992
+#: wallet/wallet.c:3993
 msgid "SELECT blockheight, txindex FROM transactions WHERE id=?"
 msgstr ""
 
-#: wallet/wallet.c:4020
+#: wallet/wallet.c:4021
 msgid "SELECT id FROM transactions WHERE blockheight=?"
 msgstr ""
 
-#: wallet/wallet.c:4039
+#: wallet/wallet.c:4040
 msgid "INSERT INTO channeltxs (  channel_id, type, transaction_id, input_num, blockheight) VALUES (?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:4063
+#: wallet/wallet.c:4064
 msgid "SELECT DISTINCT(channel_id) FROM channeltxs WHERE type = ?;"
 msgstr ""
 
-#: wallet/wallet.c:4084
+#: wallet/wallet.c:4085
 msgid "SELECT  c.type, c.blockheight, t.rawtx, c.input_num, c.blockheight - t.blockheight + 1 AS depth, t.id as txid FROM channeltxs c JOIN transactions t ON t.id = c.transaction_id WHERE c.channel_id = ? ORDER BY c.id ASC;"
 msgstr ""
 
-#: wallet/wallet.c:4129
+#: wallet/wallet.c:4130
 msgid "UPDATE forwarded_payments SET  in_msatoshi=?, out_msatoshi=?, state=?, resolved_time=?, failcode=? WHERE in_htlc_id=?"
 msgstr ""
 
-#: wallet/wallet.c:4187
+#: wallet/wallet.c:4188
 msgid "INSERT INTO forwarded_payments (  in_htlc_id, out_htlc_id, in_channel_scid, out_channel_scid, in_msatoshi, out_msatoshi, state, received_time, resolved_time, failcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:4246
+#: wallet/wallet.c:4247
 msgid "SELECT CAST(COALESCE(SUM(in_msatoshi - out_msatoshi), 0) AS BIGINT)FROM forwarded_payments WHERE state = ?;"
 msgstr ""
 
-#: wallet/wallet.c:4295
+#: wallet/wallet.c:4296
 msgid "SELECT  f.state, in_msatoshi, out_msatoshi, hin.payment_hash as payment_hash, in_channel_scid, out_channel_scid, f.received_time, f.resolved_time, f.failcode FROM forwarded_payments f LEFT JOIN channel_htlcs hin ON (f.in_htlc_id = hin.id) WHERE (1 = ? OR f.state = ?) AND (1 = ? OR f.in_channel_scid = ?) AND (1 = ? OR f.out_channel_scid = ?)"
 msgstr ""
 
-#: wallet/wallet.c:4417
+#: wallet/wallet.c:4418
 msgid "SELECT  t.id, t.rawtx, t.blockheight, t.txindex, t.type as txtype, c2.short_channel_id as txchan, a.location, a.idx as ann_idx, a.type as annotation_type, c.short_channel_id FROM  transactions t LEFT JOIN  transaction_annotations a ON (a.txid = t.id) LEFT JOIN  channels c ON (a.channel = c.id) LEFT JOIN  channels c2 ON (t.channel_id = c2.id) ORDER BY t.blockheight, t.txindex ASC"
 msgstr ""
 
-#: wallet/wallet.c:4511
+#: wallet/wallet.c:4512
 msgid "INSERT INTO penalty_bases (  channel_id, commitnum, txid, outnum, amount) VALUES (?, ?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:4536
+#: wallet/wallet.c:4537
 msgid "SELECT commitnum, txid, outnum, amount FROM penalty_bases WHERE channel_id = ?"
 msgstr ""
 
-#: wallet/wallet.c:4560
+#: wallet/wallet.c:4561
 msgid "DELETE FROM penalty_bases WHERE channel_id = ? AND commitnum = ?"
 msgstr ""
 
-#: wallet/wallet.c:4578
+#: wallet/wallet.c:4579
 msgid "SELECT 1  FROM offers WHERE offer_id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:4591
+#: wallet/wallet.c:4592
 msgid "INSERT INTO offers (  offer_id, bolt12, label, status) VALUES (?, ?, ?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:4618
+#: wallet/wallet.c:4619
 msgid "SELECT bolt12, label, status  FROM offers WHERE offer_id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:4646
+#: wallet/wallet.c:4647
 msgid "SELECT offer_id FROM offers;"
 msgstr ""
 
-#: wallet/wallet.c:4672
+#: wallet/wallet.c:4673
 msgid "UPDATE offers SET status=? WHERE offer_id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:4683
+#: wallet/wallet.c:4684
 msgid "UPDATE invoices SET state=? WHERE state=? AND local_offer_id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:4711
+#: wallet/wallet.c:4712
 msgid "SELECT status  FROM offers WHERE offer_id = ?;"
 msgstr ""
 
-#: wallet/wallet.c:4746
+#: wallet/wallet.c:4797
 msgid "UPDATE datastore SET data=?, generation=generation+1 WHERE key=?;"
 msgstr ""
 
-#: wallet/wallet.c:4757
+#: wallet/wallet.c:4808
 msgid "INSERT INTO datastore VALUES (?, ?, 0);"
 msgstr ""
 
-#: wallet/wallet.c:4771
+#: wallet/wallet.c:4819
 msgid "DELETE FROM datastore WHERE key = ?"
 msgstr ""
 
-#: wallet/wallet.c:4787
+#: wallet/wallet.c:4836
-msgid "SELECT data, generation  FROM datastore WHERE key = ?;"
+msgid "SELECT key, data, generation FROM datastore WHERE key >= ? ORDER BY key;"
 msgstr ""
 
-#: wallet/wallet.c:4812
+#: wallet/wallet.c:4843
-msgid "SELECT key, data, generation FROM datastore;"
+msgid "SELECT key, data, generation FROM datastore ORDER BY key;"
 msgstr ""
 
 #: wallet/test/run-db.c:126
@@ -1365,4 +1365,4 @@ msgstr ""
 #: wallet/test/run-wallet.c:1753
 msgid "INSERT INTO channels (id) VALUES (1);"
 msgstr ""
-#  SHA256STAMP:f68886ac022d1170ef8a4e137a6f4fedea23a7cd06a735418e5f635bb4224a58
+#  SHA256STAMP:e7f23b938c7ee86b0178ca11d8d3df3f08dec52e205e0778be1f5a0b607f52f6

wallet/wallet.c

@@ -4,6 +4,7 @@
 #include <bitcoin/psbt.h>
 #include <bitcoin/script.h>
 #include <ccan/array_size/array_size.h>
+#include <ccan/cast/cast.h>
 #include <ccan/mem/mem.h>
 #include <ccan/tal/str/str.h>
 #include <common/blockheight_states.h>
@@ -4738,78 +4739,111 @@ void wallet_offer_mark_used(struct db *db, const struct sha256 *offer_id)
 	}
 }
 
-void wallet_datastore_update(struct wallet *w, const char *key, const u8 *data)
+
+/* We join key parts with nuls for now. */
+static void db_bind_datastore_key(struct db_stmt *stmt,
+				  int pos,
+				  const char **key)
+{
+	u8 *joined;
+	size_t len;
+
+	if (tal_count(key) == 1) {
+		db_bind_text(stmt, pos, key[0]);
+		return;
+	}
+
+	len = strlen(key[0]);
+	joined = (u8 *)tal_strdup(tmpctx, key[0]);
+	for (size_t i = 1; i < tal_count(key); i++) {
+		tal_resize(&joined, len + 1 + strlen(key[i]));
+		joined[len] = '\0';
+		memcpy(joined + len + 1, key[i], strlen(key[i]));
+		len += 1 + strlen(key[i]);
+	}
+	db_bind_blob(stmt, pos, joined, len);
+}
+
+static const char **db_column_datastore_key(const tal_t *ctx,
+					    struct db_stmt *stmt,
+					    int col)
+{
+	char **key;
+	const u8 *joined = db_column_blob(stmt, col);
+	size_t len = db_column_bytes(stmt, col);
+
+	key = tal_arr(ctx, char *, 0);
+	do {
+		size_t partlen;
+		for (partlen = 0; partlen < len; partlen++) {
+			if (joined[partlen] == '\0') {
+				partlen++;
+				break;
+			}
+		}
+		tal_arr_expand(&key, tal_strndup(key, (char *)joined, partlen));
+		len -= partlen;
+		joined += partlen;
+	} while (len != 0);
+
+	return cast_const2(const char **, key);
+}
+
+void wallet_datastore_update(struct wallet *w, const char **key, const u8 *data)
 {
 	struct db_stmt *stmt;
 
 	stmt = db_prepare_v2(w->db,
 			     SQL("UPDATE datastore SET data=?, generation=generation+1 WHERE key=?;"));
 	db_bind_talarr(stmt, 0, data);
-	db_bind_text(stmt, 1, key);
+	db_bind_datastore_key(stmt, 1, key);
 	db_exec_prepared_v2(take(stmt));
 }
 
-void wallet_datastore_create(struct wallet *w, const char *key, const u8 *data)
+void wallet_datastore_create(struct wallet *w, const char **key, const u8 *data)
 {
 	struct db_stmt *stmt;
 
 	stmt = db_prepare_v2(w->db,
 			     SQL("INSERT INTO datastore VALUES (?, ?, 0);"));
 
-	db_bind_text(stmt, 0, key);
+	db_bind_datastore_key(stmt, 0, key);
 	db_bind_talarr(stmt, 1, data);
 	db_exec_prepared_v2(take(stmt));
 }
 
-u8 *wallet_datastore_remove(const tal_t *ctx, struct wallet *w, const char *key,
+void wallet_datastore_remove(struct wallet *w, const char **key)
-			    u64 *generation)
 {
-	u8 *data = wallet_datastore_fetch(ctx, w, key, generation);
-	if (data) {
 	struct db_stmt *stmt;
 
 	stmt = db_prepare_v2(w->db, SQL("DELETE FROM datastore"
 					" WHERE key = ?"));
-		db_bind_text(stmt, 0, key);
+	db_bind_datastore_key(stmt, 0, key);
 	db_exec_prepared_v2(take(stmt));
 }
-	return data;
-}
-
-u8 *wallet_datastore_fetch(const tal_t *ctx,
-			   struct wallet *w, const char *key,
-			   u64 *generation)
-{
-	struct db_stmt *stmt;
-	u8 *data;
-
-	/* Test if already exists. */
-	stmt = db_prepare_v2(w->db, SQL("SELECT data, generation"
-					"  FROM datastore"
-					" WHERE key = ?;"));
-	db_bind_text(stmt, 0, key);
-	db_query_prepared(stmt);
-
-	if (db_step(stmt)) {
-		data = db_column_talarr(ctx, stmt, 0);
-		if (generation)
-			*generation = db_column_u64(stmt, 1);
-	} else
-		data = NULL;
-	tal_free(stmt);
-	return data;
-}
 
 struct db_stmt *wallet_datastore_first(const tal_t *ctx,
 				       struct wallet *w,
-				       const char **key,
+				       const char **startkey,
+				       const char ***key,
 				       const u8 **data,
 				       u64 *generation)
 {
 	struct db_stmt *stmt;
 
+	if (startkey) {
 		stmt = db_prepare_v2(w->db,
-			     SQL("SELECT key, data, generation FROM datastore;"));
+				     SQL("SELECT key, data, generation"
+					 " FROM datastore"
+					 " WHERE key >= ?"
+					 " ORDER BY key;"));
+		db_bind_datastore_key(stmt, 0, startkey);
+	} else {
+		stmt = db_prepare_v2(w->db,
+				     SQL("SELECT key, data, generation"
+					 " FROM datastore"
+					 " ORDER BY key;"));
+	}
 	db_query_prepared(stmt);
 
 	return wallet_datastore_next(ctx, w, stmt, key, data, generation);
@@ -4818,15 +4852,17 @@ struct db_stmt *wallet_datastore_first(const tal_t *ctx,
 struct db_stmt *wallet_datastore_next(const tal_t *ctx,
 				      struct wallet *w,
 				      struct db_stmt *stmt,
-				      const char **key,
+				      const char ***key,
 				      const u8 **data,
 				      u64 *generation)
 {
 	if (!db_step(stmt))
 		return tal_free(stmt);
 
-	*key = tal_strdup(ctx, (const char *)db_column_text(stmt, 0));
+	*key = db_column_datastore_key(ctx, stmt, 0);
+	if (data)
 		*data = db_column_talarr(ctx, stmt, 1);
+	if (generation)
 		*generation = db_column_u64(stmt, 2);
 
 	return stmt;

wallet/wallet.h

@@ -1533,10 +1533,10 @@ void wallet_offer_mark_used(struct db *db, const struct sha256 *offer_id)
 /**
  * Add an new key/value to the datastore (generation 0)
  * @w: the wallet
- * @key: the first key (if returns non-NULL)
+ * @key: the new key (if returns non-NULL)
- * @data: the first data (if returns non-NULL)
+ * @data: the new data (if returns non-NULL)
  */
-void wallet_datastore_create(struct wallet *w, const char *key, const u8 *data);
+void wallet_datastore_create(struct wallet *w, const char **key, const u8 *data);
 
 /**
  * Update an existing key/value to the datastore.
@@ -1544,37 +1544,22 @@ void wallet_datastore_create(struct wallet *w, const char *key, const u8 *data);
  * @key: the first key (if returns non-NULL)
  * @data: the first data (if returns non-NULL)
  */
-void wallet_datastore_update(struct wallet *w, const char *key, const u8 *data);
+void wallet_datastore_update(struct wallet *w,
+			     const char **key,
+			     const u8 *data);
 
 /**
- * Remove a key from the datastore (return the old data).
+ * Remove a key from the datastore
- * @ctx: the tal ctx to allocate return off
  * @w: the wallet
  * @key: the key
- * @generation: the generation of deleted record
- *
- * Returns NULL if the key was not in the store.
  */
-u8 *wallet_datastore_remove(const tal_t *ctx, struct wallet *w, const char *key,
+void wallet_datastore_remove(struct wallet *w, const char **key);
-			    u64 *generation);
-
-/**
- * Retrieve a value from the datastore.
- * @ctx: the tal ctx to allocate return off
- * @w: the wallet
- * @key: the first key (if returns non-NULL)
- * @generation: the generation (if returns non-NULL), or NULL.
- *
- * Returns NULL if the key is not in the store.
- */
-u8 *wallet_datastore_fetch(const tal_t *ctx,
-			   struct wallet *w, const char *key,
-			   u64 *generation);
 
 /**
  * Iterate through the datastore.
  * @ctx: the tal ctx to allocate off
  * @w: the wallet
+ * @startkey: NULL, or the first key to start with
  * @key: the first key (if returns non-NULL)
  * @data: the first data (if returns non-NULL)
  * @generation: the first generation (if returns non-NULL)
@@ -1584,7 +1569,8 @@ u8 *wallet_datastore_fetch(const tal_t *ctx,
  */
 struct db_stmt *wallet_datastore_first(const tal_t *ctx,
 				       struct wallet *w,
-				       const char **key,
+				       const char **startkey,
+				       const char ***key,
 				       const u8 **data,
 				       u64 *generation);
 
@@ -1603,7 +1589,7 @@ struct db_stmt *wallet_datastore_first(const tal_t *ctx,
 struct db_stmt *wallet_datastore_next(const tal_t *ctx,
 				      struct wallet *w,
 				      struct db_stmt *stmt,
-				      const char **key,
+				      const char ***key,
 				      const u8 **data,
 				      u64 *generation);