datastore: allow replace/append.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

doc/lightning-datastore.7

@@ -3,7 +3,7 @@
 lightning-datastore - Command for storing (plugin) data
 .SH SYNOPSIS
 
-\fBdatastore\fR \fIkey\fR [\fIstring\fR|\fIhex\fR]
+\fBdatastore\fR \fIkey\fR [\fIstring\fR] [\fIhex\fR] [\fImode\fR]
 
 .SH DESCRIPTION
 
@@ -14,6 +14,13 @@ c-lightning database, for later retrieval\.
 There can only be one entry for each \fIkey\fR, so prefixing with the
 plugin name (e\.g\. \fBsummary.\fR) is recommended\.
 
+
+\fImode\fR is one of "must-create" (default, fails it it already exists),
+"must-replace" (fails it it doesn't already exist),
+"create-or-replace" (never fails), "must-append" (must already exist,
+append this to what's already there) or "create-or-append" (append if
+anything is there, otherwise create)\.
+
 .SH RETURN VALUE
 
 On success, an object is returned, containing:
@@ -35,7 +42,7 @@ The following error codes may occur:
 
 .RS
 .IP \[bu]
--32602: invalid parameters, including already-existing key\.
+-32602: invalid parameters, including already-existing/not-existing key\.
 
 .RE
 .SH AUTHOR
@@ -50,4 +57,4 @@ Rusty Russell \fI<rusty@rustcorp.com.au\fR> is mainly responsible\.
 
 Main web site: \fIhttps://github.com/ElementsProject/lightning\fR
 
-\" SHA256STAMP:e0ea91fb846859bc22af6e4beacd5fb726d6c4ffefea5cc4e3a1250b81665317
+\" SHA256STAMP:4bb1369465ffb76e8e1962bd9e242159e579bb3af6e01c9d1461e519d8721769

doc/lightning-datastore.7.md

@@ -4,7 +4,7 @@ lightning-datastore -- Command for storing (plugin) data
 SYNOPSIS
 --------
 
-**datastore** *key* [*string*|*hex*]
+**datastore** *key* [*string*] [*hex*] [*mode*]
 
 DESCRIPTION
 -----------
@@ -15,6 +15,12 @@ c-lightning database, for later retrieval.
 There can only be one entry for each *key*, so prefixing with the
 plugin name (e.g. `summary.`) is recommended.
 
+*mode* is one of "must-create" (default, fails it it already exists),
+"must-replace" (fails it it doesn't already exist),
+"create-or-replace" (never fails), "must-append" (must already exist,
+append this to what's already there) or "create-or-append" (append if
+anything is there, otherwise create).
+
 RETURN VALUE
 ------------
 
@@ -28,7 +34,7 @@ On success, an object is returned, containing:
 The main cause of failure is an already-existing entry.
 
 The following error codes may occur:
-- -32602: invalid parameters, including already-existing key.
+- -32602: invalid parameters, including already-existing/not-existing key.
 
 AUTHOR
 ------

lightningd/datastore.c

@@ -15,6 +15,40 @@ static void json_add_datastore(struct json_stream *response,
 		json_add_string(response, "string", str);
 }
 
+enum ds_mode {
+	DS_MUST_EXIST = 1,
+	DS_MUST_NOT_EXIST = 2,
+	DS_APPEND = 4
+};
+
+static struct command_result *param_mode(struct command *cmd,
+					 const char *name,
+					 const char *buffer,
+					 const jsmntok_t *tok,
+					 enum ds_mode **mode)
+{
+	*mode = tal(cmd, enum ds_mode);
+	if (json_tok_streq(buffer, tok, "must-create"))
+		**mode = DS_MUST_NOT_EXIST;
+	else if (json_tok_streq(buffer, tok, "must-replace"))
+		**mode = DS_MUST_EXIST;
+	else if (json_tok_streq(buffer, tok, "create-or-replace"))
+		**mode = 0;
+	else if (json_tok_streq(buffer, tok, "must-append"))
+		**mode = DS_MUST_EXIST | DS_APPEND;
+	else if (json_tok_streq(buffer, tok, "create-or-append"))
+		**mode = DS_APPEND;
+	else
+		return command_fail_badparam(cmd, name, buffer, tok,
+					     "should be 'must-create',"
+					     " 'must-replace',"
+					     " 'create-or-replace',"
+					     " 'must-append',"
+					     " or 'create-or-append'");
+
+	return NULL;
+}
+
 static struct command_result *json_datastore(struct command *cmd,
 					     const char *buffer,
 					     const jsmntok_t *obj UNNEEDED,
@@ -22,12 +56,14 @@ static struct command_result *json_datastore(struct command *cmd,
 {
 	struct json_stream *response;
 	const char *key, *strdata;
-	u8 *data;
+	u8 *data, *prevdata;
+	enum ds_mode *mode;
 
 	if (!param(cmd, buffer, params,
 		   p_req("key", param_string, &key),
 		   p_opt("string", param_string, &strdata),
 		   p_opt("hex", param_bin_from_hex, &data),
+		   p_opt_def("mode", param_mode, &mode, DS_MUST_NOT_EXIST),
 		   NULL))
 		return command_param_failed();
 
@@ -42,10 +78,27 @@ static struct command_result *json_datastore(struct command *cmd,
 					    "Must have either hex or string");
 	}
 
-	if (!wallet_datastore_add(cmd->ld->wallet, key, data))
+	prevdata = wallet_datastore_fetch(cmd, cmd->ld->wallet, key);
+	if ((*mode & DS_MUST_NOT_EXIST) && prevdata)
 		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
 				    "Key already exists");
 
+	if ((*mode & DS_MUST_EXIST) && !prevdata)
+		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
+				    "Key does not exist");
+
+	if ((*mode & DS_APPEND) && prevdata) {
+		size_t prevlen = tal_bytelen(prevdata);
+		tal_resize(&prevdata, prevlen + tal_bytelen(data));
+		memcpy(prevdata + prevlen, data, tal_bytelen(data));
+		data = prevdata;
+	}
+
+	if (prevdata)
+		wallet_datastore_update(cmd->ld->wallet, key, data);
+	else
+		wallet_datastore_create(cmd->ld->wallet, key, data);
+
 	response = json_stream_success(cmd);
 	json_add_datastore(response, key, data);
 	return command_success(cmd, response);

tests/test_misc.py

@@ -2644,11 +2644,34 @@ def test_datastore(node_factory):
     assert l1.rpc.listdatastore('somekey') == {'datastore': [somedata_expect]}
     assert l1.rpc.listdatastore('otherkey') == {'datastore': []}
 
+    # Cannot add by default.
+    with pytest.raises(RpcError, match='already exists'):
+        l1.rpc.datastore(key='somekey', hex=somedata)
+
+    with pytest.raises(RpcError, match='already exists'):
+        l1.rpc.datastore(key='somekey', hex=somedata, mode="must-create")
+
+    # But can insist on replace.
+    l1.rpc.datastore(key='somekey', hex=somedata[:-4], mode="must-replace")
+    assert only_one(l1.rpc.listdatastore('somekey')['datastore'])['hex'] == somedata[:-4]
+    # And append works.
+    l1.rpc.datastore(key='somekey', hex=somedata[-4:-2], mode="must-append")
+    assert only_one(l1.rpc.listdatastore('somekey')['datastore'])['hex'] == somedata[:-2]
+    l1.rpc.datastore(key='somekey', hex=somedata[-2:], mode="create-or-append")
+    assert only_one(l1.rpc.listdatastore('somekey')['datastore'])['hex'] == somedata
+
+    # Can't replace or append non-existing records if we say not to
+    with pytest.raises(RpcError, match='does not exist'):
+        l1.rpc.datastore(key='otherkey', hex=somedata, mode="must-replace")
+
+    with pytest.raises(RpcError, match='does not exist'):
+        l1.rpc.datastore(key='otherkey', hex=somedata, mode="must-append")
+
     otherdata = b'otherdata'.hex()
     otherdata_expect = {'key': 'otherkey',
                         'hex': otherdata,
                         'string': 'otherdata'}
-    assert l1.rpc.datastore(key='otherkey', string='otherdata') == otherdata_expect
+    assert l1.rpc.datastore(key='otherkey', string='otherdata', mode="create-or-append") == otherdata_expect
 
     assert l1.rpc.listdatastore('somekey') == {'datastore': [somedata_expect]}
     assert l1.rpc.listdatastore('otherkey') == {'datastore': [otherdata_expect]}

wallet/db_postgres_sqlgen.c

@@ -2007,10 +2007,10 @@ struct db_query db_postgres_queries[] = {
          .readonly = true,
     },
     {
-         .name = "SELECT 1  FROM datastore WHERE key = ?;",
+         .name = "UPDATE datastore SET data=? WHERE key=?;",
-         .query = "SELECT 1  FROM datastore WHERE key = $1;",
+         .query = "UPDATE datastore SET data=$1 WHERE key=$2;",
-         .placeholders = 1,
+         .placeholders = 2,
-         .readonly = true,
+         .readonly = false,
     },
     {
          .name = "INSERT INTO datastore VALUES (?, ?);",
@@ -2068,4 +2068,4 @@ struct db_query db_postgres_queries[] = {
 
 #endif /* LIGHTNINGD_WALLET_GEN_DB_POSTGRES */
 
-// SHA256STAMP:743e13b59241ab495a7ebd6e0e50887f399e7816cc73fd90154f761f1b484f89
+// SHA256STAMP:5b2863e970ee24f2552e45224fed3f248badc9fd87ee3ab3ff0458fe3bcc0a83

wallet/db_sqlite3_sqlgen.c

@@ -2007,10 +2007,10 @@ struct db_query db_sqlite3_queries[] = {
          .readonly = true,
     },
     {
-         .name = "SELECT 1  FROM datastore WHERE key = ?;",
+         .name = "UPDATE datastore SET data=? WHERE key=?;",
-         .query = "SELECT 1  FROM datastore WHERE key = ?;",
+         .query = "UPDATE datastore SET data=? WHERE key=?;",
-         .placeholders = 1,
+         .placeholders = 2,
-         .readonly = true,
+         .readonly = false,
     },
     {
          .name = "INSERT INTO datastore VALUES (?, ?);",
@@ -2068,4 +2068,4 @@ struct db_query db_sqlite3_queries[] = {
 
 #endif /* LIGHTNINGD_WALLET_GEN_DB_SQLITE3 */
 
-// SHA256STAMP:743e13b59241ab495a7ebd6e0e50887f399e7816cc73fd90154f761f1b484f89
+// SHA256STAMP:5b2863e970ee24f2552e45224fed3f248badc9fd87ee3ab3ff0458fe3bcc0a83

wallet/statements_gettextgen.po

@@ -1331,22 +1331,22 @@ msgid "SELECT status  FROM offers WHERE offer_id = ?;"
 msgstr ""
 
 #: wallet/wallet.c:4746
-msgid "SELECT 1  FROM datastore WHERE key = ?;"
+msgid "UPDATE datastore SET data=? WHERE key=?;"
 msgstr ""
 
-#: wallet/wallet.c:4759
+#: wallet/wallet.c:4757
 msgid "INSERT INTO datastore VALUES (?, ?);"
 msgstr ""
 
-#: wallet/wallet.c:4773
+#: wallet/wallet.c:4770
 msgid "DELETE FROM datastore WHERE key = ?"
 msgstr ""
 
-#: wallet/wallet.c:4788
+#: wallet/wallet.c:4785
 msgid "SELECT data  FROM datastore WHERE key = ?;"
 msgstr ""
 
-#: wallet/wallet.c:4809
+#: wallet/wallet.c:4806
 msgid "SELECT key, data FROM datastore;"
 msgstr ""
 
@@ -1365,4 +1365,4 @@ msgstr ""
 #: wallet/test/run-wallet.c:1753
 msgid "INSERT INTO channels (id) VALUES (1);"
 msgstr ""
-#  SHA256STAMP:65a48a3f7b7223e68e12148b699342842b394228141583f008d0cd7002b18f97
+#  SHA256STAMP:33c97b729f031ff86abf939de5571a7e5a9cf3d0795bf975189d90ace76d28a1

wallet/wallet.c

@@ -4738,22 +4738,20 @@ void wallet_offer_mark_used(struct db *db, const struct sha256 *offer_id)
 	}
 }
 
-bool wallet_datastore_add(struct wallet *w, const char *key, const u8 *data)
+void wallet_datastore_update(struct wallet *w, const char *key, const u8 *data)
 {
 	struct db_stmt *stmt;
 
-	/* Test if already exists. */
+	stmt = db_prepare_v2(w->db,
-	stmt = db_prepare_v2(w->db, SQL("SELECT 1"
+			     SQL("UPDATE datastore SET data=? WHERE key=?;"));
-					"  FROM datastore"
+	db_bind_talarr(stmt, 0, data);
-					" WHERE key = ?;"));
+	db_bind_text(stmt, 1, key);
-	db_bind_text(stmt, 0, key);
+	db_exec_prepared_v2(take(stmt));
-	db_query_prepared(stmt);
+}
 
-	if (db_step(stmt)) {
+void wallet_datastore_create(struct wallet *w, const char *key, const u8 *data)
-		tal_free(stmt);
+{
-		return false;
+	struct db_stmt *stmt;
-	}
-	tal_free(stmt);
 
 	stmt = db_prepare_v2(w->db,
 			     SQL("INSERT INTO datastore VALUES (?, ?);"));
@@ -4761,7 +4759,6 @@ bool wallet_datastore_add(struct wallet *w, const char *key, const u8 *data)
 	db_bind_text(stmt, 0, key);
 	db_bind_talarr(stmt, 1, data);
 	db_exec_prepared_v2(take(stmt));
-	return true;
 }
 
 u8 *wallet_datastore_remove(const tal_t *ctx, struct wallet *w, const char *key)

wallet/wallet.h

@@ -1531,14 +1531,20 @@ void wallet_offer_mark_used(struct db *db, const struct sha256 *offer_id)
 	NO_NULL_ARGS;
 
 /**
- * Add a key/value to the datastore.
+ * Add an new key/value to the datastore.
  * @w: the wallet
  * @key: the first key (if returns non-NULL)
  * @data: the first data (if returns non-NULL)
- *
- * Returns false if the key is already in the store.
  */
-bool wallet_datastore_add(struct wallet *w, const char *key, const u8 *data);
+void wallet_datastore_create(struct wallet *w, const char *key, const u8 *data);
+
+/**
+ * Update an existing key/value to the datastore.
+ * @w: the wallet
+ * @key: the first key (if returns non-NULL)
+ * @data: the first data (if returns non-NULL)
+ */
+void wallet_datastore_update(struct wallet *w, const char *key, const u8 *data);
 
 /**
  * Remove a key from the datastore (return the old data).