plugins/sql: listsqlschemas command to retrieve schemas.

Good for detection of what fields are present.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

doc/Makefile

@@ -60,6 +60,7 @@ MANPAGES := doc/lightning-cli.1 \
 	doc/lightning-listpeers.7 \
 	doc/lightning-listpeerchannels.7 \
 	doc/lightning-listsendpays.7 \
+	doc/lightning-listsqlschemas.7 \
 	doc/lightning-makesecret.7 \
 	doc/lightning-multifundchannel.7 \
 	doc/lightning-multiwithdraw.7 \

doc/index.rst

@@ -88,6 +88,7 @@ Core Lightning Documentation
    lightning-listpeerchannels <lightning-listpeerchannels.7.md>
    lightning-listpeers <lightning-listpeers.7.md>
    lightning-listsendpays <lightning-listsendpays.7.md>
+   lightning-listsqlschemas <lightning-listsqlschemas.7.md>
    lightning-listtransactions <lightning-listtransactions.7.md>
    lightning-makesecret <lightning-makesecret.7.md>
    lightning-multifundchannel <lightning-multifundchannel.7.md>

doc/lightning-listsqlschemas.7.md

@@ -0,0 +1,109 @@
+lightning-listsqlschemas -- Command to example lightning-sql schemas
+====================================================================
+
+SYNOPSIS
+--------
+
+**listsqlschemas** [*table*]
+
+DESCRIPTION
+-----------
+
+This allows you to examine the schemas at runtime; while they are fully
+documented for the current release in lightning-sql(7), as fields are
+added or deprecated, you can use this command to determine what fields
+are present.
+
+If *table* is given, only that table is in the resulting list, otherwise
+all tables are listed.
+
+EXAMPLE JSON REQUEST
+------------
+```json
+{
+  "id": 82,
+  "method": "listsqlschemas",
+  "params": {
+    "table": "offers"
+  }
+}
+```
+
+EXAMPLE JSON RESPONSE
+-----
+```json
+{
+   "schemas": [
+      {
+         "tablename": "offers",
+         "columns": [
+            {
+               "name": "offer_id",
+               "type": "BLOB"
+            },
+            {
+               "name": "active",
+               "type": "INTEGER"
+            },
+            {
+               "name": "single_use",
+               "type": "INTEGER"
+            },
+            {
+               "name": "bolt12",
+               "type": "TEXT"
+            },
+            {
+               "name": "bolt12_unsigned",
+               "type": "TEXT"
+            },
+            {
+               "name": "used",
+               "type": "INTEGER"
+            },
+            {
+               "name": "label",
+               "type": "TEXT"
+            }
+         ],
+         "indices": [
+            [
+               "offer_id"
+            ]
+         ]
+      }
+   ]
+}
+```
+
+RETURN VALUE
+------------
+
+[comment]: # (GENERATE-FROM-SCHEMA-START)
+On success, an object containing **schemas** is returned.  It is an array of objects, where each object contains:
+
+- **tablename** (string): the name of the table
+- **columns** (array of objects): the columns, in database order:
+  - **name** (string): the name the column
+  - **type** (string): the SQL type of the column (one of "INTEGER", "BLOB", "TEXT", "REAL")
+- **indices** (array of arrays, optional): Any index we created to speed lookups:
+  - The columns for this index:
+    - The column name
+
+[comment]: # (GENERATE-FROM-SCHEMA-END)
+
+AUTHOR
+------
+
+Rusty Russell <<rusty@rustcorp.com.au>> is mainly responsible.
+
+SEE ALSO
+--------
+
+lightning-sql(7).
+
+RESOURCES
+---------
+
+Main web site: <https://github.com/ElementsProject/lightning>
+[comment]: # ( SHA256STAMP:3ac985dd8ef6959b327e6e6a79079db3ad51423bc4e469799a12ae74b2e75697)

doc/lightning-sql.7.md

@@ -19,6 +19,9 @@ cache `listnodes` and `listchannels`) which then processes the results.
 It is, however faster for remote access if the result of the query is
 much smaller than the list commands would be.
 
+Note that queries like "SELECT *" are fragile, as columns will
+change across releases; see lightning-listsqlschemas(7).
+
 TREATMENT OF TYPES
 ------------------
 

doc/schemas/listsqlschemas.request.json

@@ -0,0 +1,10 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "required": [],
+  "properties": {
+    "table": {
+      "type": "string"
+    }
+  }
+}

doc/schemas/listsqlschemas.schema.json

@@ -0,0 +1,67 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemas"
+  ],
+  "properties": {
+    "schemas": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "tablename",
+          "columns"
+        ],
+        "properties": {
+          "tablename": {
+            "type": "string",
+            "description": "the name of the table"
+          },
+          "columns": {
+            "type": "array",
+            "description": "the columns, in database order",
+            "items": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "name",
+                "type"
+              ],
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "the name the column"
+                },
+                "type": {
+                  "type": "string",
+                  "enum": [
+                    "INTEGER",
+                    "BLOB",
+                    "TEXT",
+                    "REAL"
+                  ],
+                  "description": "the SQL type of the column"
+                }
+              }
+            }
+          },
+          "indices": {
+            "type": "array",
+            "description": "Any index we created to speed lookups",
+            "items": {
+              "type": "array",
+              "description": "The columns for this index",
+              "items": {
+                "type": "string",
+                "description": "The column name"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

plugins/sql.c

@@ -1001,6 +1001,111 @@ static bool ignore_column(const struct table_desc *td, const jsmntok_t *t)
 	return false;
 }
 
+static struct command_result *param_tablename(struct command *cmd,
+					      const char *name,
+					      const char *buffer,
+					      const jsmntok_t *tok,
+					      struct table_desc **td)
+{
+	*td = strmap_getn(&tablemap, buffer + tok->start,
+			  tok->end - tok->start);
+	if (!*td)
+		return command_fail_badparam(cmd, name, buffer, tok,
+					     "Unknown table");
+	return NULL;
+}
+
+static void json_add_column(struct json_stream *js,
+			    const char *dbname,
+			    const char *sqltypename)
+{
+	json_object_start(js, NULL);
+	json_add_string(js, "name", dbname);
+	json_add_string(js, "type", sqltypename);
+	json_object_end(js);
+}
+
+static void json_add_columns(struct json_stream *js,
+			     const struct table_desc *td)
+{
+	for (size_t i = 0; i < tal_count(td->columns); i++) {
+		if (td->columns[i].sub) {
+			if (td->columns[i].sub->is_subobject)
+				json_add_columns(js, td->columns[i].sub);
+			continue;
+		}
+		json_add_column(js, td->columns[i].dbname,
+				fieldtypemap[td->columns[i].ftype].sqltype);
+	}
+}
+
+static void json_add_schema(struct json_stream *js,
+			    const struct table_desc *td)
+{
+	bool have_indices;
+
+	json_object_start(js, NULL);
+	json_add_string(js, "tablename", td->name);
+	/* This needs to be an array, not a dictionary, since dicts
+	 * are often treated as unordered, and order is critical! */
+	json_array_start(js, "columns");
+	if (td->parent) {
+		json_add_column(js, "row", "INTEGER");
+		json_add_column(js, "arrindex", "INTEGER");
+	}
+	json_add_columns(js, td);
+	json_array_end(js);
+
+	/* Don't print indices entry unless we have an index! */
+	have_indices = false;
+	for (size_t i = 0; i < ARRAY_SIZE(indices); i++) {
+		if (!streq(indices[i].tablename, td->name))
+			continue;
+		if (!have_indices) {
+			json_array_start(js, "indices");
+			have_indices = true;
+		}
+		json_array_start(js, NULL);
+		for (size_t j = 0; j < ARRAY_SIZE(indices[i].fields); j++) {
+			if (indices[i].fields[j])
+				json_add_string(js, NULL, indices[i].fields[j]);
+		}
+		json_array_end(js);
+	}
+	if (have_indices)
+		json_array_end(js);
+	json_object_end(js);
+}
+
+static bool add_one_schema(const char *member, struct table_desc *td,
+			   struct json_stream *js)
+{
+	json_add_schema(js, td);
+	return true;
+}
+
+static struct command_result *json_listsqlschemas(struct command *cmd,
+						  const char *buffer,
+						  const jsmntok_t *params)
+{
+	struct table_desc *td;
+	struct json_stream *ret;
+
+	if (!param(cmd, buffer, params,
+		   p_opt("table", param_tablename, &td),
+		   NULL))
+		return command_param_failed();
+
+	ret = jsonrpc_stream_success(cmd);
+	json_array_start(ret, "schemas");
+	if (td)
+		json_add_schema(ret, td);
+	else
+		strmap_iterate(&tablemap, add_one_schema, ret);
+	json_array_end(ret);
+	return command_finished(cmd, ret);
+}
+
 /* Creates sql statements, initializes table */
 static void finish_td(struct plugin *plugin, struct table_desc *td)
 {
@@ -1353,6 +1458,13 @@ static const struct plugin_command commands[] = { {
 	"This is the greatest plugin command ever!",
 	json_sql,
 	},
+	{
+		"listsqlschemas",
+		"misc",
+		"Display schemas for internal sql tables, or just {table}",
+		"This is the greatest plugin command ever!",
+		json_listsqlschemas,
+	},
 };
 
 static const char *fmt_indexes(const tal_t *ctx, const char *table)

tests/test_plugin.py

@@ -3783,6 +3783,34 @@ def test_sql(node_factory, bitcoind):
                         {'name': 'payment_id',
                          'type': 'hex'}]}}
 
+    sqltypemap = {'string': 'TEXT',
+                  'boolean': 'INTEGER',
+                  'u8': 'INTEGER',
+                  'u16': 'INTEGER',
+                  'u32': 'INTEGER',
+                  'u64': 'INTEGER',
+                  'msat': 'INTEGER',
+                  'hex': 'BLOB',
+                  'hash': 'BLOB',
+                  'txid': 'BLOB',
+                  'pubkey': 'BLOB',
+                  'secret': 'BLOB',
+                  'number': 'REAL',
+                  'short_channel_id': 'TEXT'}
+
+    # Check schemas match.
+    for table, schema in expected_schemas.items():
+        res = only_one(l2.rpc.listsqlschemas(table)['schemas'])
+        assert res['tablename'] == table
+        assert res.get('indices') == schema.get('indices')
+        sqlcolumns = [{'name': c['name'], 'type': sqltypemap[c['type']]} for c in schema['columns']]
+        assert res['columns'] == sqlcolumns
+
+    # Make sure we didn't miss any
+    assert (sorted([s['tablename'] for s in l1.rpc.listsqlschemas()['schemas']])
+            == sorted(expected_schemas.keys()))
+    assert len(l1.rpc.listsqlschemas()['schemas']) == len(expected_schemas)
+
     # Very rough checks of other list commands (make sure l2 has one of each)
     l2.rpc.offer(1, 'desc')
     l2.rpc.invoice(1, 'label', 'desc')