common: make json_parse_input API retry friendly.

The jsmn parser is a beautiful piece of code.  In particular, you can parse
part of a string, then continue where you left off.

We don't take advantage of this, however, meaning for large JSON objects
we parse them multiple times before finally having enough to complete.

Expose the parser state and tokens through the API, so the caller can pass
them in repeatedly.  For the moment, every caller is allocates each time
(except the unit tests).

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

lightningd/jsonrpc.c

@@ -913,7 +913,8 @@ static struct io_plan *read_json(struct io_conn *conn,
 				 struct json_connection *jcon)
 {
 	jsmntok_t *toks;
-	bool valid;
+	bool complete;
+	jsmn_parser parser;
 
 	if (jcon->len_read)
 		log_io(jcon->log, LOG_IO_IN, NULL, "",
@@ -930,21 +931,18 @@ static struct io_plan *read_json(struct io_conn *conn,
 		return io_wait(conn, conn, read_json, jcon);
 	}
 
-	toks = json_parse_input(jcon->buffer, jcon->buffer, jcon->used, &valid);
-	if (!toks) {
-		if (!valid) {
-			log_unusual(jcon->log,
-				    "Invalid token in json input: '%.*s'",
-				    (int)jcon->used, jcon->buffer);
-			json_command_malformed(
-			    jcon, "null",
-			    "Invalid token in json input");
-			return io_halfclose(conn);
-		}
-		/* We need more. */
-		goto read_more;
+	toks = toks_alloc(jcon->buffer);
+	jsmn_init(&parser);
+	if (!json_parse_input(&parser, &toks, jcon->buffer, jcon->used,
+			      &complete)) {
+		json_command_malformed(jcon, "null",
+				       "Invalid token in json input");
+		return io_halfclose(conn);
 	}
 
+	if (!complete)
+		goto read_more;
+
 	/* Empty buffer? (eg. just whitespace). */
 	if (tal_count(toks) == 1) {
 		jcon->used = 0;

lightningd/plugin.c

@@ -386,10 +386,11 @@ static const char *plugin_read_json_one(struct plugin *plugin,
 					bool *complete,
 					bool *destroyed)
 {
-	bool valid;
-	const jsmntok_t *toks, *jrtok, *idtok;
+	jsmntok_t *toks;
+	const jsmntok_t *jrtok, *idtok;
 	struct plugin_destroyed *pd;
 	const char *err;
+	jsmn_parser parser;
 
 	*destroyed = false;
 	/* Note that in the case of 'plugin stop' this can free request (since
@@ -398,28 +399,30 @@ static const char *plugin_read_json_one(struct plugin *plugin,
 	/* FIXME: This could be done more efficiently by storing the
 	 * toks and doing an incremental parse, like lightning-cli
 	 * does. */
-	toks = json_parse_input(plugin->buffer, plugin->buffer, plugin->used,
-				&valid);
-	if (!toks) {
-		if (!valid) {
-			return tal_fmt(plugin,
-				       "Failed to parse JSON response '%.*s'",
-				       (int)plugin->used, plugin->buffer);
-		}
+	toks = toks_alloc(plugin);
+	jsmn_init(&parser);
+	if (!json_parse_input(&parser, &toks, plugin->buffer, plugin->used,
+			      complete)) {
+		return tal_fmt(plugin,
+			       "Failed to parse JSON response '%.*s'",
+			       (int)plugin->used, plugin->buffer);
+	}
+
+	if (!*complete) {
 		/* We need more. */
-		*complete = false;
+		tal_free(toks);
 		return NULL;
 	}
 
 	/* Empty buffer? (eg. just whitespace). */
 	if (tal_count(toks) == 1) {
+		tal_free(toks);
 		plugin->used = 0;
 		/* We need more. */
 		*complete = false;
 		return NULL;
 	}
 
-	*complete = true;
 	jrtok = json_get_member(plugin->buffer, toks, "jsonrpc");
 	idtok = json_get_member(plugin->buffer, toks, "id");
 

lightningd/test/run-jsonrpc.c

@@ -132,11 +132,12 @@ static int test_json_filter(void)
 	struct json_stream *result = new_json_stream(NULL, NULL, NULL);
 	jsmntok_t *toks;
 	const jsmntok_t *x;
-	bool valid;
+	bool valid, complete;
 	int i;
 	char *badstr = tal_arr(result, char, 256);
 	const char *str;
 	size_t len;
+	jsmn_parser parser;
 
 	/* Fill with junk, and nul-terminate (256 -> 0) */
 	for (i = 1; i < 257; i++)
@@ -150,9 +151,11 @@ static int test_json_filter(void)
 	str = json_out_contents(result->jout, &len);
 	str = tal_strndup(result, str, len);
 
-	toks = json_parse_input(str, str, strlen(str), &valid);
+	toks = toks_alloc(str);
+	jsmn_init(&parser);
+	valid = json_parse_input(&parser, &toks, str, strlen(str), &complete);
 	assert(valid);
-	assert(toks);
+	assert(complete);
 
 	assert(toks[0].type == JSMN_OBJECT);
 	x = json_get_member(str, toks, "x");
@@ -240,32 +243,54 @@ static void test_json_partial(void)
 /* Test that we can segment and parse a stream of json objects correctly. */
 static void test_json_stream(void)
 {
-	bool valid;
+	bool valid, complete;
 	char *input, *talstr;
 	jsmntok_t *toks;
+	jsmn_parser parser;
 
 	/* Multiple full messages in a single buffer (happens when buffer
 	 * boundary coincides with message boundary, or read returned after
 	 * timeout. */
 	input = "{\"x\":\"x\"}{\"y\":\"y\"}";
 	talstr = tal_strndup(NULL, input, strlen(input));
-	toks = json_parse_input(talstr, talstr, strlen(talstr), &valid);
-	assert(toks);
+	toks = toks_alloc(NULL);
+	jsmn_init(&parser);
+	valid = json_parse_input(&parser, &toks, talstr, strlen(talstr), &complete);
 	assert(tal_count(toks) == 4);
 	assert(toks[0].start == 0 && toks[0].end == 9);
 	assert(valid);
+	assert(complete);
 	tal_free(talstr);
 
 	/* Multiple messages, and the last one is partial, far more likely than
 	 * accidentally getting the boundaries to match. */
 	input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z";
 	talstr = tal_strndup(NULL, input, strlen(input));
-	toks = json_parse_input(talstr, talstr, strlen(talstr), &valid);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	valid = json_parse_input(&parser, &toks, talstr, strlen(talstr), &complete);
 	assert(toks);
 	assert(tal_count(toks) == 4);
 	assert(toks[0].start == 0 && toks[0].end == 9);
 	assert(valid);
+	assert(complete);
 	tal_free(talstr);
+
+	/* We can do this incrementally, too. */
+	toks_reset(toks);
+	input = "{\"x\":\"x\"}";
+	jsmn_init(&parser);
+	for (size_t i = 0; i <= strlen(input); i++) {
+		valid = json_parse_input(&parser, &toks, input, i, &complete);
+		assert(valid);
+		if (i == strlen(input))
+			assert(complete);
+		else
+			assert(!complete);
+	}
+	assert(tal_count(toks) == 4);
+	assert(toks[0].start == 0 && toks[0].end == 9);
+	tal_free(toks);
 }
 
 int main(void)