common: make json_parse_input API retry friendly.

The jsmn parser is a beautiful piece of code.  In particular, you can parse
part of a string, then continue where you left off.

We don't take advantage of this, however, meaning for large JSON objects
we parse them multiple times before finally having enough to complete.

Expose the parser state and tokens through the API, so the caller can pass
them in repeatedly.  For the moment, every caller is allocates each time
(except the unit tests).

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/json.c

@@ -539,60 +539,74 @@ validate_jsmn_parse_output(const jsmntok_t *p, const jsmntok_t *end)
 JSMN Result Validation Ends
 -----------------------------------------------------------------------------*/
 
-jsmntok_t *json_parse_input(const tal_t *ctx,
-			    const char *input, int len, bool *valid)
+void toks_reset(jsmntok_t *toks)
+{
+	assert(tal_count(toks) >= 1);
+	toks[0].type = JSMN_UNDEFINED;
+}
+
+jsmntok_t *toks_alloc(const tal_t *ctx)
+{
+	jsmntok_t *toks = tal_arr(ctx, jsmntok_t, 10);
+	toks_reset(toks);
+	return toks;
+}
+
+bool json_parse_input(jsmn_parser *parser,
+		      jsmntok_t **toks,
+		      const char *input, int len,
+		      bool *complete)
 {
-	jsmn_parser parser;
-	jsmntok_t *toks;
 	int ret;
 
-	toks = tal_arr(ctx, jsmntok_t, 10);
-	toks[0].type = JSMN_UNDEFINED;
-
-	jsmn_init(&parser);
 again:
-	ret = jsmn_parse(&parser, input, len, toks, tal_count(toks) - 1);
+	ret = jsmn_parse(parser, input, len, *toks, tal_count(*toks) - 1);
 
 	switch (ret) {
 	case JSMN_ERROR_INVAL:
-		*valid = false;
-		return tal_free(toks);
+		return false;
 	case JSMN_ERROR_NOMEM:
-		tal_resize(&toks, tal_count(toks) * 2);
+		tal_resize(toks, tal_count(*toks) * 2);
 		goto again;
 	}
 
 	/* Check whether we read at least one full root element, i.e., root
 	 * element has its end set. */
-	if (toks[0].type == JSMN_UNDEFINED || toks[0].end == -1) {
-		*valid = true;
-		return tal_free(toks);
+	if ((*toks)[0].type == JSMN_UNDEFINED || (*toks)[0].end == -1) {
+		*complete = false;
+		return true;
 	}
 
 	/* If we read a partial element at the end of the stream we'll get a
 	 * ret=JSMN_ERROR_PART, but due to the previous check we know we read at
 	 * least one full element, so count tokens that are part of this root
 	 * element. */
-	ret = json_next(toks) - toks;
+	ret = json_next(*toks) - *toks;
+
+	if (!validate_jsmn_parse_output(*toks, *toks + ret))
+		return false;
 
 	/* Cut to length and return. */
-	*valid = validate_jsmn_parse_output(toks, toks + ret);
-	tal_resize(&toks, ret + 1);
+	tal_resize(toks, ret + 1);
 	/* Make sure last one is always referenceable. */
-	toks[ret].type = -1;
-	toks[ret].start = toks[ret].end = toks[ret].size = 0;
+	(*toks)[ret].type = -1;
+	(*toks)[ret].start = (*toks)[ret].end = (*toks)[ret].size = 0;
 
-	return toks;
+	*complete = true;
+	return true;
 }
 
 jsmntok_t *json_parse_simple(const tal_t *ctx, const char *input, int len)
 {
-	bool valid;
-	jsmntok_t *toks;
+	bool complete;
+	jsmn_parser parser;
+	jsmntok_t *toks = toks_alloc(ctx);
 
-	toks = json_parse_input(ctx, input, len, &valid);
-	if (toks && !valid)
-		toks = tal_free(toks);
+	jsmn_init(&parser);
+
+	if (!json_parse_input(&parser, &toks, input, len, &complete)
+	    || !complete)
+		return tal_free(toks);
 	return toks;
 }
 

common/json.h

@@ -85,9 +85,35 @@ const jsmntok_t *json_get_member(const char *buffer, const jsmntok_t tok[],
 /* Get index'th array member. */
 const jsmntok_t *json_get_arr(const jsmntok_t tok[], size_t index);
 
-/* If input is complete and valid, return tokens. */
-jsmntok_t *json_parse_input(const tal_t *ctx,
-			    const char *input, int len, bool *valid);
+/* Allocate a starter array of tokens for json_parse_input */
+jsmntok_t *toks_alloc(const tal_t *ctx);
+
+/* Reset a token array to reuse it. */
+void toks_reset(jsmntok_t *toks);
+
+/**
+ * json_parse_input: parse and validate JSON.
+ * @parser: parser initialized with jsmn_init.
+ * @toks: tallocated array from toks_alloc()
+ * @input, @len: input string.
+ * @complete: set to true if the valid JSON is complete, or NULL if must be.
+ *
+ * This returns false if the JSON is invalid, true otherwise.
+ * If it returns true, *@complete indicates that (*@toks)[0] points to a
+ * valid, complete JSON element.  If @complete is NULL, then incomplete
+ * JSON returns false (i.e. is considered invalid).
+ *
+ * *@toks is resized to the complete set of tokens, with a dummy
+ * terminator (type == -1) at the end.
+ *
+ * If it returns true, and *@complete is false, you can append more
+ * data to @input and call it again (with the same perser) and the parser
+ * will continue where it left off.
+*/
+bool json_parse_input(jsmn_parser *parser,
+		      jsmntok_t **toks,
+		      const char *input, int len,
+		      bool *complete);
 
 /* Simplified version of above which parses only a complete, valid
  * JSON string */

common/test/run-json.c

@@ -104,34 +104,47 @@ static int test_json_tok_millionths(void)
 
 static void test_json_tok_size(void)
 {
-	const jsmntok_t *toks;
+	jsmntok_t *toks;
 	char *buf;
-	bool ok;
+	bool ok, complete;
+	jsmn_parser parser;
 
 	buf = "[\"e1\", [\"e2\", \"e3\"]]";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks = toks_alloc(tmpctx);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(ok);
+	assert(complete);
 	/* size only counts *direct* children */
 	assert(toks[0].size == 2);
 	assert(toks[2].size == 2);
 
 	buf = "[[\"e1\", \"e2\"], \"e3\"]";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(ok);
+	assert(complete);
 	/* size only counts *direct* children */
 	assert(toks[0].size == 2);
 	assert(toks[1].size == 2);
 
 	buf = "{\"e1\" : {\"e2\": 2, \"e3\": 3}}";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(ok);
+	assert(complete);
 	/* size only counts *direct* children */
 	assert(toks[0].size == 1);
 	assert(toks[2].size == 2);
 
 	buf = "{\"e1\" : {\"e2\": 2, \"e3\": 3}, \"e4\" : {\"e5\": 5, \"e6\": 6}}";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(ok);
+	assert(complete);
 	/* size only counts *direct* children */
 	assert(toks[0].size == 2);
 	assert(toks[2].size == 2);
@@ -139,7 +152,9 @@ static void test_json_tok_size(void)
 
 	/* This should *not* parse! (used to give toks[0]->size == 3!) */
 	buf = "{ \"\" \"\" \"\" }";
-	toks = json_parse_input(tmpctx, buf, strlen(buf), &ok);
+	toks_reset(toks);
+	jsmn_init(&parser);
+	ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete);
 	assert(!ok);
 
 	/* This should *not* parse! (used to give toks[0]->size == 2!) */