diff --git a/common/json.c b/common/json.c index d16cc811f..4b84c0220 100644 --- a/common/json.c +++ b/common/json.c @@ -539,60 +539,74 @@ validate_jsmn_parse_output(const jsmntok_t *p, const jsmntok_t *end) JSMN Result Validation Ends -----------------------------------------------------------------------------*/ -jsmntok_t *json_parse_input(const tal_t *ctx, - const char *input, int len, bool *valid) +void toks_reset(jsmntok_t *toks) +{ + assert(tal_count(toks) >= 1); + toks[0].type = JSMN_UNDEFINED; +} + +jsmntok_t *toks_alloc(const tal_t *ctx) +{ + jsmntok_t *toks = tal_arr(ctx, jsmntok_t, 10); + toks_reset(toks); + return toks; +} + +bool json_parse_input(jsmn_parser *parser, + jsmntok_t **toks, + const char *input, int len, + bool *complete) { - jsmn_parser parser; - jsmntok_t *toks; int ret; - toks = tal_arr(ctx, jsmntok_t, 10); - toks[0].type = JSMN_UNDEFINED; - - jsmn_init(&parser); again: - ret = jsmn_parse(&parser, input, len, toks, tal_count(toks) - 1); + ret = jsmn_parse(parser, input, len, *toks, tal_count(*toks) - 1); switch (ret) { case JSMN_ERROR_INVAL: - *valid = false; - return tal_free(toks); + return false; case JSMN_ERROR_NOMEM: - tal_resize(&toks, tal_count(toks) * 2); + tal_resize(toks, tal_count(*toks) * 2); goto again; } /* Check whether we read at least one full root element, i.e., root * element has its end set. */ - if (toks[0].type == JSMN_UNDEFINED || toks[0].end == -1) { - *valid = true; - return tal_free(toks); + if ((*toks)[0].type == JSMN_UNDEFINED || (*toks)[0].end == -1) { + *complete = false; + return true; } /* If we read a partial element at the end of the stream we'll get a * ret=JSMN_ERROR_PART, but due to the previous check we know we read at * least one full element, so count tokens that are part of this root * element. */ - ret = json_next(toks) - toks; + ret = json_next(*toks) - *toks; + + if (!validate_jsmn_parse_output(*toks, *toks + ret)) + return false; /* Cut to length and return. */ - *valid = validate_jsmn_parse_output(toks, toks + ret); - tal_resize(&toks, ret + 1); + tal_resize(toks, ret + 1); /* Make sure last one is always referenceable. */ - toks[ret].type = -1; - toks[ret].start = toks[ret].end = toks[ret].size = 0; + (*toks)[ret].type = -1; + (*toks)[ret].start = (*toks)[ret].end = (*toks)[ret].size = 0; - return toks; + *complete = true; + return true; } jsmntok_t *json_parse_simple(const tal_t *ctx, const char *input, int len) { - bool valid; - jsmntok_t *toks; + bool complete; + jsmn_parser parser; + jsmntok_t *toks = toks_alloc(ctx); - toks = json_parse_input(ctx, input, len, &valid); - if (toks && !valid) - toks = tal_free(toks); + jsmn_init(&parser); + + if (!json_parse_input(&parser, &toks, input, len, &complete) + || !complete) + return tal_free(toks); return toks; } diff --git a/common/json.h b/common/json.h index 7a7d98e25..c1374a89c 100644 --- a/common/json.h +++ b/common/json.h @@ -85,9 +85,35 @@ const jsmntok_t *json_get_member(const char *buffer, const jsmntok_t tok[], /* Get index'th array member. */ const jsmntok_t *json_get_arr(const jsmntok_t tok[], size_t index); -/* If input is complete and valid, return tokens. */ -jsmntok_t *json_parse_input(const tal_t *ctx, - const char *input, int len, bool *valid); +/* Allocate a starter array of tokens for json_parse_input */ +jsmntok_t *toks_alloc(const tal_t *ctx); + +/* Reset a token array to reuse it. */ +void toks_reset(jsmntok_t *toks); + +/** + * json_parse_input: parse and validate JSON. + * @parser: parser initialized with jsmn_init. + * @toks: tallocated array from toks_alloc() + * @input, @len: input string. + * @complete: set to true if the valid JSON is complete, or NULL if must be. + * + * This returns false if the JSON is invalid, true otherwise. + * If it returns true, *@complete indicates that (*@toks)[0] points to a + * valid, complete JSON element. If @complete is NULL, then incomplete + * JSON returns false (i.e. is considered invalid). + * + * *@toks is resized to the complete set of tokens, with a dummy + * terminator (type == -1) at the end. + * + * If it returns true, and *@complete is false, you can append more + * data to @input and call it again (with the same perser) and the parser + * will continue where it left off. +*/ +bool json_parse_input(jsmn_parser *parser, + jsmntok_t **toks, + const char *input, int len, + bool *complete); /* Simplified version of above which parses only a complete, valid * JSON string */ diff --git a/common/test/run-json.c b/common/test/run-json.c index ff05bbf2f..16f961ff6 100644 --- a/common/test/run-json.c +++ b/common/test/run-json.c @@ -104,34 +104,47 @@ static int test_json_tok_millionths(void) static void test_json_tok_size(void) { - const jsmntok_t *toks; + jsmntok_t *toks; char *buf; - bool ok; + bool ok, complete; + jsmn_parser parser; buf = "[\"e1\", [\"e2\", \"e3\"]]"; - toks = json_parse_input(tmpctx, buf, strlen(buf), &ok); + toks = toks_alloc(tmpctx); + jsmn_init(&parser); + ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete); assert(ok); + assert(complete); /* size only counts *direct* children */ assert(toks[0].size == 2); assert(toks[2].size == 2); buf = "[[\"e1\", \"e2\"], \"e3\"]"; - toks = json_parse_input(tmpctx, buf, strlen(buf), &ok); + toks_reset(toks); + jsmn_init(&parser); + ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete); assert(ok); + assert(complete); /* size only counts *direct* children */ assert(toks[0].size == 2); assert(toks[1].size == 2); buf = "{\"e1\" : {\"e2\": 2, \"e3\": 3}}"; - toks = json_parse_input(tmpctx, buf, strlen(buf), &ok); + toks_reset(toks); + jsmn_init(&parser); + ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete); assert(ok); + assert(complete); /* size only counts *direct* children */ assert(toks[0].size == 1); assert(toks[2].size == 2); buf = "{\"e1\" : {\"e2\": 2, \"e3\": 3}, \"e4\" : {\"e5\": 5, \"e6\": 6}}"; - toks = json_parse_input(tmpctx, buf, strlen(buf), &ok); + toks_reset(toks); + jsmn_init(&parser); + ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete); assert(ok); + assert(complete); /* size only counts *direct* children */ assert(toks[0].size == 2); assert(toks[2].size == 2); @@ -139,7 +152,9 @@ static void test_json_tok_size(void) /* This should *not* parse! (used to give toks[0]->size == 3!) */ buf = "{ \"\" \"\" \"\" }"; - toks = json_parse_input(tmpctx, buf, strlen(buf), &ok); + toks_reset(toks); + jsmn_init(&parser); + ok = json_parse_input(&parser, &toks, buf, strlen(buf), &complete); assert(!ok); /* This should *not* parse! (used to give toks[0]->size == 2!) */ diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index 97577cd58..df70e08c7 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -913,7 +913,8 @@ static struct io_plan *read_json(struct io_conn *conn, struct json_connection *jcon) { jsmntok_t *toks; - bool valid; + bool complete; + jsmn_parser parser; if (jcon->len_read) log_io(jcon->log, LOG_IO_IN, NULL, "", @@ -930,21 +931,18 @@ static struct io_plan *read_json(struct io_conn *conn, return io_wait(conn, conn, read_json, jcon); } - toks = json_parse_input(jcon->buffer, jcon->buffer, jcon->used, &valid); - if (!toks) { - if (!valid) { - log_unusual(jcon->log, - "Invalid token in json input: '%.*s'", - (int)jcon->used, jcon->buffer); - json_command_malformed( - jcon, "null", - "Invalid token in json input"); - return io_halfclose(conn); - } - /* We need more. */ - goto read_more; + toks = toks_alloc(jcon->buffer); + jsmn_init(&parser); + if (!json_parse_input(&parser, &toks, jcon->buffer, jcon->used, + &complete)) { + json_command_malformed(jcon, "null", + "Invalid token in json input"); + return io_halfclose(conn); } + if (!complete) + goto read_more; + /* Empty buffer? (eg. just whitespace). */ if (tal_count(toks) == 1) { jcon->used = 0; diff --git a/lightningd/plugin.c b/lightningd/plugin.c index a6e51766a..0ea8b92b2 100644 --- a/lightningd/plugin.c +++ b/lightningd/plugin.c @@ -386,10 +386,11 @@ static const char *plugin_read_json_one(struct plugin *plugin, bool *complete, bool *destroyed) { - bool valid; - const jsmntok_t *toks, *jrtok, *idtok; + jsmntok_t *toks; + const jsmntok_t *jrtok, *idtok; struct plugin_destroyed *pd; const char *err; + jsmn_parser parser; *destroyed = false; /* Note that in the case of 'plugin stop' this can free request (since @@ -398,28 +399,30 @@ static const char *plugin_read_json_one(struct plugin *plugin, /* FIXME: This could be done more efficiently by storing the * toks and doing an incremental parse, like lightning-cli * does. */ - toks = json_parse_input(plugin->buffer, plugin->buffer, plugin->used, - &valid); - if (!toks) { - if (!valid) { - return tal_fmt(plugin, - "Failed to parse JSON response '%.*s'", - (int)plugin->used, plugin->buffer); - } + toks = toks_alloc(plugin); + jsmn_init(&parser); + if (!json_parse_input(&parser, &toks, plugin->buffer, plugin->used, + complete)) { + return tal_fmt(plugin, + "Failed to parse JSON response '%.*s'", + (int)plugin->used, plugin->buffer); + } + + if (!*complete) { /* We need more. */ - *complete = false; + tal_free(toks); return NULL; } /* Empty buffer? (eg. just whitespace). */ if (tal_count(toks) == 1) { + tal_free(toks); plugin->used = 0; /* We need more. */ *complete = false; return NULL; } - *complete = true; jrtok = json_get_member(plugin->buffer, toks, "jsonrpc"); idtok = json_get_member(plugin->buffer, toks, "id"); diff --git a/lightningd/test/run-jsonrpc.c b/lightningd/test/run-jsonrpc.c index cdfea9955..b2a488f9f 100644 --- a/lightningd/test/run-jsonrpc.c +++ b/lightningd/test/run-jsonrpc.c @@ -132,11 +132,12 @@ static int test_json_filter(void) struct json_stream *result = new_json_stream(NULL, NULL, NULL); jsmntok_t *toks; const jsmntok_t *x; - bool valid; + bool valid, complete; int i; char *badstr = tal_arr(result, char, 256); const char *str; size_t len; + jsmn_parser parser; /* Fill with junk, and nul-terminate (256 -> 0) */ for (i = 1; i < 257; i++) @@ -150,9 +151,11 @@ static int test_json_filter(void) str = json_out_contents(result->jout, &len); str = tal_strndup(result, str, len); - toks = json_parse_input(str, str, strlen(str), &valid); + toks = toks_alloc(str); + jsmn_init(&parser); + valid = json_parse_input(&parser, &toks, str, strlen(str), &complete); assert(valid); - assert(toks); + assert(complete); assert(toks[0].type == JSMN_OBJECT); x = json_get_member(str, toks, "x"); @@ -240,32 +243,54 @@ static void test_json_partial(void) /* Test that we can segment and parse a stream of json objects correctly. */ static void test_json_stream(void) { - bool valid; + bool valid, complete; char *input, *talstr; jsmntok_t *toks; + jsmn_parser parser; /* Multiple full messages in a single buffer (happens when buffer * boundary coincides with message boundary, or read returned after * timeout. */ input = "{\"x\":\"x\"}{\"y\":\"y\"}"; talstr = tal_strndup(NULL, input, strlen(input)); - toks = json_parse_input(talstr, talstr, strlen(talstr), &valid); - assert(toks); + toks = toks_alloc(NULL); + jsmn_init(&parser); + valid = json_parse_input(&parser, &toks, talstr, strlen(talstr), &complete); assert(tal_count(toks) == 4); assert(toks[0].start == 0 && toks[0].end == 9); assert(valid); + assert(complete); tal_free(talstr); /* Multiple messages, and the last one is partial, far more likely than * accidentally getting the boundaries to match. */ input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z"; talstr = tal_strndup(NULL, input, strlen(input)); - toks = json_parse_input(talstr, talstr, strlen(talstr), &valid); + toks_reset(toks); + jsmn_init(&parser); + valid = json_parse_input(&parser, &toks, talstr, strlen(talstr), &complete); assert(toks); assert(tal_count(toks) == 4); assert(toks[0].start == 0 && toks[0].end == 9); assert(valid); + assert(complete); tal_free(talstr); + + /* We can do this incrementally, too. */ + toks_reset(toks); + input = "{\"x\":\"x\"}"; + jsmn_init(&parser); + for (size_t i = 0; i <= strlen(input); i++) { + valid = json_parse_input(&parser, &toks, input, i, &complete); + assert(valid); + if (i == strlen(input)) + assert(complete); + else + assert(!complete); + } + assert(tal_count(toks) == 4); + assert(toks[0].start == 0 && toks[0].end == 9); + tal_free(toks); } int main(void) diff --git a/plugins/libplugin.c b/plugins/libplugin.c index 1fef9c6bf..a514153f4 100644 --- a/plugins/libplugin.c +++ b/plugins/libplugin.c @@ -1089,21 +1089,25 @@ static void ld_command_handle(struct plugin *plugin, */ static bool ld_read_json_one(struct plugin *plugin) { - bool valid; - const jsmntok_t *toks; + bool complete; + jsmntok_t *toks; struct command *cmd = tal(plugin, struct command); + jsmn_parser parser; /* FIXME: This could be done more efficiently by storing the * toks and doing an incremental parse, like lightning-cli * does. */ - toks = json_parse_input(NULL, plugin->buffer, plugin->used, - &valid); - if (!toks) { - if (!valid) { - plugin_err(plugin, "Failed to parse JSON response '%.*s'", - (int)plugin->used, plugin->buffer); - return false; - } + toks = toks_alloc(NULL); + jsmn_init(&parser); + if (!json_parse_input(&parser, &toks, plugin->buffer, plugin->used, + &complete)) { + tal_free(toks); + plugin_err(plugin, "Failed to parse JSON response '%.*s'", + (int)plugin->used, plugin->buffer); + return false; + } + + if (!complete) { /* We need more. */ return false; }