common: allow optional fields in json_scan().

Currently it fails if a field is missing, but sometimes that's OK.  So
allow a fieldname ending in `?` to mean "skip over if it's missing".

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/json_parse.c

@@ -193,6 +193,9 @@ static const char *handle_percent(const char *buffer,
 		void **p;
 		p = va_arg(*ap, void **);
 		talfmt = va_arg(*ap, void *(*)(void *, const char *, const jsmntok_t *));
+		/* If we're skipping this, don't call fmt! */
+		if (!tok)
+			return NULL;
 		*p = talfmt(ctx, buffer, tok);
 		if (*p != NULL)
 			return NULL;
@@ -202,6 +205,9 @@ static const char *handle_percent(const char *buffer,
 
 		p = va_arg(*ap, void *);
 		fmt = va_arg(*ap, bool (*)(const char *, const jsmntok_t *, void *));
+		/* If we're skipping this, don't call fmt! */
+		if (!tok)
+			return NULL;
 		if (fmt(buffer, tok, p))
 			return NULL;
 	}
@@ -215,7 +221,8 @@ static const char *handle_percent(const char *buffer,
 /* GUIDE := OBJ | ARRAY | '%'
  * OBJ := '{' FIELDLIST '}'
  * FIELDLIST := FIELD [',' FIELD]*
- * FIELD := LITERAL ':' FIELDVAL
+ * FIELD := MEMBER ':' FIELDVAL
+ * MEMBER := LITERAL | LITERAL '?'
  * FIELDVAL := OBJ | ARRAY | LITERAL | '%'
  * ARRAY := '[' ARRLIST ']'
  * ARRLIST := ARRELEM [',' ARRELEM]*
@@ -264,6 +271,15 @@ static void guide_must_be(const char **guide, char c)
 	assert(actual == c);
 }
 
+static bool guide_maybe_optional(const char **guide)
+{
+	if (**guide == '?') {
+		guide_consume_one(guide);
+		return true;
+	}
+	return false;
+}
+
 /* Recursion: return NULL on success, errmsg on fail */
 static const char *parse_obj(const char *buffer,
 			     const jsmntok_t *tok,
@@ -325,8 +341,8 @@ static const char *parse_fieldval(const char *buffer,
 
 		/* Literal must match exactly (modulo quotes for strings) */
 		parse_literal(guide, &literal, &len);
-		if (!memeq(buffer + tok->start, tok->end - tok->start,
+		if (tok && !memeq(buffer + tok->start, tok->end - tok->start,
-			   literal, len)) {
+				  literal, len)) {
 			return tal_fmt(tmpctx,
 				       "%.*s does not match expected %.*s",
 				       json_tok_full_len(tok),
@@ -345,14 +361,20 @@ static const char *parse_field(const char *buffer,
 	const jsmntok_t *member;
 	size_t len;
 	const char *memname;
+	bool optional;
 
 	parse_literal(guide, &memname, &len);
+	optional = guide_maybe_optional(guide);
 	guide_must_be(guide, ':');
 
-	member = json_get_membern(buffer, tok, memname, len);
+	if (tok) {
-	if (!member) {
+		member = json_get_membern(buffer, tok, memname, len);
-		return tal_fmt(tmpctx, "object does not have member %.*s",
+		if (!member && !optional) {
-			       (int)len, memname);
+			return tal_fmt(tmpctx, "object does not have member %.*s",
+				       (int)len, memname);
+		}
+	} else {
+		member = NULL;
 	}
 
 	return parse_fieldval(buffer, member, guide, ap);
@@ -385,7 +407,7 @@ static const char *parse_obj(const char *buffer,
 
 	guide_must_be(guide, '{');
 
-	if (tok->type != JSMN_OBJECT) {
+	if (tok && tok->type != JSMN_OBJECT) {
 		return tal_fmt(tmpctx, "token is not an object: %.*s",
 			       json_tok_full_len(tok),
 			       json_tok_full(buffer, tok));
@@ -410,12 +432,16 @@ static const char *parse_arrelem(const char *buffer,
 	parse_number(guide, &idx);
 	guide_must_be(guide, ':');
 
-	member = json_get_arr(tok, idx);
+	if (tok) {
-	if (!member) {
+		member = json_get_arr(tok, idx);
-		return tal_fmt(tmpctx, "token has no index %u: %.*s",
+		if (!member) {
-			       idx,
+			return tal_fmt(tmpctx, "token has no index %u: %.*s",
-			       json_tok_full_len(tok),
+				       idx,
-			       json_tok_full(buffer, tok));
+				       json_tok_full_len(tok),
+				       json_tok_full(buffer, tok));
+		}
+	} else {
+		member = NULL;
 	}
 
 	return parse_fieldval(buffer, member, guide, ap);
@@ -448,7 +474,7 @@ static const char *parse_arr(const char *buffer,
 
 	guide_must_be(guide, '[');
 
-	if (tok->type != JSMN_ARRAY) {
+	if (tok && tok->type != JSMN_ARRAY) {
 		return tal_fmt(tmpctx, "token is not an array: %.*s",
 			       json_tok_full_len(tok),
 			       json_tok_full(buffer, tok));

common/test/run-json_scan.c

@@ -146,6 +146,24 @@ int main(int argc, char *argv[])
 	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{three:{deeper:17}},arr:[1:2]}"));
 	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{three:{deeper:17}},arr:[1:2,2:[0:3,1:4]]}"));
 
+	/* Optional fields which are present are fine. */
+	assert(!json_scan(tmpctx, buf, toks, "{1?:one}"));
+	assert(!json_scan(tmpctx, buf, toks, "{1?:one,2?:two}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2?:two,1?:one}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2?:two,1?:one,3?:{three?:{deeper?:17}}}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2?:two,1?:one,3?:{three?:{deeper?:17}},arr?:[0:{1?:arrone}]}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2?:two,1?:one,3?:{three?:{deeper?:17}},arr?:[1:2]}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2?:two,1?:one,3?:{three?:{deeper?:17}},arr?:[1:2,2:[0:3,1:4]]}"));
+
+	/* Optional field which are missing are fine too */
+	assert(!json_scan(tmpctx, buf, toks, "{5?:one}"));
+	assert(!json_scan(tmpctx, buf, toks, "{1:one,5?:two}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,5?:one}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,5?:{three:{deeper:17}}}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{five?:{deeper:17}},arr:[0:{1:arrone}]}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{three:{notdeeper?:17}},arr:[1:2]}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{three:{deeper:17}},notarr?:[1:2,2:[0:3,1:4]]}"));
+
 	/* These do not match */
 	err = json_scan(tmpctx, buf, toks, "{2:one}");
 	assert(streq(err, "Parsing '{2:one': \"two\" does not match expected one"));