check-bolt: check that comments in code match the specs.

And fix the mistakes!

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

.gitignore

@@ -11,3 +11,4 @@ libsecp256k1.a
 libsecp256k1.la
 gen_*
 daemon/lightning-cli
+check-bolt

Makefile

@@ -7,6 +7,9 @@ PROTOCC:=protoc-c
 # We use our own internal ccan copy.
 CCANDIR := ccan
 
+# Where we keep the BOLT RFCs
+BOLTDIR := ../lightning-rfc/bolts/
+
 # Bitcoin uses DER for signatures (Add BIP68 & HAS_CSV if it's supported)
 BITCOIN_FEATURES :=				\
 	-DHAS_BIP68=1				\
@@ -212,7 +215,15 @@ check-makefile: check-daemon-makefile
 	@if [ x"`ls *.h | grep -v ^gen_ | fgrep -v lightning.pb-c.h | tr '\n' ' '`" != x"$(CORE_HEADERS) " ]; then echo CORE_HEADERS incorrect; exit 1; fi
 	@if [ x"$(CCANDIR)/config.h `find $(CCANDIR)/ccan -name '*.h' | grep -v /test/ | LC_ALL=C sort | tr '\n' ' '`" != x"$(CCAN_HEADERS) " ]; then echo CCAN_HEADERS incorrect; exit 1; fi
 
-check-source: check-makefile				\
+# Any mention of BOLT# must be followed by an exact quote, modulo whitepace.
+check-source-bolt: check-bolt
+	@if [ ! -d $(BOLTDIR) ]; then echo Not checking BOLT references: BOLTDIR $(BOLTDIR) does not exist >&2; else ./check-bolt $(BOLTDIR) $(CORE_SRC) $(BITCOIN_SRC) $(DAEMON_SRC) $(CORE_HEADERS) $(BITCOIN_HEADERS) $(DAEMON_HEADERS); fi
+
+check-bolt: check-bolt.o $(CCAN_OBJS)
+
+check-bolt.o: $(CCAN_HEADERS)
+
+check-source: check-makefile check-source-bolt		\
 	$(CORE_SRC:%=check-src-include-order/%)		\
 	$(BITCOIN_SRC:%=check-src-include-order/%)	\
 	$(CORE_HEADERS:%=check-hdr-include-order/%)	\

check-bolt.c

@@ -0,0 +1,239 @@
+/* Simple program to search for BOLT references in C files and make sure
+ * they're accurate. */
+#include <ccan/err/err.h>
+#include <ccan/opt/opt.h>
+#include <ccan/str/str.h>
+#include <ccan/tal/grab_file/grab_file.h>
+#include <ccan/tal/path/path.h>
+#include <ccan/tal/str/str.h>
+#include <ccan/tal/tal.h>
+#include <sys/types.h>
+#include <dirent.h>
+
+static bool verbose = false;
+
+/* Turn any whitespace into a single space. */
+static char *canonicalize(char *str)
+{
+	char *to = str, *from = str;
+	bool have_space = true;
+
+	while (*from) {
+		if (cisspace(*from)) {
+			if (!have_space)
+				*(to++) = ' ';
+			have_space = true;
+		} else {
+			*(to++) = *from;
+			have_space = false;
+		}
+		from++;
+	}
+	if (have_space && to != str)
+		to--;
+	*to = '\0';
+	tal_resize(&str, to + 1 - str);
+	return str;
+}
+
+static char **get_bolt_files(const char *dir)
+{
+	struct dirent *e;
+	char **bolts = tal_arr(NULL, char *, 0);
+	DIR *d = opendir(dir);
+	if (!d)
+		err(1, "Opening BOLT dir %s", dir);
+
+	while ((e = readdir(d)) != NULL) {
+		char *endp;
+		unsigned long l;
+
+		/* Must start with the bold number. */
+		l = strtoul(e->d_name, &endp, 10);
+		if (endp == e->d_name)
+			continue;
+
+		/* Must end in .md */
+		if (!strends(e->d_name, ".md"))
+			continue;
+
+		if (l >= tal_count(bolts))
+			tal_resizez(&bolts, l+1);
+
+		if (verbose)
+			printf("Found bolt %s: #%lu\n", e->d_name, l);
+
+		bolts[l] = canonicalize(grab_file(NULL,
+						  path_join(NULL, dir,
+							    e->d_name)));
+	}
+	return bolts;
+}
+
+static char *find_bolt_ref(char *p, size_t *len, size_t *bolt)
+{
+	for (;;) {
+		char *end;
+
+		/* BOLT #X: */
+		p = strstr(p, "BOLT");
+		if (!p)
+			return NULL;
+		p += 4;
+		while (cisspace(*p))
+			p++;
+		if (*p != '#')
+			continue;
+		p++;
+		*bolt = strtoul(p, &end, 10);
+		if (!*bolt || p == end)
+			continue;
+		p = end;
+		while (cisspace(*p))
+			p++;
+		if (*p != ':')
+			continue;
+		p++;
+
+		end = strstr(p, "*/");
+		if (!end)
+			*len = strlen(p);
+		else
+			*len = end - p;
+		return p;
+	}
+}
+
+static char *code_to_regex(const char *code, size_t len, bool escape)
+{
+	char *pattern = tal_arr(NULL, char, len*2 + 1), *p;
+	size_t i;
+	bool after_nl = false;
+
+	/* We swallow '*' if first in line: block comments */
+	p = pattern;
+	for (i = 0; i < len; i++) {
+		/* ... matches anything. */
+		if (strstarts(code + i, "...")) {
+			*(p++) = '.';
+			*(p++) = '*';
+			i += 2;
+			continue;
+		}
+
+		switch (code[i]) {
+		case '\n':
+			after_nl = true;
+			*(p++) = code[i];
+			break;
+
+		case '*':
+			if (after_nl) {
+				after_nl = false;
+				continue;
+			}
+			/* Fall thru. */
+		case '.':
+		case '$':
+		case '^':
+		case '[':
+		case ']':
+		case '(':
+		case ')':
+		case '+':
+		case '|':
+			if (escape)
+				*(p++) = '\\';
+			/* Fall thru */
+		default:
+			*(p++) = code[i];
+		}
+	}
+	*p = '\0';
+	return canonicalize(pattern);
+}
+
+static void fail(const char *filename, const char *raw, const char *pos,
+		 size_t len, const char *bolt)
+{
+	unsigned line = 0; /* Out-by-one below */
+	const char *l = raw;
+
+	while (l < pos) {
+		l = strchr(l, '\n');
+		line++;
+		if (!l)
+			l = pos + strlen(pos);
+		else
+			l++;
+	}
+
+	if (bolt) {
+		char *try;
+
+		fprintf(stderr, "%s:%u:%.*s\n", filename, line,
+			(int)(l - pos), pos);
+		/* Try to find longest match, as a hint. */
+		try = code_to_regex(pos, len, false);
+		while (strlen(try)) {
+			const char *p = strstr(bolt, try);
+			if (p) {
+				fprintf(stderr, "Closest match: %s...[%.20s]\n",
+					try, p + strlen(try));
+				break;
+			}
+			try[strlen(try)-1] = '\0';
+		}
+	} else {
+		fprintf(stderr, "%s:%u:Unknown bolt\n", filename, line);
+	}		
+	exit(1);
+}
+	
+int main(int argc, char *argv[])
+{
+	char **bolts;
+	int i;
+
+	err_set_progname(argv[0]);
+
+	opt_register_noarg("--help|-h", opt_usage_and_exit,
+			   "<bolt-dir> <srcfile>...\n"
+			   "A source checker for BOLT RFC references.",
+			   "Print this message.");
+	opt_register_noarg("--verbose", opt_set_bool, &verbose,
+			   "Print out files as we find them");
+
+	opt_parse(&argc, argv, opt_log_stderr_exit);
+	if (argc < 2)
+		opt_usage_exit_fail("Expected a bolt directory");
+
+	bolts = get_bolt_files(argv[1]);
+
+	for (i = 2; i < argc; i++) {
+		char *f = grab_file(NULL, argv[i]), *p;
+		size_t len, bolt;
+		if (!f)
+			err(1, "Loading %s", argv[i]);
+
+		if (verbose)
+			printf("Checking %s...\n", argv[i]);
+
+		p = f;
+		while ((p = find_bolt_ref(p, &len, &bolt)) != NULL) {
+			char *pattern = code_to_regex(p, len, true);
+			if (bolt >= tal_count(bolts) || !bolts[bolt])
+				fail(argv[i], f, p, len, NULL);
+			if (!tal_strreg(f, bolts[bolt], pattern, NULL))
+				fail(argv[i], f, p, len, bolts[bolt]);
+
+			if (verbose)
+				printf("  Found %.10s... in %zu\n",
+				       p, bolt);
+			p += len;
+		}
+		tal_free(f);
+	}
+	return 0;
+}
+	

daemon/cryptopkt.c

@@ -52,8 +52,8 @@ struct enckey {
 
 
 /* BOLT #1:
- * sending-key: SHA256(shared-secret || sending-node-id)
+ * * sending-key: SHA256(shared-secret || sending-node-session-pubkey)
- * receiving-key: SHA256(shared-secret || receiving-node-id)
+ * * receiving-key: SHA256(shared-secret || receiving-node-session-pubkey)
  */
 static struct enckey enckey_from_secret(const unsigned char secret[32],
 					const unsigned char serial_pubkey[33])

daemon/lightningd.c

@@ -150,7 +150,7 @@ static void check_config(struct lightningd_state *dstate)
 {
 	/* BOLT #2:
 	 * The sender MUST set `close_fee` lower than or equal to the
-	 * fee of the final commitment transaction.
+	 * fee of the final commitment transaction
 	 */
 
 	/* We do this by ensuring it's less than the minimum we would accept. */

daemon/packets.c

@@ -811,7 +811,8 @@ Pkt *accept_pkt_close_sig(struct peer *peer, const Pkt *pkt, bool *acked,
 	/* BOLT #2:
 	 *
 	 * The receiver MUST check `sig` is valid for the close
-	 * transaction, and MUST fail the connection if it is not. */
+	 * transaction with the given `close_fee`, and MUST fail the
+	 * connection if it is not. */
 	theirsig.stype = SIGHASH_ALL;
 	if (!proto_to_signature(c->sig, &theirsig.sig))
 		return pkt_err(peer, "Invalid signature format");

daemon/peer.c

@@ -971,7 +971,7 @@ void peer_calculate_close_fee(struct peer *peer)
 
 	/* BOLT #2:
 	 * The sender MUST set `close_fee` lower than or equal to the
-	 * fee of the final commitment transaction and MUST set
+	 * fee of the final commitment transaction, and MUST set
 	 * `close_fee` to an even number of satoshis.
 	 */
 	maxfee = commit_tx_fee(peer->us.commit->tx, peer->anchor.satoshis);

funding.c

@@ -8,10 +8,9 @@ uint64_t fee_by_feerate(size_t txsize, uint32_t fee_rate)
 {
 	/* BOLT #2:
 	 * 
-	 * The fee for a commitment transaction MUST be calculated by
+	 * The fee for a transaction MUST be calculated by multiplying this
-	 * the multiplying this bytescount by the fee rate, dividing
+	 * bytecount by the fee rate, dividing by 1000 and truncating
-	 * by 1000 and truncating (rounding down) the result to an
+	 * (rounding down) the result to an even number of satoshis.
-	 * even number of satoshis.
 	 */
 	return txsize * fee_rate / 2000 * 2;
 }