gossipd: make use of new ratelimit bit in gossip_store length mask

routing.c now flags rate-limited gossip as it enters the gossip_store but
makes use of it in updating the routing graph. Flagged gossip is not
rebroadcast to gossip peers.

Changelog-Changed: gossipd: now accepts spam gossip, but squelches it for
peers.

contrib/pyln-client/pyln/client/gossmap.py

@@ -12,8 +12,10 @@ import struct
 GOSSIP_STORE_VERSIONS = [0x09, 0x0a]
 GOSSIP_STORE_LEN_DELETED_BIT = 0x80000000
 GOSSIP_STORE_LEN_PUSH_BIT = 0x40000000
+GOSSIP_STORE_LEN_RATELIMIT_BIT = 0x20000000
 GOSSIP_STORE_LEN_MASK = (~(GOSSIP_STORE_LEN_PUSH_BIT
-                           | GOSSIP_STORE_LEN_DELETED_BIT))
+                           | GOSSIP_STORE_LEN_DELETED_BIT
+                           | GOSSIP_STORE_LEN_RATELIMIT_BIT))
 
 # These duplicate constants in lightning/gossipd/gossip_store_wiregen.h
 WIRE_GOSSIP_STORE_PRIVATE_CHANNEL = 4104

devtools/dump-gossipstore.c

@@ -52,10 +52,11 @@ int main(int argc, char *argv[])
 		struct short_channel_id scid;
 		u32 msglen = be32_to_cpu(hdr.len);
 		u8 *msg, *inner;
-		bool deleted, push;
+		bool deleted, push, ratelimit;
 
 		deleted = (msglen & GOSSIP_STORE_LEN_DELETED_BIT);
 		push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT);
+		ratelimit = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT);
 
 		msglen &= GOSSIP_STORE_LEN_MASK;
 		msg = tal_arr(NULL, u8, msglen);
@@ -66,9 +67,10 @@ int main(int argc, char *argv[])
 		    != crc32c(be32_to_cpu(hdr.timestamp), msg, msglen))
 			warnx("Checksum verification failed");
 
-		printf("%zu: %s%s", off,
+		printf("%zu: %s%s%s", off,
 		       deleted ? "DELETED " : "",
-		       push ? "PUSH " : "");
+		       push ? "PUSH " : "",
+		       ratelimit ? "RATE-LIMITED " : "");
 		if (deleted && !print_deleted) {
 			printf("\n");
 			goto end;

gossipd/gossip_store.c

@@ -71,7 +71,7 @@ static ssize_t gossip_pwritev(int fd, const struct iovec *iov, int iovcnt,
 #endif /* !HAVE_PWRITEV */
 
 static bool append_msg(int fd, const u8 *msg, u32 timestamp,
-		       bool push, u64 *len)
+		       bool push, bool spam, u64 *len)
 {
 	struct gossip_hdr hdr;
 	u32 msglen;
@@ -84,6 +84,8 @@ static bool append_msg(int fd, const u8 *msg, u32 timestamp,
 	hdr.len = cpu_to_be32(msglen);
 	if (push)
 		hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_PUSH_BIT);
+	if (spam)
+		hdr.len |= CPU_TO_BE32(GOSSIP_STORE_LEN_RATELIMIT_BIT);
 	hdr.crc = cpu_to_be32(crc32c(timestamp, msg, msglen));
 	hdr.timestamp = cpu_to_be32(timestamp);
 
@@ -277,7 +279,7 @@ static u32 gossip_store_compact_offline(struct routing_state *rstate)
 	oldlen = lseek(old_fd, SEEK_END, 0);
 	newlen = lseek(new_fd, SEEK_END, 0);
 	append_msg(old_fd, towire_gossip_store_ended(tmpctx, newlen),
-		   0, true, &oldlen);
+		   0, true, false, &oldlen);
 	close(old_fd);
 	status_debug("gossip_store_compact_offline: %zu deleted, %zu copied",
 		     deleted, count);
@@ -565,7 +567,7 @@ bool gossip_store_compact(struct gossip_store *gs)
 
 	/* Write end marker now new one is ready */
 	append_msg(gs->fd, towire_gossip_store_ended(tmpctx, len),
-		   0, true, &gs->len);
+		   0, true, false, &gs->len);
 
 	gs->count = count;
 	gs->deleted = 0;
@@ -586,19 +588,19 @@ disable:
 
 u64 gossip_store_add(struct gossip_store *gs, const u8 *gossip_msg,
 		     u32 timestamp, bool push,
-		     const u8 *addendum)
+		     bool spam, const u8 *addendum)
 {
 	u64 off = gs->len;
 
 	/* Should never get here during loading! */
 	assert(gs->writable);
 
-	if (!append_msg(gs->fd, gossip_msg, timestamp, push, &gs->len)) {
+	if (!append_msg(gs->fd, gossip_msg, timestamp, push, spam, &gs->len)) {
 		status_broken("Failed writing to gossip store: %s",
 			      strerror(errno));
 		return 0;
 	}
-	if (addendum && !append_msg(gs->fd, addendum, 0, false, &gs->len)) {
+	if (addendum && !append_msg(gs->fd, addendum, 0, false, false, &gs->len)) {
 		status_broken("Failed writing addendum to gossip store: %s",
 			      strerror(errno));
 		return 0;
@@ -615,7 +617,7 @@ u64 gossip_store_add_private_update(struct gossip_store *gs, const u8 *update)
 	/* A local update for an unannounced channel: not broadcastable, but
 	 * otherwise the same as a normal channel_update */
 	const u8 *pupdate = towire_gossip_store_private_update(tmpctx, update);
-	return gossip_store_add(gs, pupdate, 0, false, NULL);
+	return gossip_store_add(gs, pupdate, 0, false, false, NULL);
 }
 
 /* Returns index of following entry. */
@@ -675,7 +677,7 @@ void gossip_store_mark_channel_deleted(struct gossip_store *gs,
 				       const struct short_channel_id *scid)
 {
 	gossip_store_add(gs, towire_gossip_store_delete_chan(tmpctx, scid),
-			 0, false, NULL);
+			 0, false, false, NULL);
 }
 
 const u8 *gossip_store_get(const tal_t *ctx,

gossipd/gossip_store.h

@@ -40,11 +40,12 @@ u64 gossip_store_add_private_update(struct gossip_store *gs, const u8 *update);
  * @gossip_msg: the gossip message to insert.
  * @timestamp: the timestamp for filtering of this messsage.
  * @push: true if this should be sent to peers despite any timestamp filters.
+ * @spam: true if this message is rate-limited and squelched to peers.
  * @addendum: another message to append immediately after this
  *            (for appending amounts to channel_announcements for internal use).
  */
 u64 gossip_store_add(struct gossip_store *gs, const u8 *gossip_msg,
-		     u32 timestamp, bool push, const u8 *addendum);
+		     u32 timestamp, bool push, bool spam, const u8 *addendum);
 
 
 /**

gossipd/routing.c

@@ -437,6 +437,7 @@ static void force_node_announce_rexmit(struct routing_state *rstate,
 					     announce,
 					     node->bcast.timestamp,
 					     is_local,
+					     false,
 					     NULL);
 }
 
@@ -790,6 +791,7 @@ static void add_channel_announce_to_broadcast(struct routing_state *rstate,
 						     channel_announce,
 						     chan->bcast.timestamp,
 						     is_local,
+						     false,
 						     addendum);
 	rstate->local_channel_announced |= is_local;
 }
@@ -1255,6 +1257,7 @@ bool routing_add_channel_update(struct routing_state *rstate,
 	struct unupdated_channel *uc;
 	u8 direction;
 	struct amount_sat sat;
+	bool spam;
 
 	/* Make sure we own msg, even if we don't save it. */
 	if (taken(update))
@@ -1360,16 +1363,19 @@ bool routing_add_channel_update(struct routing_state *rstate,
 		if (!ratelimit(rstate,
 			       &hc->tokens, hc->bcast.timestamp, timestamp)) {
 			status_peer_debug(peer ? &peer->id : NULL,
-					  "Ignoring spammy update for %s/%u"
+					  "Spammy update for %s/%u flagged"
 					  " (last %u, now %u)",
 					  type_to_string(tmpctx,
 							 struct short_channel_id,
 							 &short_channel_id),
 					  direction,
 					  hc->bcast.timestamp, timestamp);
-			/* Ignoring != failing */
+			spam = true;
-			return true;
+		} else {
+			spam = false;
 		}
+	} else {
+		spam = false;
 	}
 
 	chan->half[direction].bcast.timestamp = timestamp;
@@ -1414,7 +1420,7 @@ bool routing_add_channel_update(struct routing_state *rstate,
 			= gossip_store_add(rstate->gs, update,
 					   hc->bcast.timestamp,
 					   local_direction(rstate, chan, NULL),
-					   NULL);
+					   spam, NULL);
 		if (hc->bcast.timestamp > rstate->last_timestamp
 		    && hc->bcast.timestamp < time_now().ts.tv_sec)
 			rstate->last_timestamp = hc->bcast.timestamp;
@@ -1581,6 +1587,7 @@ bool routing_add_node_announcement(struct routing_state *rstate,
 	u8 alias[32];
 	u8 *features, *addresses;
 	struct tlv_node_ann_tlvs *na_tlv;
+	bool spam;
 
 	if (was_unknown)
 		*was_unknown = false;
@@ -1670,15 +1677,18 @@ bool routing_add_node_announcement(struct routing_state *rstate,
 		if (!ratelimit(rstate,
 			       &node->tokens, node->bcast.timestamp, timestamp)) {
 			status_peer_debug(peer ? &peer->id : NULL,
-					  "Ignoring spammy nannounce for %s"
+					  "Spammy nannounce for %s flagged"
 					  " (last %u, now %u)",
 					  type_to_string(tmpctx,
 							 struct node_id,
 							 &node_id),
 					  node->bcast.timestamp, timestamp);
-			/* Ignoring != failing */
+			spam = true;
-			return true;
+		} else {
+			spam = false;
 		}
+	} else {
+		spam = false;
 	}
 
 	/* Harmless if it was never added */
@@ -1699,6 +1709,7 @@ bool routing_add_node_announcement(struct routing_state *rstate,
 					   node->bcast.timestamp,
 					   node_id_eq(&node_id,
 						      &rstate->local_id),
+					   spam,
 					   NULL);
 		peer_supplied_good_gossip(peer, 1);
 	}
@@ -1919,7 +1930,7 @@ bool routing_add_private_channel(struct routing_state *rstate,
 		u8 *msg = towire_gossip_store_private_channel(tmpctx,
 							      capacity,
 							      chan_ann);
-		index = gossip_store_add(rstate->gs, msg, 0, false, NULL);
+		index = gossip_store_add(rstate->gs, msg, 0, false, false, NULL);
 	}
 	chan->bcast.index = index;
 	return true;

gossipd/test/run-check_channel_announcement.c

@@ -66,7 +66,7 @@ void ecdh(const struct pubkey *point UNNEEDED, struct secret *ss UNNEEDED)
 { fprintf(stderr, "ecdh called!\n"); abort(); }
 /* Generated stub for gossip_store_add */
 u64 gossip_store_add(struct gossip_store *gs UNNEEDED, const u8 *gossip_msg UNNEEDED,
-		     u32 timestamp UNNEEDED, bool push UNNEEDED, const u8 *addendum UNNEEDED)
+		     u32 timestamp UNNEEDED, bool push UNNEEDED, bool spam UNNEEDED, const u8 *addendum UNNEEDED)
 { fprintf(stderr, "gossip_store_add called!\n"); abort(); }
 /* Generated stub for gossip_store_add_private_update */
 u64 gossip_store_add_private_update(struct gossip_store *gs UNNEEDED, const u8 *update UNNEEDED)

gossipd/test/run-txout_failure.c

@@ -37,7 +37,7 @@ void ecdh(const struct pubkey *point UNNEEDED, struct secret *ss UNNEEDED)
 { fprintf(stderr, "ecdh called!\n"); abort(); }
 /* Generated stub for gossip_store_add */
 u64 gossip_store_add(struct gossip_store *gs UNNEEDED, const u8 *gossip_msg UNNEEDED,
-		     u32 timestamp UNNEEDED, bool push UNNEEDED, const u8 *addendum UNNEEDED)
+		     u32 timestamp UNNEEDED, bool push UNNEEDED, bool spam UNNEEDED, const u8 *addendum UNNEEDED)
 { fprintf(stderr, "gossip_store_add called!\n"); abort(); }
 /* Generated stub for gossip_store_add_private_update */
 u64 gossip_store_add_private_update(struct gossip_store *gs UNNEEDED, const u8 *update UNNEEDED)