diff --git a/common/gossip_store.c b/common/gossip_store.c index 4fdcde1e4..2a0ade5ab 100644 --- a/common/gossip_store.c +++ b/common/gossip_store.c @@ -53,6 +53,7 @@ u8 *gossip_store_next(const tal_t *ctx, int *gossip_store_fd, u32 timestamp_min, u32 timestamp_max, bool push_only, + bool with_spam, size_t *off, size_t *end) { u8 *msg = NULL; @@ -60,7 +61,7 @@ u8 *gossip_store_next(const tal_t *ctx, while (!msg) { struct gossip_hdr hdr; u32 msglen, checksum, timestamp; - bool push; + bool push, ratelimited; int type, r; r = pread(*gossip_store_fd, &hdr, sizeof(hdr), *off); @@ -69,6 +70,7 @@ u8 *gossip_store_next(const tal_t *ctx, msglen = be32_to_cpu(hdr.len); push = (msglen & GOSSIP_STORE_LEN_PUSH_BIT); + ratelimited = (msglen & GOSSIP_STORE_LEN_RATELIMIT_BIT); msglen &= GOSSIP_STORE_LEN_MASK; /* Skip any deleted entries. */ @@ -115,6 +117,8 @@ u8 *gossip_store_next(const tal_t *ctx, msg = tal_free(msg); } else if (!push && push_only) { msg = tal_free(msg); + } else if (!with_spam && ratelimited) { + msg = tal_free(msg); } } diff --git a/common/gossip_store.h b/common/gossip_store.h index 8fa6ee145..136186e8e 100644 --- a/common/gossip_store.h +++ b/common/gossip_store.h @@ -11,7 +11,7 @@ struct gossip_rcvd_filter; /** * gossip_store -- On-disk storage related information */ -#define GOSSIP_STORE_VERSION 9 +#define GOSSIP_STORE_VERSION 10 /** * Bit of length we use to mark a deleted record. @@ -23,9 +23,15 @@ struct gossip_rcvd_filter; */ #define GOSSIP_STORE_LEN_PUSH_BIT 0x40000000U +/** + * Bit of length used to define a rate-limited record (do not rebroadcast) + */ + #define GOSSIP_STORE_LEN_RATELIMIT_BIT 0x20000000U + /* Mask for extracting just the length part of len field */ #define GOSSIP_STORE_LEN_MASK \ - (~(GOSSIP_STORE_LEN_PUSH_BIT | GOSSIP_STORE_LEN_DELETED_BIT)) + (~(GOSSIP_STORE_LEN_PUSH_BIT | GOSSIP_STORE_LEN_DELETED_BIT | \ + GOSSIP_STORE_LEN_RATELIMIT_BIT)) /** * gossip_hdr -- On-disk format header. @@ -47,6 +53,7 @@ u8 *gossip_store_next(const tal_t *ctx, int *gossip_store_fd, u32 timestamp_min, u32 timestamp_max, bool push_only, + bool with_spam, size_t *off, size_t *end); /** diff --git a/common/test/run-gossmap_local.c b/common/test/run-gossmap_local.c index 6f1b4b724..84c97d5ab 100644 --- a/common/test/run-gossmap_local.c +++ b/common/test/run-gossmap_local.c @@ -31,7 +31,7 @@ void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id U * setup_gossip_store_test via od -v -Anone -tx1 < /tmp/ltests-kaf30pn0/test_gossip_store_compact_noappend_1/lightning-2/regtest/gossip_store */ static u8 canned_map[] = { - 0x09, 0x80, 0x00, 0x01, 0xbc, 0x09, 0x8b, 0x67, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00 + 0x0a, 0x80, 0x00, 0x01, 0xbc, 0x09, 0x8b, 0x67, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x10, 0x08, 0x00 , 0x00, 0x00, 0x00, 0x00, 0x0f, 0x42, 0x40, 0x01, 0xb0, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 , 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 , 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 diff --git a/connectd/multiplex.c b/connectd/multiplex.c index e99c78e6e..d3d189a4a 100644 --- a/connectd/multiplex.c +++ b/connectd/multiplex.c @@ -375,6 +375,7 @@ static u8 *maybe_from_gossip_store(const tal_t *ctx, struct peer *peer) return gossip_store_next(ctx, &peer->daemon->gossip_store_fd, 0, 0xFFFFFFFF, true, + false, &peer->gs.off, &peer->daemon->gossip_store_end); } @@ -391,6 +392,7 @@ again: peer->gs.timestamp_min, peer->gs.timestamp_max, false, + false, &peer->gs.off, &peer->daemon->gossip_store_end); /* Don't send back gossip they sent to us! */ diff --git a/contrib/pyln-client/pyln/client/gossmap.py b/contrib/pyln-client/pyln/client/gossmap.py index 71379d438..ed0c76500 100755 --- a/contrib/pyln-client/pyln/client/gossmap.py +++ b/contrib/pyln-client/pyln/client/gossmap.py @@ -9,7 +9,7 @@ import io import struct # These duplicate constants in lightning/common/gossip_store.h -GOSSIP_STORE_VERSION = 9 +GOSSIP_STORE_VERSIONS = [0x09, 0x0a] GOSSIP_STORE_LEN_DELETED_BIT = 0x80000000 GOSSIP_STORE_LEN_PUSH_BIT = 0x40000000 GOSSIP_STORE_LEN_MASK = (~(GOSSIP_STORE_LEN_PUSH_BIT @@ -171,9 +171,9 @@ class Gossmap(object): self.nodes: Dict[GossmapNodeId, GossmapNode] = {} self.channels: Dict[ShortChannelId, GossmapChannel] = {} self._last_scid: Optional[str] = None - version = self.store_file.read(1) - if version[0] != GOSSIP_STORE_VERSION: - raise ValueError("Invalid gossip store version {}".format(int(version))) + version = self.store_file.read(1)[0] + if version not in GOSSIP_STORE_VERSIONS: + raise ValueError("Invalid gossip store version {}".format(version)) self.bytes_read = 1 self.refresh() diff --git a/contrib/pyln-client/tests/data/gossip_store-part1.xz b/contrib/pyln-client/tests/data/gossip_store-part1.xz index 73bd769f9..f63971a86 100644 Binary files a/contrib/pyln-client/tests/data/gossip_store-part1.xz and b/contrib/pyln-client/tests/data/gossip_store-part1.xz differ diff --git a/contrib/pyln-client/tests/data/gossip_store.simple.xz b/contrib/pyln-client/tests/data/gossip_store.simple.xz index a8be99deb..23981fad9 100644 Binary files a/contrib/pyln-client/tests/data/gossip_store.simple.xz and b/contrib/pyln-client/tests/data/gossip_store.simple.xz differ diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index 1032d415c..e431ed665 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -118,7 +118,7 @@ static u8 *mk_private_channelmsg(const tal_t *ctx, /* The upgrade from version 7 is trivial */ static bool can_upgrade(u8 oldversion) { - return oldversion == 7 || oldversion == 8; + return oldversion == 7 || oldversion == 8 || oldversion == 9; } static bool upgrade_field(u8 oldversion, diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 1c94e185e..8a5f2994b 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -1127,7 +1127,7 @@ def test_gossip_store_load(node_factory): """Make sure we can read canned gossip store""" l1 = node_factory.get_node(start=False) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("09" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1159,7 +1159,7 @@ def test_gossip_store_load_announce_before_update(node_factory): """Make sure we can read canned gossip store with node_announce before update. This happens when a channel_update gets replaced, leaving node_announce before it""" l1 = node_factory.get_node(start=False) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("09" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1202,7 +1202,7 @@ def test_gossip_store_load_amount_truncated(node_factory): """Make sure we can read canned gossip store with truncated amount""" l1 = node_factory.get_node(start=False, allow_broken_log=True) with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("09" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1669,7 +1669,7 @@ def test_gossip_store_load_no_channel_update(node_factory): # A channel announcement with no channel_update. with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("09" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1696,7 +1696,7 @@ def test_gossip_store_load_no_channel_update(node_factory): l1.rpc.call('dev-compact-gossip-store') with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), "rb") as f: - assert bytearray(f.read()) == bytearray.fromhex("09") + assert bytearray(f.read()) == bytearray.fromhex("0a") @pytest.mark.developer("gossip without DEVELOPER=1 is slow")