From daa5269ea259a855dac282c17c83e0eff09821b5 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 14 Sep 2022 13:20:31 +0930 Subject: [PATCH] gossipd: bump gossip_store to indicate all channel_update have htlc_max. And in the next patch, gossipd will no longer put new ones in. Signed-off-by: Rusty Russell --- common/gossip_store.h | 2 +- gossipd/gossip_store.c | 30 ++++++++++++++++++----- tests/test_gossip.py | 54 ++++++++++++++++++++++++++++++++---------- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/common/gossip_store.h b/common/gossip_store.h index 514f32096..e74f7cba4 100644 --- a/common/gossip_store.h +++ b/common/gossip_store.h @@ -15,7 +15,7 @@ struct gossip_rcvd_filter; /* First byte of file is the version. * * Top three bits mean incompatible change. - * As of this writing, major == 0, minor == 10. + * As of this writing, major == 0, minor == 11. */ #define GOSSIP_STORE_MAJOR_VERSION_MASK 0xE0 #define GOSSIP_STORE_MINOR_VERSION_MASK 0x1F diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index 64bc313bb..5a3e9cd27 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -17,8 +17,8 @@ #include #define GOSSIP_STORE_TEMP_FILENAME "gossip_store.tmp" -/* We write it as major version 0, minor version 10 */ -#define GOSSIP_STORE_VER ((0 << 5) | 10) +/* We write it as major version 0, minor version 11 */ +#define GOSSIP_STORE_VER ((0 << 5) | 11) struct gossip_store { /* This is false when we're loading */ @@ -103,10 +103,12 @@ static bool append_msg(int fd, const u8 *msg, u32 timestamp, return true; } -/* The upgrade from version 9 is a noop: we added the spam flag. */ +/* v9 added the GOSSIP_STORE_LEN_RATELIMIT_BIT. + * v10 removed any remaining non-htlc-max channel_update. + */ static bool can_upgrade(u8 oldversion) { - return oldversion == 9; + return oldversion == 9 || oldversion == 10; } static bool upgrade_field(u8 oldversion, @@ -114,6 +116,15 @@ static bool upgrade_field(u8 oldversion, u8 **msg) { assert(can_upgrade(oldversion)); + + if (oldversion == 10) { + /* Remove old channel_update with no htlc_maximum_msat */ + if (fromwire_peektype(*msg) == WIRE_CHANNEL_UPDATE + && tal_bytelen(*msg) == 130) { + *msg = tal_free(*msg); + } + } + return true; } @@ -182,7 +193,7 @@ static u32 gossip_store_compact_offline(struct routing_state *rstate) != crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)) { status_broken("gossip_store_compact_offline: checksum verification failed? %08x should be %08x", be32_to_cpu(hdr.crc), - crc32c(be32_to_cpu(hdr.timestamp), msg + sizeof(hdr), msglen)); + crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)); tal_free(msg); goto close_and_delete; } @@ -193,6 +204,12 @@ static u32 gossip_store_compact_offline(struct routing_state *rstate) goto close_and_delete; } + /* It can tell us to delete record entirely. */ + if (msg == NULL) { + deleted++; + continue; + } + /* Recalc msglen and header */ msglen = tal_bytelen(msg); hdr.len = cpu_to_be32(msglen); @@ -723,7 +740,8 @@ u32 gossip_store_load(struct routing_state *rstate, struct gossip_store *gs) } if (checksum != crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)) { - bad = tal_fmt(tmpctx, "Checksum verification failed: should be %08x", crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)); + bad = tal_fmt(tmpctx, "Checksum verification failed: %08x should be %08x", + checksum, crc32c(be32_to_cpu(hdr.timestamp), msg, msglen)); goto badmsg; } diff --git a/tests/test_gossip.py b/tests/test_gossip.py index 1f0d35b30..a4bacc9ff 100644 --- a/tests/test_gossip.py +++ b/tests/test_gossip.py @@ -1141,11 +1141,11 @@ def test_gossip_store_load(node_factory): "00000000" # timestamp "1005" # WIRE_GOSSIP_STORE_CHANNEL_AMOUNT "0000000001000000" - "00000082" # len - "fd421aeb" # csum + "0000008a" # len + "0c6aca0e" # csum "5b8d9b44" # timestamp "0102" # WIRE_CHANNEL_UPDATE - "1ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440000009000000000000003e8000003e800000001" + "1ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440100009000000000000003e8000003e8000000010000000000FFFFFF" "00000095" # len "f036515e" # csum "5aab817c" # timestamp @@ -1154,10 +1154,36 @@ def test_gossip_store_load(node_factory): l1.start() # May preceed the Started msg waited for in 'start'. - wait_for(lambda: l1.daemon.is_in_log(r'gossip_store: Read 1/1/1/0 cannounce/cupdate/nannounce/cdelete from store \(0 deleted\) in 770 bytes')) + wait_for(lambda: l1.daemon.is_in_log(r'gossip_store: Read 1/1/1/0 cannounce/cupdate/nannounce/cdelete from store \(0 deleted\) in 778 bytes')) assert not l1.daemon.is_in_log('gossip_store.*truncating') +def test_gossip_store_v10_upgrade(node_factory): + """We remove a channel_update without an htlc_maximum_msat""" + l1 = node_factory.get_node(start=False) + with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: + f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION + "000001b0" # len + "fea676e8" # csum + "5b8d9b44" # timestamp + "0100" # WIRE_CHANNEL_ANNOUNCEMENT + "bb8d7b6998cca3c2b3ce12a6bd73a8872c808bb48de2a30c5ad9cdf835905d1e27505755087e675fb517bbac6beb227629b694ea68f49d357458327138978ebfd7adfde1c69d0d2f497154256f6d5567a5cf2317c589e0046c0cc2b3e986cf9b6d3b44742bd57bce32d72cd1180a7f657795976130b20508b239976d3d4cdc4d0d6e6fbb9ab6471f664a662972e406f519eab8bce87a8c0365646df5acbc04c91540b4c7c518cec680a4a6af14dae1aca0fd5525220f7f0e96fcd2adef3c803ac9427fe71034b55a50536638820ef21903d09ccddd38396675b598587fa886ca711415c813fc6d69f46552b9a0a539c18f265debd0e2e286980a118ba349c216000043497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b50001021bf3de4e84e3d52f9a3e36fbdcd2c4e8dbf203b9ce4fc07c2f03be6c21d0c67503f113414ebdc6c1fb0f33c99cd5a1d09dd79e7fdf2468cf1fe1af6674361695d203801fd8ab98032f11cc9e4916dd940417082727077609d5c7f8cc6e9a3ad25dd102517164b97ab46cee3826160841a36c46a2b7b9c74da37bdc070ed41ba172033a" + "0000000a" # len + "99dc98b4" # csum + "00000000" # timestamp + "1005" # WIRE_GOSSIP_STORE_CHANNEL_AMOUNT + "0000000001000000" + "00000082" # len + "fd421aeb" # csum + "5b8d9b44" # timestamp + "0102" # WIRE_CHANNEL_UPDATE + "1ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440000009000000000000003e8000003e800000001")) + + l1.start() + # May preceed the Started msg waited for in 'start'. + wait_for(lambda: l1.daemon.is_in_log(r'gossip_store: Unupdated channel_announcement at 1.')) + + def test_gossip_store_load_announce_before_update(node_factory): """Make sure we can read canned gossip store with node_announce before update. This happens when a channel_update gets replaced, leaving node_announce before it""" l1 = node_factory.get_node(start=False) @@ -1173,25 +1199,27 @@ def test_gossip_store_load_announce_before_update(node_factory): "00000000" # timestamp "1005" # WIRE_GOSSIP_STORE_CHANNEL_AMOUNT "0000000001000000" - "80000082" # len (DELETED) - "fd421aeb" # csum + "8000008a" # len (DELETED) + "ca01ed56" # csum "5b8d9b44" # timestamp "0102" # WIRE_CHANNEL_UPDATE - "1ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440000009000000000000003e8000003e800000001" + # Note - msgflags set and htlc_max added by hand, so signature doesn't match (gossipd ignores) + "1ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440100009000000000000003e8000003e8000000010000000000FFFFFF" "00000095" # len "f036515e" # csum "5aab817c" # timestamp "0101" # WIRE_NODE_ANNOUNCEMENT "cf5d870bc7ecabcb7cd16898ef66891e5f0c6c5851bd85b670f03d325bc44d7544d367cd852e18ec03f7f4ff369b06860a3b12b07b29f36fb318ca11348bf8ec00005aab817c03f113414ebdc6c1fb0f33c99cd5a1d09dd79e7fdf2468cf1fe1af6674361695d23974b250757a7a6c6549544300000000000000000000000000000000000000000000000007010566933e2607" - "00000082" # len - "fd421aeb" # csum + "0000008a" # len + "0c6aca0e" # csum "5b8d9b44" # timestamp "0102" # WIRE_CHANNEL_UPDATE - "1ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440000009000000000000003e8000003e800000001")) + # Note - msgflags set and htlc_max added by hand, so signature doesn't match (gossipd ignores) + "1ea7c2eadf8a29eb8690511a519b5656e29aa0a853771c4e38e65c5abf43d907295a915e69e451f4c7a0c3dc13dd943cfbe3ae88c0b96667cd7d58955dbfedcf43497fd7f826957108f4a30fd9cec3aeba79972084e90ead01ea33090000000013a63c0000b500015b8d9b440100009000000000000003e8000003e8000000010000000000FFFFFF")) l1.start() # May preceed the Started msg waited for in 'start'. - wait_for(lambda: l1.daemon.is_in_log(r'gossip_store: Read 1/1/1/0 cannounce/cupdate/nannounce/cdelete from store \(0 deleted\) in 770 bytes')) + wait_for(lambda: l1.daemon.is_in_log(r'gossip_store: Read 1/1/1/0 cannounce/cupdate/nannounce/cdelete from store \(0 deleted\) in 778 bytes')) assert not l1.daemon.is_in_log('gossip_store.*truncating') # Extra sanity check if we can. @@ -1670,7 +1698,7 @@ def test_gossip_store_load_no_channel_update(node_factory): # A channel announcement with no channel_update. with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), 'wb') as f: - f.write(bytearray.fromhex("0a" # GOSSIP_STORE_VERSION + f.write(bytearray.fromhex("0b" # GOSSIP_STORE_VERSION "000001b0" # len "fea676e8" # csum "5b8d9b44" # timestamp @@ -1697,7 +1725,7 @@ def test_gossip_store_load_no_channel_update(node_factory): l1.rpc.call('dev-compact-gossip-store') with open(os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'gossip_store'), "rb") as f: - assert bytearray(f.read()) == bytearray.fromhex("0a") + assert bytearray(f.read()) == bytearray.fromhex("0b") @pytest.mark.developer("gossip without DEVELOPER=1 is slow")