aljaz/lightning
1
0
Fork 0
mirror of https://github.com/aljazceru/lightning.git synced 2025-12-19 23:24:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

gossipd: don't ever use zlib compression on gossip.

Browse Source 
This was eliminated this morning in the latest spec.  We still accept them,
we just don't produce them any more.

Changelog-Removed: Protocol: We no longer create gossip messages which use zlib encoding (we still understand them, for now!)
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
This commit is contained in:
Rusty Russell
2022-05-18 10:20:16 +09:30
parent 4cd6210c19
commit 5735f71e3c
3 changed files with 30 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
gossipd/queries.c
View File

@@ -52,52 +52,6 @@ static void encoding_add_query_flag(u8 **encoded, bigsize_t flag)
towire_bigsize(encoded, flag); towire_bigsize(encoded, flag);
} }
/* Greg Maxwell asked me privately about using zlib for communicating a set,
* and suggested that we'd be better off using Golomb-Rice coding a-la BIP
* 158. However, naively using Rice encoding isn't a win: we have to get
* more complex and use separate streams. The upside is that it's between
* 2 and 5 times smaller (assuming optimal Rice encoding + gzip). We can add
* that later. */
static u8 *zencode(const tal_t *ctx, const u8 *scids, size_t len)
{
u8 *z;
int err;
unsigned long compressed_len = len;
#ifdef ZLIB_EVEN_IF_EXPANDS
/* Needed for test vectors */
compressed_len = 128 * 1024;
#endif
/* Prefer to fail if zlib makes it larger */
z = tal_arr(ctx, u8, compressed_len);
err = compress2(z, &compressed_len, scids, len, Z_DEFAULT_COMPRESSION);
if (err == Z_OK) {
tal_resize(&z, compressed_len);
return z;
}
return NULL;
}
/* Try compressing *encoded: fails if result would be longer.
* @off is offset to place result in *encoded.
*/
static bool encoding_end_zlib(u8 **encoded, size_t off)
{
u8 *z;
size_t len = tal_count(*encoded);
z = zencode(tmpctx, *encoded, len);
if (!z)
return false;
/* Successful: copy over and trim */
tal_resize(encoded, off + tal_count(z));
memcpy(*encoded + off, z, tal_count(z));
tal_free(z);
return true;
}
static void encoding_end_no_compress(u8 **encoded, size_t off) static void encoding_end_no_compress(u8 **encoded, size_t off)
{ {
size_t len = tal_count(*encoded); size_t len = tal_count(*encoded);
@@ -110,12 +64,8 @@ static void encoding_end_no_compress(u8 **encoded, size_t off)
* Prepends encoding type to @encoding. */ * Prepends encoding type to @encoding. */
static bool encoding_end_prepend_type(u8 **encoded, size_t max_bytes) static bool encoding_end_prepend_type(u8 **encoded, size_t max_bytes)
{ {
if (encoding_end_zlib(encoded, 1))
**encoded = ARR_ZLIB;
else {
encoding_end_no_compress(encoded, 1); encoding_end_no_compress(encoded, 1);
**encoded = ARR_UNCOMPRESSED; **encoded = ARR_UNCOMPRESSED;
}
#if DEVELOPER #if DEVELOPER
if (tal_count(*encoded) > dev_max_encoding_bytes) if (tal_count(*encoded) > dev_max_encoding_bytes)
@@ -127,12 +77,8 @@ static bool encoding_end_prepend_type(u8 **encoded, size_t max_bytes)
/* Try compressing, leaving type external */ /* Try compressing, leaving type external */
static bool encoding_end_external_type(u8 **encoded, u8 *type, size_t max_bytes) static bool encoding_end_external_type(u8 **encoded, u8 *type, size_t max_bytes)
{ {
if (encoding_end_zlib(encoded, 0))
*type = ARR_ZLIB;
else {
encoding_end_no_compress(encoded, 0); encoding_end_no_compress(encoded, 0);
*type = ARR_UNCOMPRESSED; *type = ARR_UNCOMPRESSED;
}
return tal_count(*encoded) <= max_bytes; return tal_count(*encoded) <= max_bytes;
} }

161
gossipd/test/run-extended-info.c
View File

@@ -1,7 +1,5 @@
#include "config.h" #include "config.h"
#define ZLIB_EVEN_IF_EXPANDS 1
#include "../queries.c" #include "../queries.c"
#include <ccan/str/hex/hex.h> #include <ccan/str/hex/hex.h>
#include <common/blinding.h> #include <common/blinding.h>
@@ -112,7 +110,8 @@ void status_fmt(enum log_level level UNNEEDED,
{ {
} }
static const char *test_vectors[] = { /* These we can reproduce */
static const char *test_vectors_nozlib[] = {
"{\n" "{\n"
" \"msg\" : {\n" " \"msg\" : {\n"
" \"type\" : \"QueryChannelRange\",\n" " \"type\" : \"QueryChannelRange\",\n"
@@ -137,34 +136,6 @@ static const char *test_vectors[] = {
" \"msg\" : {\n" " \"msg\" : {\n"
" \"type\" : \"ReplyChannelRange\",\n" " \"type\" : \"ReplyChannelRange\",\n"
" \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n"
" \"firstBlockNum\" : 756230,\n"
" \"numberOfBlocks\" : 1500,\n"
" \"complete\" : 1,\n"
" \"shortChannelIds\" : {\n"
" \"encoding\" : \"UNCOMPRESSED\",\n"
" \"array\" : [ \"0x0x142\", \"0x0x15465\", \"0x69x42692\" ]\n"
" }\n"
" },\n"
" \"hex\" : \"01080f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206000b8a06000005dc01001900000000000000008e0000000000003c69000000000045a6c4\"\n"
"}\n",
"{\n"
" \"msg\" : {\n"
" \"type\" : \"ReplyChannelRange\",\n"
" \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n"
" \"firstBlockNum\" : 1600,\n"
" \"numberOfBlocks\" : 110,\n"
" \"complete\" : 1,\n"
" \"shortChannelIds\" : {\n"
" \"encoding\" : \"COMPRESSED_ZLIB\",\n"
" \"array\" : [ \"0x0x142\", \"0x0x15465\", \"0x4x3318\" ]\n"
" }\n"
" },\n"
" \"hex\" : \"01080f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206000006400000006e01001601789c636000833e08659309a65878be010010a9023a\"\n"
"}\n",
"{\n"
" \"msg\" : {\n"
" \"type\" : \"ReplyChannelRange\",\n"
" \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n"
" \"firstBlockNum\" : 122334,\n" " \"firstBlockNum\" : 122334,\n"
" \"numberOfBlocks\" : 1500,\n" " \"numberOfBlocks\" : 1500,\n"
" \"complete\" : 1,\n" " \"complete\" : 1,\n"
@@ -202,45 +173,6 @@ static const char *test_vectors[] = {
"}\n", "}\n",
"{\n" "{\n"
" \"msg\" : {\n" " \"msg\" : {\n"
" \"type\" : \"ReplyChannelRange\",\n"
" \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n"
" \"firstBlockNum\" : 122334,\n"
" \"numberOfBlocks\" : 1500,\n"
" \"complete\" : 1,\n"
" \"shortChannelIds\" : {\n"
" \"encoding\" : \"COMPRESSED_ZLIB\",\n"
" \"array\" : [ \"0x0x12355\", \"0x7x30934\", \"0x70x57793\" ]\n"
" },\n"
" \"timestamps\" : {\n"
" \"encoding\" : \"COMPRESSED_ZLIB\",\n"
" \"timestamps\" : [ {\n"
" \"timestamp1\" : 164545,\n"
" \"timestamp2\" : 948165\n"
" }, {\n"
" \"timestamp1\" : 489645,\n"
" \"timestamp2\" : 4786864\n"
" }, {\n"
" \"timestamp1\" : 46456,\n"
" \"timestamp2\" : 9788415\n"
" } ]\n"
" },\n"
" \"checksums\" : {\n"
" \"checksums\" : [ {\n"
" \"checksum1\" : 1111,\n"
" \"checksum2\" : 2222\n"
" }, {\n"
" \"checksum1\" : 3333,\n"
" \"checksum2\" : 4444\n"
" }, {\n"
" \"checksum1\" : 5555,\n"
" \"checksum2\" : 6666\n"
" } ]\n"
" }\n"
" },\n"
" \"hex\" : \"01080f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e22060001ddde000005dc01001801789c63600001036730c55e710d4cbb3d3c080017c303b1012201789c63606a3ac8c0577e9481bd622d8327d7060686ad150c53a3ff0300554707db031800000457000008ae00000d050000115c000015b300001a0a\"\n"
"}\n",
"{\n"
" \"msg\" : {\n"
" \"type\" : \"QueryShortChannelIds\",\n" " \"type\" : \"QueryShortChannelIds\",\n"
" \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n" " \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n"
" \"shortChannelIds\" : {\n" " \"shortChannelIds\" : {\n"
@@ -251,48 +183,6 @@ static const char *test_vectors[] = {
" },\n" " },\n"
" \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206001900000000000000008e0000000000003c69000000000045a6c4\"\n" " \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206001900000000000000008e0000000000003c69000000000045a6c4\"\n"
"}\n", "}\n",
"{\n"
" \"msg\" : {\n"
" \"type\" : \"QueryShortChannelIds\",\n"
" \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n"
" \"shortChannelIds\" : {\n"
" \"encoding\" : \"COMPRESSED_ZLIB\",\n"
" \"array\" : [ \"0x0x4564\", \"0x2x47550\", \"0x69x42692\" ]\n"
" },\n"
" \"extensions\" : [ ]\n"
" },\n"
" \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206001801789c63600001c12b608a69e73e30edbaec0800203b040e\"\n"
"}\n",
"{\n"
" \"msg\" : {\n"
" \"type\" : \"QueryShortChannelIds\",\n"
" \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n"
" \"shortChannelIds\" : {\n"
" \"encoding\" : \"UNCOMPRESSED\",\n"
" \"array\" : [ \"0x0x12232\", \"0x0x15556\", \"0x69x42692\" ]\n"
" },\n"
" \"extensions\" : [ {\n"
" \"encoding\" : \"COMPRESSED_ZLIB\",\n"
" \"array\" : [ 1, 2, 4 ]\n"
" } ]\n"
" },\n"
" \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e22060019000000000000002fc80000000000003cc4000000000045a6c4010c01789c6364620100000e0008\"\n"
"}\n",
"{\n"
" \"msg\" : {\n"
" \"type\" : \"QueryShortChannelIds\",\n"
" \"chainHash\" : \"0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206\",\n"
" \"shortChannelIds\" : {\n"
" \"encoding\" : \"COMPRESSED_ZLIB\",\n"
" \"array\" : [ \"0x0x14200\", \"0x0x46645\", \"0x69x42692\" ]\n"
" },\n"
" \"extensions\" : [ {\n"
" \"encoding\" : \"COMPRESSED_ZLIB\",\n"
" \"array\" : [ 1, 2, 4 ]\n"
" } ]\n"
" },\n"
" \"hex\" : \"01050f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206001801789c63600001f30a30c5b0cd144cb92e3b020017c6034a010c01789c6364620100000e0008\"\n"
"}\n",
}; };
static void get_chainhash(const char *test_vector, static void get_chainhash(const char *test_vector,
@@ -323,14 +213,9 @@ static u8 *get_scid_array(const tal_t *ctx,
assert(json_to_short_channel_id(test_vector, t, &scid)); assert(json_to_short_channel_id(test_vector, t, &scid));
encoding_add_short_channel_id(&encoded, &scid); encoding_add_short_channel_id(&encoded, &scid);
} }
if (json_tok_streq(test_vector, encoding, "UNCOMPRESSED")) { assert(json_tok_streq(test_vector, encoding, "UNCOMPRESSED"));
encoding_end_no_compress(&encoded, 1); encoding_end_no_compress(&encoded, 1);
encoded[0] = ARR_UNCOMPRESSED; encoded[0] = ARR_UNCOMPRESSED;
} else {
assert(json_tok_streq(test_vector, encoding, "COMPRESSED_ZLIB"));
assert(encoding_end_zlib(&encoded, 1));
encoded[0] = ARR_ZLIB;
}
return encoded; return encoded;
} }
@@ -404,17 +289,10 @@ static u8 *test_reply_channel_range(const char *test_vector, const jsmntok_t *ob
&ts.timestamp_node_id_2)); &ts.timestamp_node_id_2));
encoding_add_timestamps(&tlvs->timestamps_tlv->encoded_timestamps, &ts); encoding_add_timestamps(&tlvs->timestamps_tlv->encoded_timestamps, &ts);
} }
if (json_tok_streq(test_vector, encodingtok, "UNCOMPRESSED")) { assert(json_tok_streq(test_vector, encodingtok, "UNCOMPRESSED"));
encoding_end_no_compress(&tlvs->timestamps_tlv->encoded_timestamps, encoding_end_no_compress(&tlvs->timestamps_tlv->encoded_timestamps,
0); 0);
tlvs->timestamps_tlv->encoding_type = ARR_UNCOMPRESSED; tlvs->timestamps_tlv->encoding_type = ARR_UNCOMPRESSED;
} else {
assert(json_tok_streq(test_vector, encodingtok,
"COMPRESSED_ZLIB"));
assert(encoding_end_zlib(&tlvs->timestamps_tlv->encoded_timestamps,
0));
tlvs->timestamps_tlv->encoding_type = ARR_ZLIB;
}
} }
opt = json_get_member(test_vector, obj, "checksums"); opt = json_get_member(test_vector, obj, "checksums");
@@ -464,14 +342,9 @@ get_query_flags_array(const tal_t *ctx,
assert(json_to_u64(test_vector, t, &f)); assert(json_to_u64(test_vector, t, &f));
encoding_add_query_flag(&tlv->encoded_query_flags, f); encoding_add_query_flag(&tlv->encoded_query_flags, f);
} }
if (json_tok_streq(test_vector, encoding, "UNCOMPRESSED")) { assert(json_tok_streq(test_vector, encoding, "UNCOMPRESSED"));
encoding_end_no_compress(&tlv->encoded_query_flags, 0); encoding_end_no_compress(&tlv->encoded_query_flags, 0);
tlv->encoding_type = ARR_UNCOMPRESSED; tlv->encoding_type = ARR_UNCOMPRESSED;
} else {
assert(json_tok_streq(test_vector, encoding, "COMPRESSED_ZLIB"));
assert(encoding_end_zlib(&tlv->encoded_query_flags, 0));
tlv->encoding_type = ARR_ZLIB;
}
return tlv; return tlv;
} }
@@ -509,7 +382,7 @@ int main(int argc, char *argv[])
common_setup(argv[0]); common_setup(argv[0]);
for (size_t i = 0; i < ARRAY_SIZE(test_vectors); i++) { for (size_t i = 0; i < ARRAY_SIZE(test_vectors_nozlib); i++) {
jsmn_parser parser; jsmn_parser parser;
u8 *m; u8 *m;
const char *hex_m; const char *hex_m;
@@ -517,22 +390,22 @@ int main(int argc, char *argv[])
jsmn_init(&parser); jsmn_init(&parser);
assert(jsmn_parse(&parser, assert(jsmn_parse(&parser,
test_vectors[i], strlen(test_vectors[i]), test_vectors_nozlib[i], strlen(test_vectors_nozlib[i]),
toks, tal_count(toks)) > 0); toks, tal_count(toks)) > 0);
msg = json_get_member(test_vectors[i], toks, "msg"); msg = json_get_member(test_vectors_nozlib[i], toks, "msg");
hex = json_get_member(test_vectors[i], toks, "hex"); hex = json_get_member(test_vectors_nozlib[i], toks, "hex");
type = json_get_member(test_vectors[i], msg, "type"); type = json_get_member(test_vectors_nozlib[i], msg, "type");
if (json_tok_streq(test_vectors[i], type, "QueryChannelRange")) if (json_tok_streq(test_vectors_nozlib[i], type, "QueryChannelRange"))
m = test_query_channel_range(test_vectors[i], msg); m = test_query_channel_range(test_vectors_nozlib[i], msg);
else if (json_tok_streq(test_vectors[i], type, "ReplyChannelRange")) else if (json_tok_streq(test_vectors_nozlib[i], type, "ReplyChannelRange"))
m = test_reply_channel_range(test_vectors[i], msg); m = test_reply_channel_range(test_vectors_nozlib[i], msg);
else if (json_tok_streq(test_vectors[i], type, "QueryShortChannelIds")) else if (json_tok_streq(test_vectors_nozlib[i], type, "QueryShortChannelIds"))
m = test_query_short_channel_ids(test_vectors[i], msg); m = test_query_short_channel_ids(test_vectors_nozlib[i], msg);
else else
abort(); abort();
hex_m = tal_hex(m, m); hex_m = tal_hex(m, m);
assert(json_tok_streq(test_vectors[i], hex, hex_m)); assert(json_tok_streq(test_vectors_nozlib[i], hex, hex_m));
tal_free(m); tal_free(m);
} }
tal_free(toks); tal_free(toks);

4
tests/test_gossip.py
View File

@@ -872,7 +872,7 @@ def test_gossip_query_channel_range(node_factory, bitcoind, chainparams):
filters=['0109', '0012']) filters=['0109', '0012'])
assert len(msgs) == 2 assert len(msgs) == 2
# This should actually be large enough for zlib to kick in! # This used to be large enough for zlib to kick in, but no longer!
scid34, _ = l3.fundchannel(l4, 10**5) scid34, _ = l3.fundchannel(l4, 10**5)
mine_funding_to_announce(bitcoind, [l1, l2, l3, l4]) mine_funding_to_announce(bitcoind, [l1, l2, l3, l4])
l2.daemon.wait_for_log('Received node_announcement for node ' + l4.info['id']) l2.daemon.wait_for_log('Received node_announcement for node ' + l4.info['id'])
@@ -886,7 +886,7 @@ def test_gossip_query_channel_range(node_factory, bitcoind, chainparams):
genesis_blockhash, genesis_blockhash,
0, 65535, 0, 65535,
filters=['0109', '0012']) filters=['0109', '0012'])
encoded = subprocess.run(['devtools/mkencoded', '--scids', '01', scid12, scid23, scid34], encoded = subprocess.run(['devtools/mkencoded', '--scids', '00', scid12, scid23, scid34],
check=True, check=True,
timeout=TIMEOUT, timeout=TIMEOUT,
stdout=subprocess.PIPE).stdout.strip().decode() stdout=subprocess.PIPE).stdout.strip().decode()