From d013450f1456d946ecacf1b45e1af499b88afbc7 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Fri, 24 Sep 2021 12:28:27 +0200
Subject: [PATCH] historian: Instead of exiting just skip large messages

This check was introduced to protect against the length being
corrupted but we now get some node_announcements with huge
feature-bits, triggering this rule. Let's skip them instead.
---
 historian/historian.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/historian/historian.py b/historian/historian.py
index bd3024c..4aa49ff 100755
--- a/historian/historian.py
+++ b/historian/historian.py
@@ -12,6 +12,10 @@ import gossipd
 import struct
 import time
 
+# Any message that is larger than this threshold will not be processed
+# as it bloats the database.
+MAX_MSG_SIZE = 1024
+
 plugin = Plugin()
 
 
@@ -44,6 +48,7 @@ class FileTailer():
             self.version, = struct.unpack("!B", f.read(1))
             f.seek(self.pos)
             while True:
+                skip = False
                 diff = 8
                 hdr = f.read(8)
                 if len(hdr) < 8:
@@ -58,10 +63,6 @@ class FileTailer():
                 # important = (length & 0x40000000 != 0)
                 length = length & (~0x80000000) & (~0x40000000)
 
-                if length > 1000:
-                    raise ValueError(
-                        f"Unreasonably large message: {length} bytes long"
-                    )
                 msg = f.read(length)
 
                 # Incomplete write, will try again
@@ -85,7 +86,14 @@ class FileTailer():
                     f.seek(self.pos)
                     continue
 
+                if length > MAX_MSG_SIZE:
+                    logging.warn(
+                        f"Unreasonably large message type {typ} at position {self.pos} ({length} bytes), skipping"
+                    )
+                    continue
+
                 ev_count += 1
+
                 yield msg
         logging.debug(
             f"Reached end of {self.filename} at {self.pos} after {ev_count} "
@@ -200,4 +208,5 @@ plugin.add_option(
     "SQL DSN defining where the gossip data should be stored."
 )
 
-plugin.run()
+if __name__ == "__main__":
+    plugin.run()