pyln.testing: remove Throttler.

CI seems to block; Christian suggests the throttler may be to blame somehow?

Since trying to fix it made it worse, let's just remove it.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

contrib/pyln-testing/pyln/testing/fixtures.py

@@ -1,6 +1,6 @@
 from concurrent import futures
 from pyln.testing.db import SqliteDbProvider, PostgresDbProvider
-from pyln.testing.utils import NodeFactory, BitcoinD, ElementsD, env, DEVELOPER, LightningNode, TEST_DEBUG, Throttler
+from pyln.testing.utils import NodeFactory, BitcoinD, ElementsD, env, DEVELOPER, LightningNode, TEST_DEBUG
 from pyln.client import Millisatoshi
 from typing import Dict
 
@@ -206,11 +206,6 @@ def teardown_checks(request):
         raise ValueError(str(errors))
 
 
-@pytest.fixture
-def throttler(test_base_dir):
-    yield Throttler(test_base_dir)
-
-
 def _extra_validator(is_request: bool):
     """JSON Schema validator with additions for our specialized types"""
     def is_hex(checker, instance):
@@ -451,7 +446,7 @@ def jsonschemas():
 
 
 @pytest.fixture
-def node_factory(request, directory, test_name, bitcoind, executor, db_provider, teardown_checks, node_cls, throttler, jsonschemas):
+def node_factory(request, directory, test_name, bitcoind, executor, db_provider, teardown_checks, node_cls, jsonschemas):
     nf = NodeFactory(
         request,
         test_name,
@@ -460,7 +455,6 @@ def node_factory(request, directory, test_name, bitcoind, executor, db_provider,
         directory=directory,
         db_provider=db_provider,
         node_cls=node_cls,
-        throttler=throttler,
         jsonschemas=jsonschemas,
     )
 

contrib/pyln-testing/pyln/testing/utils.py

@@ -18,7 +18,6 @@ import logging
 import lzma
 import math
 import os
-import psutil  # type: ignore
 import random
 import re
 import shutil
@@ -1330,57 +1329,11 @@ def flock(directory: Path):
     fname.unlink()
 
 
-class Throttler(object):
-    """Throttles the creation of system-processes to avoid overload.
-
-    There is no reason to overload the system with too many processes
-    being spawned or run at the same time. It causes timeouts by
-    aggressively preempting processes and swapping if the memory limit is
-    reached. In order to reduce this loss of performance we provide a
-    `wait()` method which will serialize the creation of processes, but
-    also delay if the system load is too high.
-
-    Notice that technically we are throttling too late, i.e., we react
-    to an overload, but chances are pretty good that some other
-    already running process is about to terminate, and so the overload
-    is short-lived. We throttle when the process object is first
-    created, not when restarted, in order to avoid delaying running
-    tests, which could cause more timeouts.
-
-    """
-    def __init__(self, directory: str, target: float = 90):
-        """If specified we try to stick to a load of target (in percent).
-        """
-        self.target = target
-        self.current_load = self.target  # Start slow
-        psutil.cpu_percent()  # Prime the internal load metric
-        self.directory = directory
-
-    def wait(self):
-        start_time = time.time()
-        with flock(self.directory):
-            # We just got the lock, assume someone else just released it
-            self.current_load = 100
-            while self.load() >= self.target:
-                time.sleep(1)
-
-            self.current_load = 100  # Back off slightly to avoid triggering right away
-        print("Throttler delayed startup for {} seconds".format(time.time() - start_time))
-
-    def load(self):
-        """An exponential moving average of the load
-        """
-        decay = 0.5
-        load = psutil.cpu_percent()
-        self.current_load = decay * load + (1 - decay) * self.current_load
-        return self.current_load
-
-
 class NodeFactory(object):
     """A factory to setup and start `lightningd` daemons.
     """
     def __init__(self, request, testname, bitcoind, executor, directory,
-                 db_provider, node_cls, throttler, jsonschemas):
+                 db_provider, node_cls, jsonschemas):
         if request.node.get_closest_marker("slow_test") and SLOW_MACHINE:
             self.valgrind = False
         else:
@@ -1395,7 +1348,6 @@ class NodeFactory(object):
         self.lock = threading.Lock()
         self.db_provider = db_provider
         self.node_cls = node_cls
-        self.throttler = throttler
         self.jsonschemas = jsonschemas
 
     def split_options(self, opts):
@@ -1463,7 +1415,6 @@ class NodeFactory(object):
                  bkpr_dbfile=None, feerates=(15000, 11000, 7500, 3750),
                  start=True, wait_for_bitcoind_sync=True, may_fail=False,
                  expect_fail=False, cleandir=True, **kwargs):
-        self.throttler.wait()
         node_id = self.get_node_id() if not node_id else node_id
         port = reserve_unused_port()
 

tests/fixtures.py

@@ -1,5 +1,5 @@
 from utils import DEVELOPER, TEST_NETWORK, VALGRIND  # noqa: F401,F403
-from pyln.testing.fixtures import directory, test_base_dir, test_name, chainparams, node_factory, bitcoind, teardown_checks, throttler, db_provider, executor, setup_logging, jsonschemas  # noqa: F401,F403
+from pyln.testing.fixtures import directory, test_base_dir, test_name, chainparams, node_factory, bitcoind, teardown_checks, db_provider, executor, setup_logging, jsonschemas  # noqa: F401,F403
 from pyln.testing import utils
 from utils import COMPAT
 from pathlib import Path