pytest: Remove all trace of python's "flaky" module.

Over time, it has cost us more developer cycles than it has gained. It has hidden intermittant bugs, and allowed cruft to accumulate: when we eventually tried to figure out what was going wrong, the actual change which caused it was now stale and forgotten. This was a particular bane during the connectd rewrite, and I worked through some issues which had occurred before, but were not more likely. Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
2025-12-18 22:54:25 +01:00 · 2022-06-26 13:56:01 +09:30
parent 56dde2cb77
commit 575b94c1ef
16 changed files with 9 additions and 38 deletions
--- a/.github/scripts/build.sh
+++ b/.github/scripts/build.sh
@@ -33,7 +33,7 @@ cat config.vars

 cat << EOF > pytest.ini
 [pytest]
-addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42
 markers =
    slow_test: marks tests as slow (deselect with '-m "not slow_test"')
 EOF
--- a/.github/workflows/bsd.yml
+++ b/.github/workflows/bsd.yml
@@ -79,7 +79,7 @@ jobs:

          cat << EOF > pytest.ini
          [pytest]
-          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
          markers =
              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
          EOF
--- a/.github/workflows/macos.yaml
+++ b/.github/workflows/macos.yaml
@@ -59,7 +59,7 @@ jobs:

          cat << EOF > pytest.ini
          [pytest]
-          addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+          addopts=-p no:logging --color=yes --timeout=600 --timeout-method=thread --test-group-random-seed=42 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
          markers =
              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
          EOF
--- a/contrib/docker/Dockerfile.builder
+++ b/contrib/docker/Dockerfile.builder
@@ -56,7 +56,6 @@ RUN pip3 install --upgrade pip && \
 	Flask==1.0.2 \
 	cheroot==8.2.1 \
 	ephemeral-port-reserve==1.1.0 \
-	flaky==3.4.0 \
 	pytest-benchmark==3.1.1 \
        pytest-forked==0.2 \
 	pytest-timeout==1.3.3 \
--- a/contrib/docker/scripts/build.sh
+++ b/contrib/docker/scripts/build.sh
@@ -34,7 +34,7 @@ cat config.vars

 cat << EOF > pytest.ini
 [pytest]
-addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42
 markers =
    slow_test: marks tests as slow (deselect with '-m "not slow_test"')
 EOF
--- a/contrib/pyln-client/Makefile
+++ b/contrib/pyln-client/Makefile
@@ -46,7 +46,7 @@ test-release: check $(ARTEFACTS) pyproject.toml
 	# tests against it (make sure not to use any virtualenv that may have
 	# pyln-${PKG} already installed).
 	virtualenv testpypi --python=/usr/bin/python3 --download --always-copy --clear
-	testpypi/bin/python3 -m pip install -r requirements.txt flaky pytest-timeout
+	testpypi/bin/python3 -m pip install -r requirements.txt pytest-timeout
 	testpypi/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-${PKG}==${VERSION}
 	testpypi/bin/python3 -c "from pyln import ${PKG};assert(${PKG}.__version__ == '$(VERSION)')"
 	testpypi/bin/pytest tests
--- a/contrib/pyln-proto/Makefile
+++ b/contrib/pyln-proto/Makefile
@@ -46,7 +46,7 @@ test-release: check $(ARTEFACTS) pyproject.toml
 	# tests against it (make sure not to use any virtualenv that may have
 	# pyln-${PKG} already installed).
 	virtualenv testpypi --python=/usr/bin/python3 --download --always-copy --clear
-	testpypi/bin/python3 -m pip install -r requirements.txt flaky pytest-timeout
+	testpypi/bin/python3 -m pip install -r requirements.txt pytest-timeout
 	testpypi/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-${PKG}==${VERSION}
 	testpypi/bin/python3 -c "from pyln import ${PKG};assert(${PKG}.__version__ == '$(VERSION)')"
 	testpypi/bin/pytest tests
--- a/contrib/pyln-spec/Makefile
+++ b/contrib/pyln-spec/Makefile
@@ -58,7 +58,7 @@ test-release-bolt%: $(ARTEFACTS)
 	# pyln-proto already installed).
 	virtualenv testpypi-$* --python=/usr/bin/python3 --download --always-copy --clear
 	# Install the requirements from the prod repo, they are not being kept up to date on the test repo
-	testpypi-$*/bin/python3 -m pip install -r requirements.txt pytest flaky pytest-timeout
+	testpypi-$*/bin/python3 -m pip install -r requirements.txt pytest pytest-timeout
 	testpypi-$*/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-bolt$*
 	testpypi-$*/bin/python3 -c "from pyln.spec import bolt$* as bolt;assert(bolt.__version__ == '$(call version,$*)')"
 	testpypi-$*/bin/pytest bolt$*/tests
--- a/contrib/pyln-testing/Makefile
+++ b/contrib/pyln-testing/Makefile
@@ -46,7 +46,7 @@ test-release: check $(ARTEFACTS) pyproject.toml
 	# tests against it (make sure not to use any virtualenv that may have
 	# pyln-${PKG} already installed).
 	virtualenv testpypi --python=/usr/bin/python3 --download --always-copy --clear
-	testpypi/bin/python3 -m pip install -r requirements.txt flaky pytest-timeout
+	testpypi/bin/python3 -m pip install -r requirements.txt pytest-timeout
 	testpypi/bin/python3 -m pip install -I --index-url https://test.pypi.org/simple/ --no-deps pyln-${PKG}==${VERSION}
 	testpypi/bin/python3 -c "from pyln import ${PKG};assert(${PKG}.__version__ == '$(VERSION)')"
 	testpypi/bin/pytest tests
--- a/poetry.lock
+++ b/poetry.lock
@@ -166,14 +166,6 @@ mccabe = ">=0.6.0,<0.7.0"
 pycodestyle = ">=2.8.0,<2.9.0"
 pyflakes = ">=2.4.0,<2.5.0"

-[[package]]
-name = "flaky"
-version = "3.7.0"
-description = "Plugin for nose or pytest that automatically reruns flaky tests."
-category = "dev"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
-
 [[package]]
 name = "flask"
 version = "2.0.3"
@@ -982,10 +974,6 @@ flake8 = [
    {file = "flake8-4.0.1-py2.py3-none-any.whl", hash = "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d"},
    {file = "flake8-4.0.1.tar.gz", hash = "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d"},
 ]
-flaky = [
-    {file = "flaky-3.7.0-py2.py3-none-any.whl", hash = "sha256:d6eda73cab5ae7364504b7c44670f70abed9e75f77dd116352f662817592ec9c"},
-    {file = "flaky-3.7.0.tar.gz", hash = "sha256:3ad100780721a1911f57a165809b7ea265a7863305acb66708220820caf8aa0d"},
-]
 flask = [
    {file = "Flask-2.0.3-py3-none-any.whl", hash = "sha256:59da8a3170004800a2837844bfa84d49b022550616070f7cb1a659682b2e7c9f"},
    {file = "Flask-2.0.3.tar.gz", hash = "sha256:e1120c228ca2f553b470df4a5fa927ab66258467526069981b3eb0a91902687d"},
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,7 +24,6 @@ flake8 = "^4.0.1"
 mypy = "^0.931"
 pytest-custom-exit-code = "0.3.0"
 pyln-testing = { path = "./contrib/pyln-testing", develop = true }
-flaky = "^3.7.0"

 [build-system]
 requires = ["poetry-core>=1.0.0"]
--- a/tests/test_connection.py
+++ b/tests/test_connection.py
@@ -1,6 +1,5 @@
 from fixtures import *  # noqa: F401,F403
 from fixtures import TEST_NETWORK
-from flaky import flaky  # noqa: F401
 from ephemeral_port_reserve import reserve  # type: ignore
 from pyln.client import RpcError, Millisatoshi
 import pyln.proto.wire as wire
@@ -658,7 +657,6 @@ def test_reconnect_gossiping(node_factory):
    l2.daemon.wait_for_log('processing now old peer gone')


-@flaky
@pytest.mark.developer("needs dev-disconnect")
@pytest.mark.openchannel('v1')
@pytest.mark.openchannel('v2')
@@ -853,7 +851,6 @@ def test_reconnect_receiver_fulfill(node_factory):
    assert only_one(l2.rpc.listinvoices('testpayment2')['invoices'])['status'] == 'paid'


-@flaky
@pytest.mark.developer
@pytest.mark.openchannel('v1')
@pytest.mark.openchannel('v2')
@@ -886,7 +883,6 @@ def test_shutdown_reconnect(node_factory):
    assert l1.bitcoin.rpc.getmempoolinfo()['size'] == 1


-@flaky
@pytest.mark.developer
 def test_reconnect_remote_sends_no_sigs(node_factory):
    """We re-announce, even when remote node doesn't send its announcement_signatures on reconnect.
--- a/tests/test_misc.py
+++ b/tests/test_misc.py
@@ -2,7 +2,6 @@ from bitcoin.rpc import RawProxy
 from decimal import Decimal
 from fixtures import *  # noqa: F401,F403
 from fixtures import LightningNode, TEST_NETWORK
-from flaky import flaky  # noqa: F401
 from pyln.client import RpcError
 from threading import Event
 from pyln.testing.utils import (
@@ -1031,7 +1030,6 @@ def test_daemon_option(node_factory):
        assert 'No child process' not in f.read()


-@flaky
@pytest.mark.developer("needs DEVELOPER=1")
 def test_blockchaintrack(node_factory, bitcoind):
    """Check that we track the blockchain correctly across reorgs
@@ -1276,7 +1274,6 @@ def test_bitcoind_goes_backwards(node_factory, bitcoind):
    l1.daemon.wait_for_log('Adding block 111')


-@flaky
@pytest.mark.openchannel('v1')
@pytest.mark.openchannel('v2')
 def test_reserve_enforcement(node_factory, executor):
--- a/tests/test_pay.py
+++ b/tests/test_pay.py
@@ -1,6 +1,5 @@
 from fixtures import *  # noqa: F401,F403
 from fixtures import TEST_NETWORK
-from flaky import flaky  # noqa: F401
 from io import BytesIO
 from pyln.client import RpcError, Millisatoshi
 from pyln.proto.onion import TlvPayload
@@ -2765,13 +2764,11 @@ def test_pay_no_secret(node_factory, bitcoind):
        l1.rpc.pay(inv_nosecret)


-@flaky
 def test_shadow_routing(node_factory):
    """
    Test the value randomization through shadow routing

-    Since there is a very low (0.5**10) probability that it fails we mark it
-    as flaky.
+    Note there is a very low (0.5**10) probability that it fails.
    """
    # We need l3 for random walk
    l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
--- a/tests/test_plugin.py
+++ b/tests/test_plugin.py
@@ -1,7 +1,6 @@
 from collections import OrderedDict
 from datetime import datetime
 from fixtures import *  # noqa: F401,F403
-from flaky import flaky  # noqa: F401
 from hashlib import sha256
 from pyln.client import RpcError, Millisatoshi
 from pyln.proto import Invoice
--- a/tests/test_wallet.py
+++ b/tests/test_wallet.py
@@ -2,7 +2,6 @@ from bitcoin.rpc import JSONRPCError
 from decimal import Decimal
 from fixtures import *  # noqa: F401,F403
 from fixtures import TEST_NETWORK
-from flaky import flaky  # noqa: F401
 from pyln.client import RpcError, Millisatoshi
 from utils import (
    only_one, wait_for, sync_blockheight, EXPERIMENTAL_FEATURES,
@@ -1295,13 +1294,10 @@ def test_withdraw_nlocktime(node_factory):
    assert nlocktime > 0 and nlocktime <= tip


-@flaky
@unittest.skipIf(VALGRIND, "A big loop is used to check fuzz.")
 def test_withdraw_nlocktime_fuzz(node_factory, bitcoind):
    """
    Test that we eventually fuzz nLockTime for withdrawal transactions.
-    Marked flaky "just in case" as we fuzz from 0 to 100 with a 10%
-    probability.
    """
    l1 = node_factory.get_node(1)
    l1.fundwallet(10**8)