gci: Move the setup and build steps into their own scripts

We'd like to have some more structure to the tests, with smoke-tests
going first, followed by valgrind and normal tests running in
parallel. So to cut down on the copy-pasta we extract the scripts from
the yaml, into their own files from where they can be easily invoked.

.github/scripts/build.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+echo "Running in $(pwd)"
+export ARCH=${ARCH:-64}
+export BOLTDIR=lightning-rfc
+export CC=${COMPILER:-gcc}
+export COMPAT=${COMPAT:-1}
+export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0}
+export DEVELOPER=${DEVELOPER:-1}
+export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
+export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
+export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}"
+export PYTEST_PAR=${PYTEST_PAR:-10}
+export PYTEST_SENTRY_ALWAYS_REPORT=1
+export SLOW_MACHINE=1
+export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
+export TEST_DB_PROVIDER=${DB:-"sqlite3"}
+export TEST_NETWORK=${NETWORK:-"regtest"}
+export TIMEOUT=900
+export VALGRIND=${VALGRIND:-0}
+
+env
+
+pip3 install --user -U \
+     -r requirements.txt \
+     -r contrib/pyln-client/requirements.txt \
+     -r contrib/pyln-proto/requirements.txt \
+     -r contrib/pyln-testing/requirements.txt
+
+# Install utilities that aren't dependencies, but make
+# running tests easier/feasible on CI (and pytest which
+# keeps breaking the rerunfailures plugin).
+pip3 install --user -U \
+     flaky \
+     blinker \
+     pytest-sentry \
+     pytest-test-groups==1.0.3 \
+     pytest-custom-exit-code==0.3.0 \
+     pytest-json-report \
+     mypy==0.790
+
+git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc
+git submodule update --init --recursive
+
+./configure CC="$CC"
+cat config.vars
+
+cat << EOF > pytest.ini
+[pytest]
+addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
+markers =
+    slow_test: marks tests as slow (deselect with '-m "not slow_test"')
+EOF
+
+if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
+then
+    export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
+    export MAKE_HOST="$TARGET_HOST"
+    export BUILD=x86_64-pc-linux-gnu
+    export AR="$TARGET_HOST"-ar
+    export AS="$TARGET_HOST"-as
+    export CC="$TARGET_HOST"-gcc
+    export CXX="$TARGET_HOST"-g++
+    export LD="$TARGET_HOST"-ld
+    export STRIP="$TARGET_HOST"-strip
+    export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static
+
+    wget -q https://zlib.net/zlib-1.2.11.tar.gz
+    tar xf zlib-1.2.11.tar.gz
+    cd zlib-1.2.11 || exit 1
+    ./configure --prefix="$QEMU_LD_PREFIX"
+    make
+    sudo make install
+    cd .. || exit 1
+    rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
+
+    wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip
+    unzip -q sqlite-src-3260000.zip
+    cd sqlite-src-3260000 || exit 1
+    automake --add-missing --force-missing --copy || true
+    ./configure --disable-tcl \
+     --enable-static \
+     --disable-readline \
+     --disable-threadsafe \
+     --disable-load-extension \
+     --host="$TARGET_HOST" \
+     --prefix="$QEMU_LD_PREFIX"
+    make
+    sudo make install
+    cd .. || exit 1
+    rm sqlite-src-3260000.zip
+    rm -rf sqlite-src-3260000
+
+    wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
+    tar xf gmp-6.1.2.tar.xz
+    cd gmp-6.1.2 || exit 1
+    ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST"
+    make
+    sudo make install
+    cd ..
+    rm gmp-6.1.2.tar.xz
+    rm -rf gmp-6.1.2
+
+    ./configure CC="$TARGET_HOST-gcc" --enable-static
+
+    make -j32 CC="$TARGET_HOST-gcc" > /dev/null
+else
+    eatmydata make -j32
+    # shellcheck disable=SC2086
+    eatmydata $TEST_CMD
+fi

.github/scripts/setup.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+export DEBIAN_FRONTEND=noninteractive
+export BITCOIN_VERSION=0.20.1
+export ELEMENTS_VERSION=0.18.1.8
+sudo useradd -ms /bin/bash tester
+sudo apt-get update -qq
+
+sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
+     autoconf \
+     automake \
+     binfmt-support \
+     build-essential \
+     clang \
+     cppcheck \
+     docbook-xml \
+     eatmydata \
+     gcc-aarch64-linux-gnu \
+     gcc-arm-linux-gnueabihf \
+     gcc-arm-none-eabi \
+     gettext \
+     git \
+     libc6-dev-arm64-cross \
+     libc6-dev-armhf-cross \
+     libgmp-dev \
+     libpq-dev \
+     libprotobuf-c-dev \
+     libsqlite3-dev \
+     libtool \
+     libxml2-utils \
+     locales \
+     net-tools \
+     postgresql \
+     python-pkg-resources \
+     python3 \
+     python3-dev \
+     python3-pip \
+     python3-setuptools \
+     qemu \
+     qemu-system-arm \
+     qemu-user-static \
+     shellcheck \
+     software-properties-common \
+     sudo \
+     tcl \
+     unzip \
+     valgrind \
+     wget \
+     xsltproc \
+     zlib1g-dev
+
+echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
+sudo chmod 0440 /etc/sudoers.d/tester
+
+(
+    cd /tmp/ || exit 1
+    wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2
+    wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2
+    tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2
+    tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2
+    sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin
+    sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin
+    rm -rf \
+       bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \
+       bitcoin-$BITCOIN_VERSION \
+       elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \
+       elements-$ELEMENTS_VERSION
+)

.github/workflows/ci.yaml

@@ -57,70 +57,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          export DEBIAN_FRONTEND=noninteractive
+          bash -x .github/scripts/setup.sh
-          export BITCOIN_VERSION=0.20.1
-          export ELEMENTS_VERSION=0.18.1.8
-          sudo useradd -ms /bin/bash tester
-          sudo apt-get update -qq
-
-          sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
-          autoconf \
-          automake \
-          binfmt-support \
-          build-essential \
-          clang \
-          cppcheck \
-          docbook-xml \
-          eatmydata \
-          gcc-aarch64-linux-gnu \
-          gcc-arm-linux-gnueabihf \
-          gcc-arm-none-eabi \
-          gettext \
-          git \
-          libc6-dev-arm64-cross \
-          libc6-dev-armhf-cross \
-          libgmp-dev \
-          libpq-dev \
-          libprotobuf-c-dev \
-          libsqlite3-dev \
-          libtool \
-          libxml2-utils \
-          locales \
-          net-tools \
-          postgresql \
-          python-pkg-resources \
-          python3 \
-          python3-dev \
-          python3-pip \
-          python3-setuptools \
-          qemu \
-          qemu-system-arm \
-          qemu-user-static \
-          shellcheck \
-          software-properties-common \
-          sudo \
-          tcl \
-          unzip \
-          valgrind \
-          wget \
-          xsltproc \
-          zlib1g-dev
-
-          echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester
-          sudo chmod 0440 /etc/sudoers.d/tester
-
-          cd /tmp/ && \
-          wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 && \
-          wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 && \
-          tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 && \
-          tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 && \
-          sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin && \
-          sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin && \
-          rm -rf \
-          bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \
-          bitcoin-$BITCOIN_VERSION \
-          elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \
-          elements-$ELEMENTS_VERSION
 
       - name: Build
         env:
@@ -137,117 +74,7 @@ jobs:
           TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }}
           TEST_GROUP: ${{ matrix.TEST_GROUP }}
         run: |
-          echo Running in $(pwd)
+          bash -x .github/scripts/build.sh
-          export ARCH=${ARCH:-64}
-          export BOLTDIR=lightning-rfc
-          export CC=${COMPILER:-gcc}
-          export COMPAT=${COMPAT:-1}
-          export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0}
-          export DEVELOPER=${DEVELOPER:-1}
-          export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0}
-          export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH"
-          export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}"
-          export PYTEST_PAR=${PYTEST_PAR:-10}
-          export PYTEST_SENTRY_ALWAYS_REPORT=1
-          export SLOW_MACHINE=1
-          export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"}
-          export TEST_DB_PROVIDER=${DB:-"sqlite3"}
-          export TEST_NETWORK=${NETWORK:-"regtest"}
-          export TIMEOUT=900
-          export VALGRIND=${VALGRIND:-0}
-
-          env
-
-          pip3 install --user -U \
-            -r requirements.txt \
-            -r contrib/pyln-client/requirements.txt \
-            -r contrib/pyln-proto/requirements.txt \
-            -r contrib/pyln-testing/requirements.txt
-
-          # Install utilities that aren't dependencies, but make
-          # running tests easier/feasible on CI (and pytest which
-          # keeps breaking the rerunfailures plugin).
-          pip3 install --user -U \
-            flaky \
-            blinker \
-            pytest-sentry \
-            pytest-test-groups==1.0.3 \
-            pytest-custom-exit-code==0.3.0 \
-            pytest-json-report \
-            mypy==0.790
-
-          git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc
-
-          git submodule init
-          git submodule update
-          (cd external/libwally-core/src/; git submodule init; git submodule update)
-
-          ./configure CC="$CC"
-          cat config.vars
-
-          cat << 'EOF' > pytest.ini
-          [pytest]
-          addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2
-          markers =
-              slow_test: marks tests as slow (deselect with '-m "not slow_test"')
-          EOF
-
-          if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ]
-          then
-            export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/
-            export MAKE_HOST="$TARGET_HOST"
-            export BUILD=x86_64-pc-linux-gnu
-            export AR="$TARGET_HOST"-ar
-            export AS="$TARGET_HOST"-as
-            export CC="$TARGET_HOST"-gcc
-            export CXX="$TARGET_HOST"-g++
-            export LD="$TARGET_HOST"-ld
-            export STRIP="$TARGET_HOST"-strip
-            export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static
-
-            wget -q https://zlib.net/zlib-1.2.11.tar.gz
-            tar xf zlib-1.2.11.tar.gz
-            cd zlib-1.2.11
-            ./configure --prefix="$QEMU_LD_PREFIX"
-            make
-            sudo make install
-            cd ..
-            rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11
-
-            wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip
-            unzip -q sqlite-src-3260000.zip
-            cd sqlite-src-3260000
-            automake --add-missing --force-missing --copy || true
-            ./configure --disable-tcl \
-              --enable-static \
-              --disable-readline \
-              --disable-threadsafe \
-              --disable-load-extension \
-              --host="$TARGET_HOST" \
-              --prefix="$QEMU_LD_PREFIX"
-            make
-            sudo make install
-            cd ..
-            rm sqlite-src-3260000.zip
-            rm -rf sqlite-src-3260000
-
-            wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz
-            tar xf gmp-6.1.2.tar.xz
-            cd gmp-6.1.2
-            ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST"
-            make
-            sudo make install
-            cd ..
-            rm gmp-6.1.2.tar.xz
-            rm -rf gmp-6.1.2
-
-            ./configure CC="$TARGET_HOST-gcc" --enable-static
-
-            make -j32 CC="$TARGET_HOST-gcc" > /dev/null
-          else
-            eatmydata make -j32
-            eatmydata $TEST_CMD
-          fi
 
       - name: Upload Unit Test Results
         if: always()