From 2b5e0c9f35a831c747011838088d5ecd1ec2f671 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 1 Feb 2021 17:09:40 +0100 Subject: [PATCH] gci: Move the setup and build steps into their own scripts We'd like to have some more structure to the tests, with smoke-tests going first, followed by valgrind and normal tests running in parallel. So to cut down on the copy-pasta we extract the scripts from the yaml, into their own files from where they can be easily invoked. --- .github/scripts/build.sh | 111 ++++++++++++++++++++++++ .github/scripts/setup.sh | 68 +++++++++++++++ .github/workflows/ci.yaml | 177 +------------------------------------- 3 files changed, 181 insertions(+), 175 deletions(-) create mode 100755 .github/scripts/build.sh create mode 100755 .github/scripts/setup.sh diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh new file mode 100755 index 000000000..f74b4b967 --- /dev/null +++ b/.github/scripts/build.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +echo "Running in $(pwd)" +export ARCH=${ARCH:-64} +export BOLTDIR=lightning-rfc +export CC=${COMPILER:-gcc} +export COMPAT=${COMPAT:-1} +export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0} +export DEVELOPER=${DEVELOPER:-1} +export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0} +export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH" +export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}" +export PYTEST_PAR=${PYTEST_PAR:-10} +export PYTEST_SENTRY_ALWAYS_REPORT=1 +export SLOW_MACHINE=1 +export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"} +export TEST_DB_PROVIDER=${DB:-"sqlite3"} +export TEST_NETWORK=${NETWORK:-"regtest"} +export TIMEOUT=900 +export VALGRIND=${VALGRIND:-0} + +env + +pip3 install --user -U \ + -r requirements.txt \ + -r contrib/pyln-client/requirements.txt \ + -r contrib/pyln-proto/requirements.txt \ + -r contrib/pyln-testing/requirements.txt + +# Install utilities that aren't dependencies, but make +# running tests easier/feasible on CI (and pytest which +# keeps breaking the rerunfailures plugin). +pip3 install --user -U \ + flaky \ + blinker \ + pytest-sentry \ + pytest-test-groups==1.0.3 \ + pytest-custom-exit-code==0.3.0 \ + pytest-json-report \ + mypy==0.790 + +git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc +git submodule update --init --recursive + +./configure CC="$CC" +cat config.vars + +cat << EOF > pytest.ini +[pytest] +addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2 +markers = + slow_test: marks tests as slow (deselect with '-m "not slow_test"') +EOF + +if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ] +then + export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/ + export MAKE_HOST="$TARGET_HOST" + export BUILD=x86_64-pc-linux-gnu + export AR="$TARGET_HOST"-ar + export AS="$TARGET_HOST"-as + export CC="$TARGET_HOST"-gcc + export CXX="$TARGET_HOST"-g++ + export LD="$TARGET_HOST"-ld + export STRIP="$TARGET_HOST"-strip + export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static + + wget -q https://zlib.net/zlib-1.2.11.tar.gz + tar xf zlib-1.2.11.tar.gz + cd zlib-1.2.11 || exit 1 + ./configure --prefix="$QEMU_LD_PREFIX" + make + sudo make install + cd .. || exit 1 + rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11 + + wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip + unzip -q sqlite-src-3260000.zip + cd sqlite-src-3260000 || exit 1 + automake --add-missing --force-missing --copy || true + ./configure --disable-tcl \ + --enable-static \ + --disable-readline \ + --disable-threadsafe \ + --disable-load-extension \ + --host="$TARGET_HOST" \ + --prefix="$QEMU_LD_PREFIX" + make + sudo make install + cd .. || exit 1 + rm sqlite-src-3260000.zip + rm -rf sqlite-src-3260000 + + wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz + tar xf gmp-6.1.2.tar.xz + cd gmp-6.1.2 || exit 1 + ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST" + make + sudo make install + cd .. + rm gmp-6.1.2.tar.xz + rm -rf gmp-6.1.2 + + ./configure CC="$TARGET_HOST-gcc" --enable-static + + make -j32 CC="$TARGET_HOST-gcc" > /dev/null +else + eatmydata make -j32 + # shellcheck disable=SC2086 + eatmydata $TEST_CMD +fi diff --git a/.github/scripts/setup.sh b/.github/scripts/setup.sh new file mode 100755 index 000000000..3833cdd80 --- /dev/null +++ b/.github/scripts/setup.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive +export BITCOIN_VERSION=0.20.1 +export ELEMENTS_VERSION=0.18.1.8 +sudo useradd -ms /bin/bash tester +sudo apt-get update -qq + +sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ + autoconf \ + automake \ + binfmt-support \ + build-essential \ + clang \ + cppcheck \ + docbook-xml \ + eatmydata \ + gcc-aarch64-linux-gnu \ + gcc-arm-linux-gnueabihf \ + gcc-arm-none-eabi \ + gettext \ + git \ + libc6-dev-arm64-cross \ + libc6-dev-armhf-cross \ + libgmp-dev \ + libpq-dev \ + libprotobuf-c-dev \ + libsqlite3-dev \ + libtool \ + libxml2-utils \ + locales \ + net-tools \ + postgresql \ + python-pkg-resources \ + python3 \ + python3-dev \ + python3-pip \ + python3-setuptools \ + qemu \ + qemu-system-arm \ + qemu-user-static \ + shellcheck \ + software-properties-common \ + sudo \ + tcl \ + unzip \ + valgrind \ + wget \ + xsltproc \ + zlib1g-dev + +echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester +sudo chmod 0440 /etc/sudoers.d/tester + +( + cd /tmp/ || exit 1 + wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 + wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 + tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 + tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 + sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin + sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin + rm -rf \ + bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \ + bitcoin-$BITCOIN_VERSION \ + elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \ + elements-$ELEMENTS_VERSION +) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f3f7c7ff9..30f80fc81 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,70 +57,7 @@ jobs: - name: Install dependencies run: | - export DEBIAN_FRONTEND=noninteractive - export BITCOIN_VERSION=0.20.1 - export ELEMENTS_VERSION=0.18.1.8 - sudo useradd -ms /bin/bash tester - sudo apt-get update -qq - - sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ - autoconf \ - automake \ - binfmt-support \ - build-essential \ - clang \ - cppcheck \ - docbook-xml \ - eatmydata \ - gcc-aarch64-linux-gnu \ - gcc-arm-linux-gnueabihf \ - gcc-arm-none-eabi \ - gettext \ - git \ - libc6-dev-arm64-cross \ - libc6-dev-armhf-cross \ - libgmp-dev \ - libpq-dev \ - libprotobuf-c-dev \ - libsqlite3-dev \ - libtool \ - libxml2-utils \ - locales \ - net-tools \ - postgresql \ - python-pkg-resources \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - qemu \ - qemu-system-arm \ - qemu-user-static \ - shellcheck \ - software-properties-common \ - sudo \ - tcl \ - unzip \ - valgrind \ - wget \ - xsltproc \ - zlib1g-dev - - echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester - sudo chmod 0440 /etc/sudoers.d/tester - - cd /tmp/ && \ - wget https://storage.googleapis.com/c-lightning-tests/bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 && \ - wget -q https://storage.googleapis.com/c-lightning-tests/elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 && \ - tar -xjf bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.bz2 && \ - tar -xjf elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 && \ - sudo mv bitcoin-$BITCOIN_VERSION/bin/* /usr/local/bin && \ - sudo mv elements-$ELEMENTS_VERSION/bin/* /usr/local/bin && \ - rm -rf \ - bitcoin-$BITCOIN_VERSION-x86_64-linux-gnu.tar.gz \ - bitcoin-$BITCOIN_VERSION \ - elements-$ELEMENTS_VERSION-x86_64-linux-gnu.tar.bz2 \ - elements-$ELEMENTS_VERSION + bash -x .github/scripts/setup.sh - name: Build env: @@ -137,117 +74,7 @@ jobs: TEST_GROUP_COUNT: ${{ matrix.TEST_GROUP_COUNT }} TEST_GROUP: ${{ matrix.TEST_GROUP }} run: | - echo Running in $(pwd) - export ARCH=${ARCH:-64} - export BOLTDIR=lightning-rfc - export CC=${COMPILER:-gcc} - export COMPAT=${COMPAT:-1} - export TEST_CHECK_DBSTMTS=${TEST_CHECK_DBSTMTS:-0} - export DEVELOPER=${DEVELOPER:-1} - export EXPERIMENTAL_FEATURES=${EXPERIMENTAL_FEATURES:-0} - export PATH=$CWD/dependencies/bin:"$HOME"/.local/bin:"$PATH" - export PYTEST_OPTS="--maxfail=5 --suppress-no-test-exit-code ${PYTEST_OPTS}" - export PYTEST_PAR=${PYTEST_PAR:-10} - export PYTEST_SENTRY_ALWAYS_REPORT=1 - export SLOW_MACHINE=1 - export TEST_CMD=${TEST_CMD:-"make -j $PYTEST_PAR pytest"} - export TEST_DB_PROVIDER=${DB:-"sqlite3"} - export TEST_NETWORK=${NETWORK:-"regtest"} - export TIMEOUT=900 - export VALGRIND=${VALGRIND:-0} - - env - - pip3 install --user -U \ - -r requirements.txt \ - -r contrib/pyln-client/requirements.txt \ - -r contrib/pyln-proto/requirements.txt \ - -r contrib/pyln-testing/requirements.txt - - # Install utilities that aren't dependencies, but make - # running tests easier/feasible on CI (and pytest which - # keeps breaking the rerunfailures plugin). - pip3 install --user -U \ - flaky \ - blinker \ - pytest-sentry \ - pytest-test-groups==1.0.3 \ - pytest-custom-exit-code==0.3.0 \ - pytest-json-report \ - mypy==0.790 - - git clone https://github.com/lightningnetwork/lightning-rfc.git ../lightning-rfc - - git submodule init - git submodule update - (cd external/libwally-core/src/; git submodule init; git submodule update) - - ./configure CC="$CC" - cat config.vars - - cat << 'EOF' > pytest.ini - [pytest] - addopts=-p no:logging --color=yes --timeout=1800 --timeout-method=thread --test-group-random-seed=42 --force-flaky --no-success-flaky-report --max-runs=3 --junitxml=report.xml --json-report --json-report-file=report.json --json-report-indent=2 - markers = - slow_test: marks tests as slow (deselect with '-m "not slow_test"') - EOF - - if [ "$TARGET_HOST" == "arm-linux-gnueabihf" ] || [ "$TARGET_HOST" == "aarch64-linux-gnu" ] - then - export QEMU_LD_PREFIX=/usr/"$TARGET_HOST"/ - export MAKE_HOST="$TARGET_HOST" - export BUILD=x86_64-pc-linux-gnu - export AR="$TARGET_HOST"-ar - export AS="$TARGET_HOST"-as - export CC="$TARGET_HOST"-gcc - export CXX="$TARGET_HOST"-g++ - export LD="$TARGET_HOST"-ld - export STRIP="$TARGET_HOST"-strip - export CONFIGURATION_WRAPPER=qemu-"${TARGET_HOST%%-*}"-static - - wget -q https://zlib.net/zlib-1.2.11.tar.gz - tar xf zlib-1.2.11.tar.gz - cd zlib-1.2.11 - ./configure --prefix="$QEMU_LD_PREFIX" - make - sudo make install - cd .. - rm zlib-1.2.11.tar.gz && rm -rf zlib-1.2.11 - - wget -q https://www.sqlite.org/2018/sqlite-src-3260000.zip - unzip -q sqlite-src-3260000.zip - cd sqlite-src-3260000 - automake --add-missing --force-missing --copy || true - ./configure --disable-tcl \ - --enable-static \ - --disable-readline \ - --disable-threadsafe \ - --disable-load-extension \ - --host="$TARGET_HOST" \ - --prefix="$QEMU_LD_PREFIX" - make - sudo make install - cd .. - rm sqlite-src-3260000.zip - rm -rf sqlite-src-3260000 - - wget -q https://gmplib.org/download/gmp/gmp-6.1.2.tar.xz - tar xf gmp-6.1.2.tar.xz - cd gmp-6.1.2 - ./configure --disable-assembly --prefix="$QEMU_LD_PREFIX" --host="$TARGET_HOST" - make - sudo make install - cd .. - rm gmp-6.1.2.tar.xz - rm -rf gmp-6.1.2 - - ./configure CC="$TARGET_HOST-gcc" --enable-static - - make -j32 CC="$TARGET_HOST-gcc" > /dev/null - else - eatmydata make -j32 - eatmydata $TEST_CMD - fi + bash -x .github/scripts/build.sh - name: Upload Unit Test Results if: always()