From d06bb70514692fde47c9613cddf92a1dd28d7882 Mon Sep 17 00:00:00 2001 From: Jussi Saurio Date: Mon, 2 Jun 2025 12:13:24 +0300 Subject: [PATCH] Add simulator-docker-runner for running limbo-sim in a loop on AWS --- .github/workflows/build-sim.yml | 44 +++++ simulator-docker-runner/Dockerfile.simulator | 74 ++++++++ simulator-docker-runner/README.MD | 20 ++ simulator-docker-runner/bun.lock | 93 ++++++++++ .../docker-entrypoint.simulator.ts | 174 ++++++++++++++++++ simulator-docker-runner/github.ts | 150 +++++++++++++++ simulator-docker-runner/levenshtein.test.ts | 26 +++ simulator-docker-runner/levenshtein.ts | 40 ++++ simulator-docker-runner/logParse.ts | 58 ++++++ simulator-docker-runner/package.json | 14 ++ simulator-docker-runner/random.ts | 5 + simulator-docker-runner/tsconfig.json | 23 +++ 12 files changed, 721 insertions(+) create mode 100644 .github/workflows/build-sim.yml create mode 100644 simulator-docker-runner/Dockerfile.simulator create mode 100644 simulator-docker-runner/README.MD create mode 100644 simulator-docker-runner/bun.lock create mode 100644 simulator-docker-runner/docker-entrypoint.simulator.ts create mode 100644 simulator-docker-runner/github.ts create mode 100644 simulator-docker-runner/levenshtein.test.ts create mode 100644 simulator-docker-runner/levenshtein.ts create mode 100644 simulator-docker-runner/logParse.ts create mode 100644 simulator-docker-runner/package.json create mode 100644 simulator-docker-runner/random.ts create mode 100644 simulator-docker-runner/tsconfig.json diff --git a/.github/workflows/build-sim.yml b/.github/workflows/build-sim.yml new file mode 100644 index 000000000..5871edf8c --- /dev/null +++ b/.github/workflows/build-sim.yml @@ -0,0 +1,44 @@ +name: Build and push limbo-sim image + +on: + push: + branches: + - main + +# Add permissions needed for OIDC authentication with AWS +permissions: + id-token: write # allow getting OIDC token + contents: read # allow reading repository contents + +# Ensure only one build runs at a time. A new push to main will cancel any in-progress build. +concurrency: + group: "build-sim" + cancel-in-progress: true + +env: + AWS_REGION: ${{ secrets.LIMBO_SIM_AWS_REGION }} + IAM_ROLE: ${{ secrets.LIMBO_SIM_DEPLOYER_IAM_ROLE }} + ECR_URL: ${{ secrets.LIMBO_SIM_ECR_URL }} + ECR_IMAGE_NAME: ${{ secrets.LIMBO_SIM_IMAGE_NAME }} + GIT_HASH: ${{ github.sha }} + +jobs: + deploy: + runs-on: blacksmith + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.IAM_ROLE }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and push limbo-sim docker image + run: | + docker build -f simulator-docker-runner/Dockerfile.simulator -t ${{ env.ECR_URL }}/${{ env.ECR_IMAGE_NAME }} --build-arg GIT_HASH=${{ env.GIT_HASH }} . + docker push ${{ env.ECR_URL }}/${{ env.ECR_IMAGE_NAME }} diff --git a/simulator-docker-runner/Dockerfile.simulator b/simulator-docker-runner/Dockerfile.simulator new file mode 100644 index 000000000..e0cd8b4c4 --- /dev/null +++ b/simulator-docker-runner/Dockerfile.simulator @@ -0,0 +1,74 @@ +FROM lukemathwalker/cargo-chef:latest-rust-1.87.0 AS chef +RUN apt update \ + && apt install -y git libssl-dev pkg-config\ + && apt clean \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app + +# +# Cache dependencies +# + +FROM chef AS planner +# cargo-chef requires all this crap to be present in the workspace +COPY Cargo.toml Cargo.lock ./ +COPY core ./core/ +COPY simulator ./simulator/ +COPY bindings ./bindings/ +COPY extensions ./extensions/ +COPY macros ./macros/ +COPY vendored ./vendored/ +COPY cli ./cli/ +COPY sqlite3 ./sqlite3/ +COPY stress ./stress/ +COPY tests ./tests/ +RUN cargo chef prepare --bin limbo_sim --recipe-path recipe.json + +# +# Build the project. +# + +FROM chef AS builder + +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --bin limbo_sim --release --recipe-path recipe.json +COPY --from=planner /app/core ./core/ +COPY --from=planner /app/vendored ./vendored/ +COPY --from=planner /app/extensions ./extensions/ +COPY --from=planner /app/macros ./macros/ +COPY --from=planner /app/simulator ./simulator/ + +RUN cargo build --bin limbo_sim --release + +# +# The final image. +# + +FROM debian:bookworm-slim AS runtime +# Install Bun (we want to use a fancy shell so we can e.g. post github issues when simulator fails) +RUN apt-get update && apt-get install -y \ + curl \ + unzip \ + ca-certificates \ + libssl3 \ + openssl \ + && curl -fsSL https://bun.sh/install | bash \ + && ln -s /root/.bun/bin/bun /usr/local/bin/bun \ + && rm -rf /var/lib/apt/lists/* + +# Accept git hash as build argument +ARG GIT_HASH +ENV GIT_HASH=${GIT_HASH:-unknown} + +WORKDIR /app +COPY --from=builder /app/target/release/limbo_sim /app/limbo_sim +COPY simulator-docker-runner/package.json simulator-docker-runner/bun.lock simulator-docker-runner/tsconfig.json ./ +RUN bun install +COPY simulator-docker-runner/* ./ + +RUN chmod +x /app/docker-entrypoint.simulator.ts +RUN chmod +x /app/limbo_sim + +ENTRYPOINT ["bun", "/app/docker-entrypoint.simulator.ts"] +# Arguments can be passed at runtime +CMD [] diff --git a/simulator-docker-runner/README.MD b/simulator-docker-runner/README.MD new file mode 100644 index 000000000..41ef71175 --- /dev/null +++ b/simulator-docker-runner/README.MD @@ -0,0 +1,20 @@ +# Limbo Simulator Docker Runner + +This directory contains the script that runs inside the `limbo-sim` Docker image. The script continuously runs the `limbo-sim` program in a loop until it encounters a panic, at which point it automatically creates a GitHub issue in the `limbo` repository. + +## What it does + +1. The limbo-sim image is built and pushed to ECR by [.github/workflows/build-sim.yaml](../.github/workflows/build-sim.yaml) on every main commit +2. When the container starts, this script: + - Runs the [limbo-sim](../simulator/) program with a random seed + - If a panic occurs: + - Captures the seed value and commit hash + - Creates a GitHub issue with reproduction steps + - Includes panic output and relevant metadata + - Continues running with a new seed until a panic occurs or TIME_LIMIT_MINUTES is reached + +The script acts as the entrypoint for the Docker container, automatically starting the simulation loop when the container launches. + +## How do I see the open issues created by the simulator? + +[GitHub issues](https://github.com/tursodatabase/limbo/issues?q=is%3Aissue+is%3Aopen+label%3A%22automated%22) diff --git a/simulator-docker-runner/bun.lock b/simulator-docker-runner/bun.lock new file mode 100644 index 000000000..ae3622f63 --- /dev/null +++ b/simulator-docker-runner/bun.lock @@ -0,0 +1,93 @@ +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "simulator-docker-runner", + "dependencies": { + "octokit": "4.1.2", + }, + "devDependencies": { + "@types/node": "22.13.5", + "bun-types": "1.2.4", + "typescript": "5.7.3", + }, + }, + }, + "packages": { + "@octokit/app": ["@octokit/app@15.1.5", "", { "dependencies": { "@octokit/auth-app": "^7.1.5", "@octokit/auth-unauthenticated": "^6.1.2", "@octokit/core": "^6.1.4", "@octokit/oauth-app": "^7.1.6", "@octokit/plugin-paginate-rest": "^11.4.2", "@octokit/types": "^13.8.0", "@octokit/webhooks": "^13.6.1" } }, "sha512-6cxLT9U8x7GGQ7lNWsKtFr4ccg9oLkGvowk373sX9HvX5U37kql5d55SzaQUxPE8PwgX2cqkzDm5NF5aPKevqg=="], + + "@octokit/auth-app": ["@octokit/auth-app@7.1.5", "", { "dependencies": { "@octokit/auth-oauth-app": "^8.1.3", "@octokit/auth-oauth-user": "^5.1.3", "@octokit/request": "^9.2.1", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.8.0", "toad-cache": "^3.7.0", "universal-github-app-jwt": "^2.2.0", "universal-user-agent": "^7.0.0" } }, "sha512-boklS4E6LpbA3nRx+SU2fRKRGZJdOGoSZne/i3Y0B5rfHOcGwFgcXrwDLdtbv4igfDSnAkZaoNBv1GYjPDKRNw=="], + + "@octokit/auth-oauth-app": ["@octokit/auth-oauth-app@8.1.3", "", { "dependencies": { "@octokit/auth-oauth-device": "^7.1.3", "@octokit/auth-oauth-user": "^5.1.3", "@octokit/request": "^9.2.1", "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.0" } }, "sha512-4e6OjVe5rZ8yBe8w7byBjpKtSXFuro7gqeGAAZc7QYltOF8wB93rJl2FE0a4U1Mt88xxPv/mS+25/0DuLk0Ewg=="], + + "@octokit/auth-oauth-device": ["@octokit/auth-oauth-device@7.1.3", "", { "dependencies": { "@octokit/oauth-methods": "^5.1.4", "@octokit/request": "^9.2.1", "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.0" } }, "sha512-BECO/N4B/Uikj0w3GCvjf/odMujtYTP3q82BJSjxC2J3rxTEiZIJ+z2xnRlDb0IE9dQSaTgRqUPVOieSbFcVzg=="], + + "@octokit/auth-oauth-user": ["@octokit/auth-oauth-user@5.1.3", "", { "dependencies": { "@octokit/auth-oauth-device": "^7.1.3", "@octokit/oauth-methods": "^5.1.3", "@octokit/request": "^9.2.1", "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.0" } }, "sha512-zNPByPn9K7TC+OOHKGxU+MxrE9SZAN11UHYEFLsK2NRn3akJN2LHRl85q+Eypr3tuB2GrKx3rfj2phJdkYCvzw=="], + + "@octokit/auth-token": ["@octokit/auth-token@5.1.2", "", {}, "sha512-JcQDsBdg49Yky2w2ld20IHAlwr8d/d8N6NiOXbtuoPCqzbsiJgF633mVUw3x4mo0H5ypataQIX7SFu3yy44Mpw=="], + + "@octokit/auth-unauthenticated": ["@octokit/auth-unauthenticated@6.1.2", "", { "dependencies": { "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2" } }, "sha512-07DlUGcz/AAVdzu3EYfi/dOyMSHp9YsOxPl/MPmtlVXWiD//GlV8HgZsPhud94DEyx+RfrW0wSl46Lx+AWbOlg=="], + + "@octokit/core": ["@octokit/core@6.1.4", "", { "dependencies": { "@octokit/auth-token": "^5.0.0", "@octokit/graphql": "^8.1.2", "@octokit/request": "^9.2.1", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2", "before-after-hook": "^3.0.2", "universal-user-agent": "^7.0.0" } }, "sha512-lAS9k7d6I0MPN+gb9bKDt7X8SdxknYqAMh44S5L+lNqIN2NuV8nvv3g8rPp7MuRxcOpxpUIATWprO0C34a8Qmg=="], + + "@octokit/endpoint": ["@octokit/endpoint@10.1.3", "", { "dependencies": { "@octokit/types": "^13.6.2", "universal-user-agent": "^7.0.2" } }, "sha512-nBRBMpKPhQUxCsQQeW+rCJ/OPSMcj3g0nfHn01zGYZXuNDvvXudF/TYY6APj5THlurerpFN4a/dQAIAaM6BYhA=="], + + "@octokit/graphql": ["@octokit/graphql@8.2.1", "", { "dependencies": { "@octokit/request": "^9.2.2", "@octokit/types": "^13.8.0", "universal-user-agent": "^7.0.0" } }, "sha512-n57hXtOoHrhwTWdvhVkdJHdhTv0JstjDbDRhJfwIRNfFqmSo1DaK/mD2syoNUoLCyqSjBpGAKOG0BuwF392slw=="], + + "@octokit/oauth-app": ["@octokit/oauth-app@7.1.6", "", { "dependencies": { "@octokit/auth-oauth-app": "^8.1.3", "@octokit/auth-oauth-user": "^5.1.3", "@octokit/auth-unauthenticated": "^6.1.2", "@octokit/core": "^6.1.4", "@octokit/oauth-authorization-url": "^7.1.1", "@octokit/oauth-methods": "^5.1.4", "@types/aws-lambda": "^8.10.83", "universal-user-agent": "^7.0.0" } }, "sha512-OMcMzY2WFARg80oJNFwWbY51TBUfLH4JGTy119cqiDawSFXSIBujxmpXiKbGWQlvfn0CxE6f7/+c6+Kr5hI2YA=="], + + "@octokit/oauth-authorization-url": ["@octokit/oauth-authorization-url@7.1.1", "", {}, "sha512-ooXV8GBSabSWyhLUowlMIVd9l1s2nsOGQdlP2SQ4LnkEsGXzeCvbSbCPdZThXhEFzleGPwbapT0Sb+YhXRyjCA=="], + + "@octokit/oauth-methods": ["@octokit/oauth-methods@5.1.4", "", { "dependencies": { "@octokit/oauth-authorization-url": "^7.0.0", "@octokit/request": "^9.2.1", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2" } }, "sha512-Jc/ycnePClOvO1WL7tlC+TRxOFtyJBGuTDsL4dzXNiVZvzZdrPuNw7zHI3qJSUX2n6RLXE5L0SkFmYyNaVUFoQ=="], + + "@octokit/openapi-types": ["@octokit/openapi-types@23.0.1", "", {}, "sha512-izFjMJ1sir0jn0ldEKhZ7xegCTj/ObmEDlEfpFrx4k/JyZSMRHbO3/rBwgE7f3m2DHt+RrNGIVw4wSmwnm3t/g=="], + + "@octokit/openapi-webhooks-types": ["@octokit/openapi-webhooks-types@10.1.1", "", {}, "sha512-qBfqQVIDQaCFeGCofXieJDwvXcGgDn17+UwZ6WW6lfEvGYGreLFzTiaz9xjet9Us4zDf8iasoW3ixUj/R5lMhA=="], + + "@octokit/plugin-paginate-graphql": ["@octokit/plugin-paginate-graphql@5.2.4", "", { "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-pLZES1jWaOynXKHOqdnwZ5ULeVR6tVVCMm+AUbp0htdcyXDU95WbkYdU4R2ej1wKj5Tu94Mee2Ne0PjPO9cCyA=="], + + "@octokit/plugin-paginate-rest": ["@octokit/plugin-paginate-rest@11.4.3", "", { "dependencies": { "@octokit/types": "^13.7.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-tBXaAbXkqVJlRoA/zQVe9mUdb8rScmivqtpv3ovsC5xhje/a+NOCivs7eUhWBwCApJVsR4G5HMeaLbq7PxqZGA=="], + + "@octokit/plugin-rest-endpoint-methods": ["@octokit/plugin-rest-endpoint-methods@13.3.1", "", { "dependencies": { "@octokit/types": "^13.8.0" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-o8uOBdsyR+WR8MK9Cco8dCgvG13H1RlM1nWnK/W7TEACQBFux/vPREgKucxUfuDQ5yi1T3hGf4C5ZmZXAERgwQ=="], + + "@octokit/plugin-retry": ["@octokit/plugin-retry@7.1.4", "", { "dependencies": { "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2", "bottleneck": "^2.15.3" }, "peerDependencies": { "@octokit/core": ">=6" } }, "sha512-7AIP4p9TttKN7ctygG4BtR7rrB0anZqoU9ThXFk8nETqIfvgPUANTSYHqWYknK7W3isw59LpZeLI8pcEwiJdRg=="], + + "@octokit/plugin-throttling": ["@octokit/plugin-throttling@9.4.0", "", { "dependencies": { "@octokit/types": "^13.7.0", "bottleneck": "^2.15.3" }, "peerDependencies": { "@octokit/core": "^6.1.3" } }, "sha512-IOlXxXhZA4Z3m0EEYtrrACkuHiArHLZ3CvqWwOez/pURNqRuwfoFlTPbN5Muf28pzFuztxPyiUiNwz8KctdZaQ=="], + + "@octokit/request": ["@octokit/request@9.2.2", "", { "dependencies": { "@octokit/endpoint": "^10.1.3", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.6.2", "fast-content-type-parse": "^2.0.0", "universal-user-agent": "^7.0.2" } }, "sha512-dZl0ZHx6gOQGcffgm1/Sf6JfEpmh34v3Af2Uci02vzUYz6qEN6zepoRtmybWXIGXFIK8K9ylE3b+duCWqhArtg=="], + + "@octokit/request-error": ["@octokit/request-error@6.1.7", "", { "dependencies": { "@octokit/types": "^13.6.2" } }, "sha512-69NIppAwaauwZv6aOzb+VVLwt+0havz9GT5YplkeJv7fG7a40qpLt/yZKyiDxAhgz0EtgNdNcb96Z0u+Zyuy2g=="], + + "@octokit/types": ["@octokit/types@13.8.0", "", { "dependencies": { "@octokit/openapi-types": "^23.0.1" } }, "sha512-x7DjTIbEpEWXK99DMd01QfWy0hd5h4EN+Q7shkdKds3otGQP+oWE/y0A76i1OvH9fygo4ddvNf7ZvF0t78P98A=="], + + "@octokit/webhooks": ["@octokit/webhooks@13.7.4", "", { "dependencies": { "@octokit/openapi-webhooks-types": "10.1.1", "@octokit/request-error": "^6.1.7", "@octokit/webhooks-methods": "^5.1.1" } }, "sha512-f386XyLTieQbgKPKS6ZMlH4dq8eLsxNddwofiKRZCq0bZ2gikoFwMD99K6l1oAwqe/KZNzrEziGicRgnzplplQ=="], + + "@octokit/webhooks-methods": ["@octokit/webhooks-methods@5.1.1", "", {}, "sha512-NGlEHZDseJTCj8TMMFehzwa9g7On4KJMPVHDSrHxCQumL6uSQR8wIkP/qesv52fXqV1BPf4pTxwtS31ldAt9Xg=="], + + "@types/aws-lambda": ["@types/aws-lambda@8.10.147", "", {}, "sha512-nD0Z9fNIZcxYX5Mai2CTmFD7wX7UldCkW2ezCF8D1T5hdiLsnTWDGRpfRYntU6VjTdLQjOvyszru7I1c1oCQew=="], + + "@types/node": ["@types/node@22.13.5", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-+lTU0PxZXn0Dr1NBtC7Y8cR21AJr87dLLU953CWA6pMxxv/UDc7jYAY90upcrie1nRcD6XNG5HOYEDtgW5TxAg=="], + + "@types/ws": ["@types/ws@8.5.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw=="], + + "before-after-hook": ["before-after-hook@3.0.2", "", {}, "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A=="], + + "bottleneck": ["bottleneck@2.19.5", "", {}, "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="], + + "bun-types": ["bun-types@1.2.4", "", { "dependencies": { "@types/node": "*", "@types/ws": "~8.5.10" } }, "sha512-nDPymR207ZZEoWD4AavvEaa/KZe/qlrbMSchqpQwovPZCKc7pwMoENjEtHgMKaAjJhy+x6vfqSBA1QU3bJgs0Q=="], + + "fast-content-type-parse": ["fast-content-type-parse@2.0.1", "", {}, "sha512-nGqtvLrj5w0naR6tDPfB4cUmYCqouzyQiz6C5y/LtcDllJdrcc6WaWW6iXyIIOErTa/XRybj28aasdn4LkVk6Q=="], + + "octokit": ["octokit@4.1.2", "", { "dependencies": { "@octokit/app": "^15.1.4", "@octokit/core": "^6.1.4", "@octokit/oauth-app": "^7.1.6", "@octokit/plugin-paginate-graphql": "^5.2.4", "@octokit/plugin-paginate-rest": "^11.4.2", "@octokit/plugin-rest-endpoint-methods": "^13.3.1", "@octokit/plugin-retry": "^7.1.4", "@octokit/plugin-throttling": "^9.4.0", "@octokit/request-error": "^6.1.7", "@octokit/types": "^13.7.0" } }, "sha512-0kcTxJOK3yQrJsRb8wKa28hlTze4QOz4sLuUnfXXnhboDhFKgv8LxS86tFwbsafDW9JZ08ByuVAE8kQbYJIZkA=="], + + "toad-cache": ["toad-cache@3.7.0", "", {}, "sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw=="], + + "typescript": ["typescript@5.7.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw=="], + + "undici-types": ["undici-types@6.20.0", "", {}, "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="], + + "universal-github-app-jwt": ["universal-github-app-jwt@2.2.0", "", {}, "sha512-G5o6f95b5BggDGuUfKDApKaCgNYy2x7OdHY0zSMF081O0EJobw+1130VONhrA7ezGSV2FNOGyM+KQpQZAr9bIQ=="], + + "universal-user-agent": ["universal-user-agent@7.0.2", "", {}, "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q=="], + } +} diff --git a/simulator-docker-runner/docker-entrypoint.simulator.ts b/simulator-docker-runner/docker-entrypoint.simulator.ts new file mode 100644 index 000000000..3b97e6ead --- /dev/null +++ b/simulator-docker-runner/docker-entrypoint.simulator.ts @@ -0,0 +1,174 @@ +#!/usr/bin/env bun + +import { spawn } from "bun"; +import { GithubClient } from "./github"; +import { extractFailureInfo } from "./logParse"; +import { randomSeed } from "./random"; + +// Configuration from environment variables +const SLEEP_BETWEEN_RUNS_SECONDS = Number.isInteger(Number(process.env.SLEEP_BETWEEN_RUNS_SECONDS)) ? Number(process.env.SLEEP_BETWEEN_RUNS_SECONDS) : 0; +const TIME_LIMIT_MINUTES = Number.isInteger(Number(process.env.TIME_LIMIT_MINUTES)) ? Number(process.env.TIME_LIMIT_MINUTES) : 24 * 60; +const PER_RUN_TIMEOUT_SECONDS = Number.isInteger(Number(process.env.PER_RUN_TIMEOUT_SECONDS)) ? Number(process.env.PER_RUN_TIMEOUT_SECONDS) : 10 * 60; +const LOG_TO_STDOUT = process.env.LOG_TO_STDOUT === "true"; + +const github = new GithubClient(); + +process.env.RUST_BACKTRACE = "1"; + +console.log("Starting limbo_sim in a loop..."); +console.log(`Git hash: ${github.GIT_HASH}`); +console.log(`GitHub issues enabled: ${github.mode === 'real'}`); +console.log(`Time limit: ${TIME_LIMIT_MINUTES} minutes`); +console.log(`Log simulator output to stdout: ${LOG_TO_STDOUT}`); +console.log(`Sleep between runs: ${SLEEP_BETWEEN_RUNS_SECONDS} seconds`); +console.log(`Per run timeout: ${PER_RUN_TIMEOUT_SECONDS} seconds`); + +process.on("SIGINT", () => { + console.log("Received SIGINT, exiting..."); + process.exit(0); +}); +process.on("SIGTERM", () => { + console.log("Received SIGTERM, exiting..."); + process.exit(0); +}); + +class TimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = 'TimeoutError'; + } +} + +/** + * Returns a promise that rejects when the timeout is reached. + * Prints a message to the console every 10 seconds. + * @param seconds - The number of seconds to timeout. + * @param runNumber - The number of the run. + * @returns A promise that rejects when the timeout is reached. + */ +const timeouter = (seconds: number, runNumber: number) => { + const start = new Date(); + const stdoutNotifyInterval = setInterval(() => { + const elapsedSeconds = Math.round((new Date().getTime() - start.getTime()) / 1000); + console.log(`Run ${runNumber} - ${elapsedSeconds}s elapsed (timeout: ${seconds}s)`); + }, 10 * 1000); + let timeout: Timer; + const timeouterPromise = new Promise((_, reject) => { + timeout = setTimeout(() => { + clearInterval(stdoutNotifyInterval); + reject(new TimeoutError("Timeout")); + }, seconds * 1000); + }); + // @ts-ignore + timeouterPromise.clear = () => { + clearInterval(stdoutNotifyInterval); + if (timeout) { + clearTimeout(timeout); + } + } + return timeouterPromise; +} + +const run = async (seed: string, bin: string, args: string[]) => { + const proc = spawn([`/app/${bin}`, ...args], { + stdout: LOG_TO_STDOUT ? "inherit" : "pipe", + stderr: LOG_TO_STDOUT ? "inherit" : "pipe", + env: { ...process.env, SIMULATOR_SEED: seed } + }); + + const timeout = timeouter(PER_RUN_TIMEOUT_SECONDS, runNumber); + timeout.catch(async (err) => { + if (err instanceof TimeoutError) { + console.log(`Timeout on seed ${seed}, exiting...`); + proc.kill(); + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + const output = stdout + '\n' + stderr; + const seedForGithubIssue = seed; + const lastLines = output.split('\n').slice(-100).join('\n'); + console.log(`Simulator seed: ${seedForGithubIssue}`); + await github.postGitHubIssue({ + type: "timeout", + seed: seedForGithubIssue, + command: args.join(" "), + output: lastLines, + }); + process.exit(0); + } + }); + + try { + const exitCode = await proc.exited; + const stdout = await new Response(proc.stdout).text(); + const stderr = await new Response(proc.stderr).text(); + + if (exitCode !== 0) { + console.log(`[${new Date().toISOString()}]: ${bin} ${args.join(" ")} exited with code ${exitCode}`); + const output = stdout + stderr; + + // Extract simulator seed and stack trace + try { + const seedForGithubIssue = seed; + const failureInfo = extractFailureInfo(output); + + console.log(`Simulator seed: ${seedForGithubIssue}`); + + // Post the issue to Github and continue + if (failureInfo.type === "panic") { + await github.postGitHubIssue({ + type: "panic", + seed: seedForGithubIssue, + command: args.join(" "), + stackTrace: failureInfo, + }); + } else { + await github.postGitHubIssue({ + type: "assertion", + seed: seedForGithubIssue, + command: args.join(" "), + output: output, + }); + } + } catch (err2) { + console.error(`Error extracting simulator seed and stack trace: ${err2}`); + console.log("Last 100 lines of stdout: ", (stdout?.toString() || "").split("\n").slice(-100).join("\n")); + console.log("Last 100 lines of stderr: ", (stderr?.toString() || "").split("\n").slice(-100).join("\n")); + console.log(`Simulator seed: ${seed}`); + process.exit(1); + } + } + } catch (err) { + throw err; + } finally { + // @ts-ignore + timeout.clear(); + } +} + +// Main execution loop +const startTime = new Date(); +const limboSimArgs = process.argv.slice(2); +let runNumber = 0; +while (new Date().getTime() - startTime.getTime() < TIME_LIMIT_MINUTES * 60 * 1000) { + const timestamp = new Date().toISOString(); + const args = [...limboSimArgs]; + const seed = randomSeed(); + // Reproducible seed + args.push('--seed', seed); + // Bugbase wants to have .git available, so we disable it + args.push("--disable-bugbase"); + args.push(...["--minimum-tests", "100", "--maximum-tests", "1000"]); + const loop = args.includes("loop") ? [] : ["loop", "-n", "10", "--short-circuit"] + args.push(...loop); + + console.log(`[${timestamp}]: Running "limbo_sim ${args.join(" ")}" - (seed ${seed}, run number ${runNumber})`); + await run(seed, "limbo_sim", args); + + runNumber++; + + SLEEP_BETWEEN_RUNS_SECONDS > 0 && (await sleep(SLEEP_BETWEEN_RUNS_SECONDS)); +} + +async function sleep(sec: number) { + return new Promise(resolve => setTimeout(resolve, sec * 1000)); +} diff --git a/simulator-docker-runner/github.ts b/simulator-docker-runner/github.ts new file mode 100644 index 000000000..c76e03735 --- /dev/null +++ b/simulator-docker-runner/github.ts @@ -0,0 +1,150 @@ +import { App } from "octokit"; +import { StackTraceInfo } from "./logParse"; +import { levenshtein } from "./levenshtein"; + +type FaultPanic = { + type: "panic" + seed: string + command: string + stackTrace: StackTraceInfo +} + +type FaultAssertion = { + type: "assertion" + seed: string + command: string + output: string +} + +type FaultTimeout = { + type: "timeout" + seed: string + command: string + output: string +} + +type Fault = FaultPanic | FaultTimeout | FaultAssertion; + +export class GithubClient { + /* This is the git hash of the commit that the simulator was built from. */ + GIT_HASH: string; + /* Github app ID. */ + GITHUB_APP_ID: string; + /* This is the private key of the "Turso Github Handyman" Github App. It's stored in AWS secrets manager and will be injected into the container at runtime. */ + GITHUB_APP_PRIVATE_KEY: string; + /* This is the unique installation id of the above app into the tursodatabase organization. */ + GITHUB_APP_INSTALLATION_ID: number; + GITHUB_REPO: string; + GITHUB_ORG: string = "tursodatabase"; + GITHUB_REPO_NAME: string = "limbo"; + mode: 'real' | 'dry-run'; + app: App | null; + initialized: boolean = false; + openIssueTitles: string[] = []; + constructor() { + this.GIT_HASH = process.env.GIT_HASH || "unknown"; + this.GITHUB_APP_PRIVATE_KEY = process.env.GITHUB_APP_PRIVATE_KEY || ""; + this.GITHUB_APP_ID = process.env.GITHUB_APP_ID || ""; + this.GITHUB_APP_INSTALLATION_ID = parseInt(process.env.GITHUB_APP_INSTALLATION_ID || "0", 10); + this.mode = this.GITHUB_APP_PRIVATE_KEY ? 'real' : 'dry-run'; + this.GITHUB_REPO = `${this.GITHUB_ORG}/${this.GITHUB_REPO_NAME}`; + + // Initialize GitHub OAuth App + this.app = this.mode === 'real' ? new App({ + appId: this.GITHUB_APP_ID, + privateKey: this.GITHUB_APP_PRIVATE_KEY, + }) : null; + } + + private async getOpenIssues(): Promise { + const octokit = await this.app!.getInstallationOctokit(this.GITHUB_APP_INSTALLATION_ID); + const issues = await octokit.request('GET /repos/{owner}/{repo}/issues', { + owner: this.GITHUB_ORG, + repo: this.GITHUB_REPO_NAME, + state: 'open', + creator: 'app/turso-github-handyman', + }); + return issues.data.map((issue) => issue.title); + } + + async initialize(): Promise { + if (this.mode === 'dry-run') { + console.log("Dry-run mode: Skipping initialization"); + this.initialized = true; + return; + } + this.openIssueTitles = await this.getOpenIssues(); + this.initialized = true; + } + + async postGitHubIssue(fault: Fault): Promise { + if (!this.initialized) { + await this.initialize(); + } + + const title = ((f: Fault) => { + if (f.type === "panic") { + return `Simulator panic: "${f.stackTrace.mainError}"`; + } else + if (f.type === "assertion") { + return `Simulator assertion failure: "${f.output}"`; + } + return `Simulator timeout using git hash ${this.GIT_HASH}`; + })(fault); + for (const existingIssueTitle of this.openIssueTitles) { + const MAGIC_NUMBER = 6; + if (levenshtein(existingIssueTitle, title) < MAGIC_NUMBER) { + console.log(`Not creating issue ${title} because it is too similar to ${existingIssueTitle}`); + return; + } + } + + const body = this.createIssueBody(fault); + + if (this.mode === 'dry-run') { + console.log(`Dry-run mode: Would create issue in ${this.GITHUB_REPO} with title: ${title} and body: ${body}`); + return; + } + + const [owner, repo] = this.GITHUB_REPO.split('/'); + + const octokit = await this.app!.getInstallationOctokit(this.GITHUB_APP_INSTALLATION_ID); + + const response = await octokit.request('POST /repos/{owner}/{repo}/issues', { + owner, + repo, + title, + body, + labels: ['bug', 'simulator', 'automated'] + }); + + console.log(`Successfully created GitHub issue: ${response.data.html_url}`); + this.openIssueTitles.push(title); + } + + private createIssueBody(fault: Fault): string { + const gitShortHash = this.GIT_HASH.substring(0, 7); + return ` + ## Simulator ${fault.type} + + - **Seed**: ${fault.seed} + - **Git Hash**: ${this.GIT_HASH} + - **Command**: \`limbo-sim ${fault.command}\` + - **Timestamp**: ${new Date().toISOString()} + + ### Run locally with Docker + + \`\`\` + git checkout ${this.GIT_HASH} + docker buildx build -t limbo-sim:${gitShortHash} -f simulator-docker-runner/Dockerfile.simulator . --build-arg GIT_HASH=$(git rev-parse HEAD) + docker run --network host limbo-sim:${gitShortHash} ${fault.command} + \`\`\` + + ### ${fault.type === "panic" ? "Stack Trace" : "Output"} + + \`\`\` + ${fault.type === "panic" ? fault.stackTrace.trace : fault.output} + \`\`\` + `; + } +} \ No newline at end of file diff --git a/simulator-docker-runner/levenshtein.test.ts b/simulator-docker-runner/levenshtein.test.ts new file mode 100644 index 000000000..adf14b8f7 --- /dev/null +++ b/simulator-docker-runner/levenshtein.test.ts @@ -0,0 +1,26 @@ +import { test, expect } from "bun:test"; +import { levenshtein } from "./levenshtein"; + +test("levenshtein distance between empty strings is 0", () => { + expect(levenshtein("", "")).toBe(0); +}); + +test("levenshtein distance between completely different same length strings is their length", () => { + expect(levenshtein("abcde", "fghij")).toBe(5); + expect(levenshtein("fghij", "abcde")).toBe(5); +}); + +test("levenshtein distance between strings with one edit is 1", () => { + expect(levenshtein("hello", "hallo")).toBe(1); + expect(levenshtein("hallo", "hello")).toBe(1); +}); + +test("levenshtein distance between otherwise identical strings with length difference is their length difference", () => { + expect(levenshtein("hello", "hello world")).toBe(6); + expect(levenshtein("hello world", "hello")).toBe(6); +}); + +test("levenshtein distance between strings with multiple edits is the sum of the edits", () => { + expect(levenshtein("hello", "hallu")).toBe(2); + expect(levenshtein("hallu", "hello")).toBe(2); +}); \ No newline at end of file diff --git a/simulator-docker-runner/levenshtein.ts b/simulator-docker-runner/levenshtein.ts new file mode 100644 index 000000000..e2a199850 --- /dev/null +++ b/simulator-docker-runner/levenshtein.ts @@ -0,0 +1,40 @@ +/** + * Calculate the Levenshtein distance between two strings. + * @param a - The first string. + * @param b - The second string. + * @returns The Levenshtein distance between the two strings. + */ +export function levenshtein(a: string, b: string): number { + const an = a ? a.length : 0; + const bn = b ? b.length : 0; + if (an === 0) { + return bn; + } + if (bn === 0) { + return an; + } + const matrix = new Array(bn + 1); + for (let i = 0; i <= bn; ++i) { + let row = matrix[i] = new Array(an + 1); + row[0] = i; + } + const firstRow = matrix[0]; + for (let j = 1; j <= an; ++j) { + firstRow[j] = j; + } + for (let i = 1; i <= bn; ++i) { + for (let j = 1; j <= an; ++j) { + if (b.charAt(i - 1) === a.charAt(j - 1)) { + matrix[i][j] = matrix[i - 1][j - 1]; + } + else { + matrix[i][j] = Math.min( + matrix[i - 1][j - 1], // substitution + matrix[i][j - 1], // insertion + matrix[i - 1][j] // deletion + ) + 1; + } + } + } + return matrix[bn][an]; +}; \ No newline at end of file diff --git a/simulator-docker-runner/logParse.ts b/simulator-docker-runner/logParse.ts new file mode 100644 index 000000000..e454a51f1 --- /dev/null +++ b/simulator-docker-runner/logParse.ts @@ -0,0 +1,58 @@ +// Helper functions + +export type StackTraceInfo = { + type: "panic"; + trace: string; + mainError: string; +} + +export type AssertionFailureInfo = { + type: "assertion"; + output: string; +} + +/** + * Extract failure information from log output + */ +export function extractFailureInfo(output: string): StackTraceInfo | AssertionFailureInfo { + const lines = output.split('\n'); + + const panicLineIndex = lines.findIndex(line => line.includes("panic occurred")); + + const info = getTraceFromOutput(lines) ?? getAssertionFailureInfo(lines); + + if (!info) { + throw new Error("No failure information found"); + } + + return info; +} + +function getTraceFromOutput(lines: string[]): StackTraceInfo | null { + const panicLineIndex = lines.findIndex(line => line.includes("panic occurred")); + if (panicLineIndex === -1) { + return null; + } + + const startIndex = panicLineIndex + 1; + const endIndex = Math.min(lines.length, startIndex + 50); + + const trace = lines.slice(startIndex, endIndex).join('\n'); + const mainError = lines[startIndex] ?? "???"; + + return { type: "panic", trace, mainError }; +} + +function getAssertionFailureInfo(lines: string[]): AssertionFailureInfo | null { + const simulationFailedLineIndex = lines.findIndex(line => line.includes("simulation failed:")); + if (simulationFailedLineIndex === -1) { + return null; + } + + const startIndex = simulationFailedLineIndex; + const endIndex = Math.min(lines.length, startIndex + 1); + + const output = lines.slice(startIndex, endIndex).join('\n'); + + return { type: "assertion", output }; +} \ No newline at end of file diff --git a/simulator-docker-runner/package.json b/simulator-docker-runner/package.json new file mode 100644 index 000000000..46f953b61 --- /dev/null +++ b/simulator-docker-runner/package.json @@ -0,0 +1,14 @@ +{ + "name": "simulator-docker-runner", + "description": "A script to run the simulator in a docker container in a loop and post issues to github when it panics", + "module": "docker-entrypoint.simulator.ts", + "type": "module", + "dependencies": { + "octokit": "4.1.2" + }, + "devDependencies": { + "@types/node": "22.13.5", + "bun-types": "1.2.4", + "typescript": "5.7.3" + } +} \ No newline at end of file diff --git a/simulator-docker-runner/random.ts b/simulator-docker-runner/random.ts new file mode 100644 index 000000000..6a3da25f7 --- /dev/null +++ b/simulator-docker-runner/random.ts @@ -0,0 +1,5 @@ +export function randomSeed() { + const high32 = Math.floor(Math.random() * Math.pow(2, 32)); + const low32 = Math.floor(Math.random() * Math.pow(2, 32)); + return ((BigInt(high32) << 32n) | BigInt(low32)).toString(); +} \ No newline at end of file diff --git a/simulator-docker-runner/tsconfig.json b/simulator-docker-runner/tsconfig.json new file mode 100644 index 000000000..cf7063cf2 --- /dev/null +++ b/simulator-docker-runner/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "lib": ["ESNext", "DOM"], + "module": "esnext", + "target": "esnext", + "moduleResolution": "bundler", + "moduleDetection": "force", + "allowImportingTsExtensions": true, + "noEmit": true, + "composite": true, + "strict": true, + "downlevelIteration": true, + "skipLibCheck": true, + "jsx": "preserve", + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "allowJs": true, + "types": [ + "bun-types", + "node" + ] + } +} \ No newline at end of file