From 60f64aed771f6f0e5c21edc3d8131bcbf87c40ad Mon Sep 17 00:00:00 2001
From: Jussi Saurio <jussi.saurio@gmail.com>
Date: Thu, 24 Jul 2025 14:51:51 +0300
Subject: [PATCH] sim/aws: ignore child process exits with code 137

i suspect the child process is being killed due to memory consumption
or something, but hard to be sure. either way, let's not stop the sim-loop
parent process when this happens.
---
 .../docker-entrypoint.simulator.ts                |  8 +++++++-
 simulator-docker-runner/slack.ts                  | 15 +++++++++++----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/simulator-docker-runner/docker-entrypoint.simulator.ts b/simulator-docker-runner/docker-entrypoint.simulator.ts
index 6c977c221..9d59c8e8b 100644
--- a/simulator-docker-runner/docker-entrypoint.simulator.ts
+++ b/simulator-docker-runner/docker-entrypoint.simulator.ts
@@ -72,6 +72,8 @@ const timeouter = (seconds: number, runNumber: number) => {
   return timeouterPromise;
 }
 
+let unexpectedExits = 0;
+
 const run = async (seed: string, bin: string, args: string[]): Promise<boolean> => {
   const proc = spawn([`/app/${bin}`, ...args], {
     stdout: LOG_TO_STDOUT ? "inherit" : "pipe",
@@ -87,7 +89,7 @@ const run = async (seed: string, bin: string, args: string[]): Promise<boolean>
     const stdout = await new Response(proc.stdout).text();
     const stderr = await new Response(proc.stderr).text();
 
-    if (exitCode !== 0) {
+    if (exitCode !== 0 && exitCode !== 137) {
       console.log(`[${new Date().toISOString()}]: ${bin} ${args.join(" ")} exited with code ${exitCode}`);
       const output = stdout + stderr;
 
@@ -123,6 +125,9 @@ const run = async (seed: string, bin: string, args: string[]): Promise<boolean>
         console.log(`Simulator seed: ${seed}`);
         process.exit(1);
       }
+    } else if (exitCode === 137) {
+      console.error("Child process exited due to sigkill, ignoring...");
+      unexpectedExits++;
     }
   } catch (err) {
     if (err instanceof TimeoutError) {
@@ -190,6 +195,7 @@ console.log(`\nRun completed! Total runs: ${runNumber}, Issues posted: ${totalIs
 await slack.postRunSummary({
   totalRuns: runNumber,
   issuesPosted: totalIssuesPosted,
+  unexpectedExits,
   timeElapsed,
   gitHash: github.GIT_HASH,
 });
diff --git a/simulator-docker-runner/slack.ts b/simulator-docker-runner/slack.ts
index 2e3356d28..8b6bbf6e1 100644
--- a/simulator-docker-runner/slack.ts
+++ b/simulator-docker-runner/slack.ts
@@ -22,6 +22,7 @@ export class SlackClient {
   async postRunSummary(stats: {
     totalRuns: number;
     issuesPosted: number;
+    unexpectedExits: number;
     timeElapsed: number;
     gitHash: string;
   }): Promise<void> {
@@ -65,32 +66,34 @@ export class SlackClient {
   private createFallbackText(stats: {
     totalRuns: number;
     issuesPosted: number;
+    unexpectedExits: number;
     timeElapsed: number;
     gitHash: string;
   }): string {
-    const { totalRuns, issuesPosted, timeElapsed, gitHash } = stats;
+    const { totalRuns, issuesPosted, unexpectedExits, timeElapsed, gitHash } = stats;
     const hours = Math.floor(timeElapsed / 3600);
     const minutes = Math.floor((timeElapsed % 3600) / 60);
     const seconds = Math.floor(timeElapsed % 60);
     const timeString = `${hours}h ${minutes}m ${seconds}s`;
     const gitShortHash = gitHash.substring(0, 7);
     
-    return `🤖 Turso Simulator Run Complete - ${totalRuns} runs, ${issuesPosted} issues posted, ${timeString} elapsed (${gitShortHash})`;
+    return `🤖 Turso Simulator Run Complete - ${totalRuns} runs, ${issuesPosted} issues posted, ${unexpectedExits} unexpected exits, ${timeString} elapsed (${gitShortHash})`;
   }
 
   private createSummaryBlocks(stats: {
     totalRuns: number;
     issuesPosted: number;
+    unexpectedExits: number;
     timeElapsed: number;
     gitHash: string;
   }): any[] {
-    const { totalRuns, issuesPosted, timeElapsed, gitHash } = stats;
+    const { totalRuns, issuesPosted, unexpectedExits, timeElapsed, gitHash } = stats;
     const hours = Math.floor(timeElapsed / 3600);
     const minutes = Math.floor((timeElapsed % 3600) / 60);
     const seconds = Math.floor(timeElapsed % 60);
     const timeString = `${hours}h ${minutes}m ${seconds}s`;
     
-    const statusEmoji = issuesPosted > 0 ? "🔴" : "✅";
+    const statusEmoji = issuesPosted > 0 || unexpectedExits > 0 ? "🔴" : "✅";
     const statusText = issuesPosted > 0 ? `${issuesPosted} issues found` : "No issues found";
     const gitShortHash = gitHash.substring(0, 7);
     
@@ -123,6 +126,10 @@ export class SlackClient {
             "type": "mrkdwn",
             "text": `*Issues posted:*\n${issuesPosted}`
           },
+          {
+            "type": "mrkdwn",
+            "text": `*Unexpected exits:*\n${unexpectedExits}`
+          },
           {
             "type": "mrkdwn",
             "text": `*Time elapsed:*\n${timeString}`