Merge 'sim: post summary to slack' from Jussi Saurio

Closes #2016

simulator-docker-runner/docker-entrypoint.simulator.ts

@@ -2,6 +2,7 @@
 
 import { spawn } from "bun";
 import { GithubClient } from "./github";
+import { SlackClient } from "./slack";
 import { extractFailureInfo } from "./logParse";
 import { randomSeed } from "./random";
 
@@ -12,12 +13,14 @@ const PER_RUN_TIMEOUT_SECONDS = Number.isInteger(Number(process.env.PER_RUN_TIME
 const LOG_TO_STDOUT = process.env.LOG_TO_STDOUT === "true";
 
 const github = new GithubClient();
+const slack = new SlackClient();
 
 process.env.RUST_BACKTRACE = "1";
 
 console.log("Starting limbo_sim in a loop...");
 console.log(`Git hash: ${github.GIT_HASH}`);
 console.log(`GitHub issues enabled: ${github.mode === 'real'}`);
+console.log(`Slack notifications enabled: ${slack.mode === 'real'}`);
 console.log(`Time limit: ${TIME_LIMIT_MINUTES} minutes`);
 console.log(`Log simulator output to stdout: ${LOG_TO_STDOUT}`);
 console.log(`Sleep between runs: ${SLEEP_BETWEEN_RUNS_SECONDS} seconds`);
@@ -69,7 +72,7 @@ const timeouter = (seconds: number, runNumber: number) => {
   return timeouterPromise;
 }
 
-const run = async (seed: string, bin: string, args: string[]) => {
+const run = async (seed: string, bin: string, args: string[]): Promise<boolean> => {
   const proc = spawn([`/app/${bin}`, ...args], {
     stdout: LOG_TO_STDOUT ? "inherit" : "pipe",
     stderr: LOG_TO_STDOUT ? "inherit" : "pipe",
@@ -77,6 +80,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
   });
 
   const timeout = timeouter(PER_RUN_TIMEOUT_SECONDS, runNumber);
+  let issuePosted = false;
 
   try {
     const exitCode = await Promise.race([proc.exited, timeout]);
@@ -102,6 +106,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
             command: args.join(" "),
             stackTrace: failureInfo,
           });
+          issuePosted = true;
         } else {
           await github.postGitHubIssue({
             type: "assertion",
@@ -109,6 +114,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
             command: args.join(" "),
             failureInfo,
           });
+          issuePosted = true;
         }
       } catch (err2) {
         console.error(`Error extracting simulator seed and stack trace: ${err2}`);
@@ -134,6 +140,7 @@ const run = async (seed: string, bin: string, args: string[]) => {
         command: args.join(" "),
         output: lastLines,
       });
+      issuePosted = true;
     } else {
       throw err;
     }
@@ -141,12 +148,16 @@ const run = async (seed: string, bin: string, args: string[]) => {
     // @ts-ignore
     timeout.clear();
   }
+  
+  return issuePosted;
 }
 
 // Main execution loop
 const startTime = new Date();
 const limboSimArgs = process.argv.slice(2);
 let runNumber = 0;
+let totalIssuesPosted = 0;
+
 while (new Date().getTime() - startTime.getTime() < TIME_LIMIT_MINUTES * 60 * 1000) {
   const timestamp = new Date().toISOString();
   const args = [...limboSimArgs];
@@ -160,13 +171,29 @@ while (new Date().getTime() - startTime.getTime() < TIME_LIMIT_MINUTES * 60 * 10
   args.push(...loop);
 
   console.log(`[${timestamp}]: Running "limbo_sim ${args.join(" ")}" - (seed ${seed}, run number ${runNumber})`);
-  await run(seed, "limbo_sim", args);
+  const issuePosted = await run(seed, "limbo_sim", args);
+  
+  if (issuePosted) {
+    totalIssuesPosted++;
+  }
 
   runNumber++;
 
   SLEEP_BETWEEN_RUNS_SECONDS > 0 && (await sleep(SLEEP_BETWEEN_RUNS_SECONDS));
 }
 
+// Post summary to Slack after the run completes
+const endTime = new Date();
+const timeElapsed = Math.floor((endTime.getTime() - startTime.getTime()) / 1000);
+console.log(`\nRun completed! Total runs: ${runNumber}, Issues posted: ${totalIssuesPosted}, Time elapsed: ${timeElapsed}s`);
+
+await slack.postRunSummary({
+  totalRuns: runNumber,
+  issuesPosted: totalIssuesPosted,
+  timeElapsed,
+  gitHash: github.GIT_HASH,
+});
+
 async function sleep(sec: number) {
   return new Promise(resolve => setTimeout(resolve, sec * 1000));
 }

simulator-docker-runner/slack.ts

@@ -0,0 +1,154 @@
+export class SlackClient {
+  private botToken: string;
+  private channel: string;
+  mode: 'real' | 'dry-run';
+  
+  constructor() {
+    this.botToken = process.env.SLACK_BOT_TOKEN || "";
+    this.channel = process.env.SLACK_CHANNEL || "#simulator-results-fake";
+    this.mode = this.botToken ? 'real' : 'dry-run';
+
+    if (this.mode === 'real') {
+      if (this.channel === "#simulator-results-fake") {
+        throw new Error("SLACK_CHANNEL must be set to a real channel when running in real mode");
+      }
+    } else {
+      if (this.channel !== "#simulator-results-fake") {
+        throw new Error("SLACK_CHANNEL must be set to #simulator-results-fake when running in dry-run mode");
+      }
+    }
+  }
+
+  async postRunSummary(stats: {
+    totalRuns: number;
+    issuesPosted: number;
+    timeElapsed: number;
+    gitHash: string;
+  }): Promise<void> {
+    const blocks = this.createSummaryBlocks(stats);
+    const fallbackText = this.createFallbackText(stats);
+    
+    if (this.mode === 'dry-run') {
+      console.log(`Dry-run mode: Would post to Slack channel ${this.channel}`);
+      console.log(`Fallback text: ${fallbackText}`);
+      console.log(`Blocks: ${JSON.stringify(blocks, null, 2)}`);
+      return;
+    }
+
+    try {
+      const response = await fetch('https://slack.com/api/chat.postMessage', {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.botToken}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          channel: this.channel,
+          text: fallbackText,
+          blocks: blocks,
+        }),
+      });
+
+      const result = await response.json();
+      
+      if (!result.ok) {
+        console.error(`Failed to post to Slack: ${result.error}`);
+        return;
+      }
+      
+      console.log(`Successfully posted summary to Slack channel ${this.channel}`);
+    } catch (error) {
+      console.error(`Error posting to Slack: ${error}`);
+    }
+  }
+
+  private createFallbackText(stats: {
+    totalRuns: number;
+    issuesPosted: number;
+    timeElapsed: number;
+    gitHash: string;
+  }): string {
+    const { totalRuns, issuesPosted, timeElapsed, gitHash } = stats;
+    const hours = Math.floor(timeElapsed / 3600);
+    const minutes = Math.floor((timeElapsed % 3600) / 60);
+    const seconds = Math.floor(timeElapsed % 60);
+    const timeString = `${hours}h ${minutes}m ${seconds}s`;
+    const gitShortHash = gitHash.substring(0, 7);
+    
+    return `🤖 Turso Simulator Run Complete - ${totalRuns} runs, ${issuesPosted} issues posted, ${timeString} elapsed (${gitShortHash})`;
+  }
+
+  private createSummaryBlocks(stats: {
+    totalRuns: number;
+    issuesPosted: number;
+    timeElapsed: number;
+    gitHash: string;
+  }): any[] {
+    const { totalRuns, issuesPosted, timeElapsed, gitHash } = stats;
+    const hours = Math.floor(timeElapsed / 3600);
+    const minutes = Math.floor((timeElapsed % 3600) / 60);
+    const seconds = Math.floor(timeElapsed % 60);
+    const timeString = `${hours}h ${minutes}m ${seconds}s`;
+    
+    const statusEmoji = issuesPosted > 0 ? "🔴" : "✅";
+    const statusText = issuesPosted > 0 ? `${issuesPosted} issues found` : "No issues found";
+    const gitShortHash = gitHash.substring(0, 7);
+    
+    return [
+      {
+        "type": "header",
+        "text": {
+          "type": "plain_text",
+          "text": "🤖 Turso Simulator Run Complete"
+        }
+      },
+      {
+        "type": "section",
+        "text": {
+          "type": "mrkdwn",
+          "text": `${statusEmoji} *${statusText}*`
+        }
+      },
+      {
+        "type": "divider"
+      },
+      {
+        "type": "section",
+        "fields": [
+          {
+            "type": "mrkdwn",
+            "text": `*Total runs:*\n${totalRuns}`
+          },
+          {
+            "type": "mrkdwn",
+            "text": `*Issues posted:*\n${issuesPosted}`
+          },
+          {
+            "type": "mrkdwn",
+            "text": `*Time elapsed:*\n${timeString}`
+          },
+          {
+            "type": "mrkdwn",
+            "text": `*Git hash:*\n\`${gitShortHash}\``
+          },
+          {
+            "type": "mrkdwn",
+            "text": `*See open issues:*\n<https://github.com/tursodatabase/turso/issues?q=is%3Aissue%20state%3Aopen%20simulator%20author%3Aapp%2Fturso-github-handyman|Open issues>`
+          }
+        ]
+      },
+      {
+        "type": "divider"
+      },
+      {
+        "type": "context",
+        "elements": [
+          {
+            "type": "mrkdwn",
+            "text": `Full git hash: \`${gitHash}\` | Timestamp: ${new Date().toISOString()}`
+          }
+        ]
+      }
+    ];
+  }
+}