Update benchmarking

scripts/benchmark.py

@@ -13,36 +13,55 @@ from typer import run
 def main(
     n_benchmarks: int | None = None,
 ):
-    processes = []
-    files = []
     path = Path("benchmark")
 
+    folders = path.iterdir()
+
     if n_benchmarks:
-        benchmarks = islice(path.iterdir(), n_benchmarks)
+        folders = islice(folders, n_benchmarks)
 
-    for folder in benchmarks:
-        if os.path.isdir(folder):
-            print("Running benchmark for {}".format(folder))
+    benchmarks = []
+    for bench_folder in folders:
+        if os.path.isdir(bench_folder):
+            print("Running benchmark for {}".format(bench_folder))
 
-            log_path = folder / "log.txt"
+            log_path = bench_folder / "log.txt"
             log_file = open(log_path, "w")
-            processes.append(
-                subprocess.Popen(
-                    ["python", "-m", "gpt_engineer.main", folder],
-                    stdout=log_file,
-                    stderr=log_file,
-                    bufsize=0,
-                )
+            process = subprocess.Popen(
+                [
+                    "python",
+                    "-u", # Unbuffered output
+                    "-m",
+                    "gpt_engineer.main",
+                    bench_folder,
+                    "--steps-config",
+                    "benchmark",
+                ],
+                stdout=log_file,
+                stderr=log_file,
+                bufsize=0,
             )
-            files.append(log_file)
+            benchmarks.append((bench_folder, process, log_file))
 
             print("You can stream the log file by running: tail -f {}".format(log_path))
 
-    for process, file in zip(processes, files):
+    for bench_folder, process, file in benchmarks:
         process.wait()
-        print("process finished with code", process.returncode)
         file.close()
 
+        print("process", bench_folder.name, "finished with code", process.returncode)
+        print('Running it. Original benchmark prompt:')
+        print()
+        with open(bench_folder / "main_prompt") as f:
+            print(f.read())
+        print()
+
+        try:
+            subprocess.run(
+                ['python', "-m", "gpt_engineer.main", bench_folder, "--steps-config", "execute_only"],
+            )
+        except KeyboardInterrupt:
+            pass
 
 if __name__ == "__main__":
     run(main)