aljaz/gpt-engineer
1
0
Fork 0
mirror of https://github.com/aljazceru/gpt-engineer.git synced 2025-12-17 20:55:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update benchmarking

Browse Source
This commit is contained in:
Anton Osika
2023-06-18 15:16:20 +02:00
parent a038117efa
commit 2e4baf7fe1
2 changed files with 38 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gpt_engineer/steps.py
View File

@@ -142,9 +142,9 @@ def execute_entrypoint(ai, dbs):
print() print()
print(command) print(command)
print() print()
print('If yes, type "yes". If no, press enter.') print('If yes, press enter. Otherwise, type "no"')
print() print()
if input().lower() != "yes": if input() != "":
print("Ok, not executing the code.") print("Ok, not executing the code.")
return [] return []
print("Executing the code...") print("Executing the code...")

47
scripts/benchmark.py
View File

@@ -13,36 +13,55 @@ from typer import run
def main( def main(
n_benchmarks: int | None = None, n_benchmarks: int | None = None,
): ):
processes = []
files = []
path = Path("benchmark") path = Path("benchmark")
folders = path.iterdir()
if n_benchmarks: if n_benchmarks:
benchmarks = islice(path.iterdir(), n_benchmarks) folders = islice(folders, n_benchmarks)
for folder in benchmarks: benchmarks = []
if os.path.isdir(folder): for bench_folder in folders:
print("Running benchmark for {}".format(folder)) if os.path.isdir(bench_folder):
print("Running benchmark for {}".format(bench_folder))
log_path = folder / "log.txt" log_path = bench_folder / "log.txt"
log_file = open(log_path, "w") log_file = open(log_path, "w")
processes.append( process = subprocess.Popen(
subprocess.Popen( [
["python", "-m", "gpt_engineer.main", folder], "python",
"-u", # Unbuffered output
"-m",
"gpt_engineer.main",
bench_folder,
"--steps-config",
"benchmark",
],
stdout=log_file, stdout=log_file,
stderr=log_file, stderr=log_file,
bufsize=0, bufsize=0,
) )
) benchmarks.append((bench_folder, process, log_file))
files.append(log_file)
print("You can stream the log file by running: tail -f {}".format(log_path)) print("You can stream the log file by running: tail -f {}".format(log_path))
for process, file in zip(processes, files): for bench_folder, process, file in benchmarks:
process.wait() process.wait()
print("process finished with code", process.returncode)
file.close() file.close()
print("process", bench_folder.name, "finished with code", process.returncode)
print('Running it. Original benchmark prompt:')
print()
with open(bench_folder / "main_prompt") as f:
print(f.read())
print()
try:
subprocess.run(
['python', "-m", "gpt_engineer.main", bench_folder, "--steps-config", "execute_only"],
)
except KeyboardInterrupt:
pass
if __name__ == "__main__": if __name__ == "__main__":
run(main) run(main)