aljaz/gpt-engineer
1
0
Fork 0
mirror of https://github.com/aljazceru/gpt-engineer.git synced 2025-12-17 12:45:26 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update benchmarking

Browse Source
This commit is contained in:
Anton Osika
2023-06-18 15:02:42 +02:00
parent c999f7c2c8
commit 4a212d968d
2 changed files with 43 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
gpt_engineer/steps.py
View File

@@ -143,9 +143,9 @@ def execute_entrypoint(ai, dbs):
print() print()
print(command) print(command)
print() print()
print('If yes, type "yes". If no, press enter.') print('If yes, press enter. Otherwise, type "no"')
print() print()
if input().lower() != "yes": if input() != "":
print("Ok, not executing the code.") print("Ok, not executing the code.")
return [] return []
print("Executing the code...") print("Executing the code...")

58
scripts/benchmark.py
View File

@@ -16,31 +16,55 @@ from itertools import islice
def main( def main(
n_benchmarks: int | None = None, n_benchmarks: int | None = None,
): ):
processes = [] path = Path("benchmark")
files = []
path = Path('benchmark') folders = path.iterdir()
if n_benchmarks: if n_benchmarks:
benchmarks = islice(path.iterdir(), n_benchmarks) folders = islice(folders, n_benchmarks)
for folder in benchmarks: benchmarks = []
if os.path.isdir(folder): for bench_folder in folders:
print('Running benchmark for {}'.format(folder)) if os.path.isdir(bench_folder):
print("Running benchmark for {}".format(bench_folder))
log_path = folder / 'log.txt' log_path = bench_folder / "log.txt"
log_file = open(log_path, 'w') log_file = open(log_path, "w")
processes.append(subprocess.Popen(['python', '-m', 'gpt_engineer.main', folder], stdout=log_file, stderr=log_file, bufsize=0)) process = subprocess.Popen(
files.append(log_file) [
"python",
"-u", # Unbuffered output
"-m",
"gpt_engineer.main",
bench_folder,
"--steps-config",
"benchmark",
],
stdout=log_file,
stderr=log_file,
bufsize=0,
)
benchmarks.append((bench_folder, process, log_file))
print('You can stream the log file by running: tail -f {}'.format(log_path)) print("You can stream the log file by running: tail -f {}".format(log_path))
for process, file in zip(processes, files): for bench_folder, process, file in benchmarks:
process.wait() process.wait()
print('process finished with code', process.returncode)
file.close() file.close()
print("process", bench_folder.name, "finished with code", process.returncode)
print('Running it. Original benchmark prompt:')
print()
with open(bench_folder / "main_prompt") as f:
print(f.read())
print()
if __name__ == '__main__': try:
subprocess.run(
['python', "-m", "gpt_engineer.main", bench_folder, "--steps-config", "execute_only"],
)
except KeyboardInterrupt:
pass
if __name__ == "__main__":
run(main) run(main)