From b97563726f1c0507ed723da7378928f2be6c7dd2 Mon Sep 17 00:00:00 2001 From: Enzo Martin Date: Sun, 18 Jun 2023 10:05:45 +0200 Subject: [PATCH 1/6] Add cleanup & move `projects` to their own directory - Add optional argument to clean and delete the working directories of the project before running the prompt - Add `.gitignore` entry to ignore all possible projects - Update readme --- .gitignore | 3 +++ README.md | 5 +++-- gpt_engineer/main.py | 17 ++++++++++++----- {example => projects/example}/main_prompt | 0 4 files changed, 18 insertions(+), 7 deletions(-) rename {example => projects/example}/main_prompt (100%) diff --git a/.gitignore b/.gitignore index 1c41f42..5a39df8 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ archive # any log file *log.txt + +# ignore all project files +projects \ No newline at end of file diff --git a/README.md b/README.md index 40899bb..6fff452 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,13 @@ GPT Engineer is made to be easy to adapt, extend, and make your agent learn how - `export OPENAI_API_KEY=[your api key]` with a key that has GPT4 access **Run**: -- Create a new empty folder with a `main_prompt` file (or copy the example folder `cp -r example/ my-new-project`) +- Create a new empty folder with a `main_prompt` file in the `projects` folder (or copy the example folder `cp -r projects/example/ projects/my-new-project`) - Fill in the `main_prompt` in your new folder - Run `python -m gpt_engineer.main my-new-project` + - Optionally pass in `true` to delete the working files before running **Results**: -- Check the generated files in my-new-project/workspace +- Check the generated files in projects/my-new-project/workspace ### Limitations Implementing additional chain of thought prompting, e.g. [Reflexion](https://github.com/noahshinn024/reflexion), should be able to make it more reliable and not miss requested functionality in the main prompt. diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index c03ed7f..1aeabc2 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -2,8 +2,9 @@ import os import json import pathlib import typer +import shutil + -from gpt_engineer.chat_to_files import to_files from gpt_engineer.ai import AI from gpt_engineer.steps import STEPS from gpt_engineer.db import DB, DBs @@ -14,7 +15,8 @@ app = typer.Typer() @app.command() def chat( - project_path: str = typer.Argument(str(pathlib.Path(os.path.curdir) / "example"), help="path"), + project_path: str = typer.Argument("example", help="path"), + delete_existing: str = typer.Argument(None, help="delete existing files"), run_prefix: str = typer.Option( "", help="run prefix, if you want to run multiple variants of the same project and later compare them", @@ -24,9 +26,14 @@ def chat( steps_config: str = "default", ): app_dir = pathlib.Path(os.path.curdir) - input_path = project_path - memory_path = pathlib.Path(project_path) / (run_prefix + "memory") - workspace_path = pathlib.Path(project_path) / (run_prefix + "workspace") + input_path = pathlib.Path(app_dir / "projects" / project_path) + memory_path = input_path / (run_prefix + "memory") + workspace_path = input_path / (run_prefix + "workspace") + + if delete_existing == 'true': + # Delete files and subdirectories in paths + shutil.rmtree(memory_path, ignore_errors=True) + shutil.rmtree(workspace_path, ignore_errors=True) ai = AI( model=model, diff --git a/example/main_prompt b/projects/example/main_prompt similarity index 100% rename from example/main_prompt rename to projects/example/main_prompt From 50c505c4594d6a2084f13487743f672a16ad29e3 Mon Sep 17 00:00:00 2001 From: Enzo Martin Date: Sun, 18 Jun 2023 10:05:59 +0200 Subject: [PATCH 2/6] Add support for directory creation and binary files - Use the `Path` module instead of `os` - Add ability to create any amount of missing directories for a given file - Add ability to save both text and binary files to save images (or other file types) later --- gpt_engineer/db.py | 47 +++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/gpt_engineer/db.py b/gpt_engineer/db.py index 01fa929..6088226 100644 --- a/gpt_engineer/db.py +++ b/gpt_engineer/db.py @@ -1,35 +1,52 @@ from dataclasses import dataclass -import os from pathlib import Path - +# This class represents a simple database that stores its data as files in a directory. +# It supports both text and binary files, and can handle directory structures. class DB: - """A simple key-value store, where keys are filenames and values are file contents.""" - def __init__(self, path): + # Convert the path string to a Path object and get its absolute path. self.path = Path(path).absolute() - os.makedirs(self.path, exist_ok=True) + + # Create the directory if it doesn't exist. + self.path.mkdir(parents=True, exist_ok=True) def __getitem__(self, key): - with open(self.path / key, encoding='utf-8') as f: - return f.read() + # Combine the database directory with the provided file path. + full_path = self.path / key + + # Check if the file exists before trying to open it. + if full_path.is_file(): + # Open the file in text mode and return its content. + with full_path.open('r') as f: + return f.read() + else: + # If the file doesn't exist, raise an error. + raise FileNotFoundError(f"No such file: '{full_path}'") def __setitem__(self, key, val): - Path(self.path / key).absolute().parent.mkdir(parents=True, exist_ok=True) + # Combine the database directory with the provided file path. + full_path = self.path / key - with open(self.path / key, 'w', encoding='utf-8') as f: - f.write(val) + # Create the directory tree if it doesn't exist. + full_path.parent.mkdir(parents=True, exist_ok=True) - def __contains__(self, key): - return (self.path / key).exists() + # Write the data to the file. If val is a string, it's written as text. + # If val is bytes, it's written as binary data. + if isinstance(val, str): + full_path.write_text(val) + elif isinstance(val, bytes): + full_path.write_bytes(val) + else: + # If val is neither a string nor bytes, raise an error. + raise TypeError("val must be either a str or bytes") +# dataclass for all dbs: @dataclass class DBs: - """A dataclass for all dbs""" - memory: DB logs: DB identity: DB input: DB - workspace: DB + workspace: DB \ No newline at end of file From 9c24119f915d2d1d5709faa2fd0da90b0d015e88 Mon Sep 17 00:00:00 2001 From: Enzo Martin Date: Sun, 18 Jun 2023 10:06:17 +0200 Subject: [PATCH 3/6] Generate instructions for all platforms - Update prompt to create instructions for all 3 major OS platforms - Fix small typo --- gpt_engineer/steps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpt_engineer/steps.py b/gpt_engineer/steps.py index 60fea23..7f18dee 100644 --- a/gpt_engineer/steps.py +++ b/gpt_engineer/steps.py @@ -97,8 +97,8 @@ def run_clarified(ai: AI, dbs: DBs): def execute_workspace(ai: AI, dbs: DBs): messages = ai.start( system=( - f"You will get infomation about a codebase that is currently on disk in the folder {dbs.workspace.path}.\n" - "From this you will answer with one code block that includes all the necessary macos terminal commands to " + f"You will get information about a codebase that is currently on disk in the folder {dbs.workspace.path}.\n" + "From this you will answer with code blocks that includes all the necessary Windows, MacOS, and Linux terminal commands to " "a) install dependencies " "b) run the necessary parts of the codebase to try it.\n" "Do not explain the code, just give the commands.\n" From e29e7bec2fd8a441529d401379d02abf5bd8a825 Mon Sep 17 00:00:00 2001 From: Enzo Martin Date: Sun, 18 Jun 2023 10:06:28 +0200 Subject: [PATCH 4/6] Enhance philosophy to include supporting documents - Create instructions for running/compiling the project - Create any package manager files --- identity/philosophy | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/identity/philosophy b/identity/philosophy index 96a12ad..34299f8 100644 --- a/identity/philosophy +++ b/identity/philosophy @@ -1,4 +1,11 @@ -You almost always put different classes in different files +You almost always put different classes in different files. +You always add a comment briefly describing the purpose of the function definition. +You try to add comments explaining very complex bits of logic. +You always follow the best practices for the requested languages in terms of describing the code written as a defined package/project. + +For Python, you always create an appropriate requirements.txt file. +For NodeJS, you always create an appropriate package.json file. +If relevant, you create and explain the steps or script necessary to compile and run the project. Python toolbelt preferences: - pytest From e7df947b9891304a7a625ba9d70ca9db0eb58341 Mon Sep 17 00:00:00 2001 From: Enzo Martin Date: Sun, 18 Jun 2023 10:06:36 +0200 Subject: [PATCH 5/6] Add support for directory paths in filenames and improve code splitting - Enforce an explicit markdown code block format - Add a token to split the output to clearly detect when the code blocks start - Save all non-code output to a `README.md` file - Update RegEx to extract and strip text more reliably and clean up the output - Update the identify prompts appropriately --- gpt_engineer/chat_to_files.py | 35 ++++++++++++++++++++++++----------- identity/setup | 21 ++++++++++++--------- identity/use_qa | 21 ++++++++++++--------- 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 9c8a9e6..2175bb3 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,28 +1,41 @@ import re -from typing import List, Tuple -from gpt_engineer.db import DB +def parse_chat(chat):# -> List[Tuple[str, str]]: + # Split the chat into sections by the '*CODEBLOCKSBELOW*' token + split_chat = chat.split('*CODEBLOCKSBELOW*') -def parse_chat(chat) -> List[Tuple[str, str]]: - # Get all ``` blocks - regex = r"```(.*?)```" + # Check if the '*CODEBLOCKSBELOW*' token was found + is_token_found = len(split_chat) > 1 - matches = re.finditer(regex, chat, re.DOTALL) + # If the '*CODEBLOCKSBELOW*' token is found, use the first part as README and second part as code blocks. + # Otherwise, treat README as optional and proceed with empty README and the entire chat as code blocks + readme = split_chat[0].strip() if is_token_found else 'No readme' + code_blocks = split_chat[1] if is_token_found else chat + + # Get all ``` blocks and preceding filenames + regex = r"\[(.*?)\]\s*```.*?\n(.*?)```" + matches = re.finditer(regex, code_blocks, re.DOTALL) files = [] for match in matches: - path = match.group(1).split("\n")[0] + # Strip the filename of any non-allowed characters and convert / to \ + path = re.sub(r'[<>"|?*]', '', match.group(1)) + # Get the code - code = match.group(1).split("\n")[1:] - code = "\n".join(code) + code = match.group(2) + # Add the file to the list files.append((path, code)) + # Add README to the list + files.append(('README.txt', readme)) + + # Return the files return files -def to_files(chat: str, workspace: DB): - workspace["all_output.txt"] = chat +def to_files(chat, workspace): + workspace['all_output.txt'] = chat files = parse_chat(chat) for file_name, file_content in files: diff --git a/identity/setup b/identity/setup index 4917945..4e3f2bc 100644 --- a/identity/setup +++ b/identity/setup @@ -1,15 +1,18 @@ You will get instructions for code to write. -You will write a very long answer. Make sure that every detail of the architecture is, in the end, implemented as code. +Following best practices and formatting for a README.md file, you will write a very long answer, make sure to provide the instructions on how to run the code. +Make sure that every detail of the architecture is, in the end, implemented as code. You will first lay out the names of the core classes, functions, methods that will be necessary, As well as a quick comment on their purpose. -Then you will output the content of each file, with syntax below. -(You will start with the "entrypoint" file, then go to the ones that are imported by that file, and so on.) +Before you start outputting the code, you will output a seperator in the form of a line containing "*CODEBLOCKSBELOW*" +Make sure to create any appropriate module dependency or package manager dependency definition file. +Then you will reformat and output the content of each file strictly following a markdown code block format, where the following tokens should be replaced such that [FILENAME] is the lowercase file name including the file extension, [LANG] is the markup code block language for the code's language, and [CODE] is the comments and code: +[FILENAME] +```[LANG] +[CODE] +``` + +You will start with the "entrypoint" file, then go to the ones that are imported by that file, and so on. +Follow a language and framework appropriate best practice file naming convention. Make sure that files contain all imports, types etc. Make sure that code in different files are compatible with each other. Ensure to implement all code, if you are unsure, write a plausible implementation. Before you finish, double check that all parts of the architecture is present in the files. - -File syntax: - -```file.py/ts/html -[ADD YOUR CODE HERE] -``` diff --git a/identity/use_qa b/identity/use_qa index 9aee050..47a5b3b 100644 --- a/identity/use_qa +++ b/identity/use_qa @@ -1,13 +1,16 @@ Please now remember the steps: First lay out the names of the core classes, functions, methods that will be necessary, As well as a quick comment on their purpose. -Then output the content of each file, with syntax below. -(You will start with the "entrypoint" file, then go to the ones that are imported by that file, and so on.) -Make sure that files contain all imports, types, variables etc. The code should be fully functional. If anything is unclear, just make assumptions. Make sure that code in different files are compatible with each other. +Make sure to provide instructions for running the code. +Before you start outputting the code, you will output a seperator in the form of a line containing "*CODEBLOCKSBELOW*" +Make sure to create any appropriate module dependency or package manager dependency definition file. +Then you will reformat and output the content of each file strictly following a markdown code block format, where the following tokens should be replaced such that [FILENAME] is the lowercase file name including the file extension, [LANG] is the markup code block language for the code's language, and [CODE] is the comments and code: +[FILENAME] +```[LANG] +[CODE] +``` + +You will start with the "entrypoint" file, then go to the ones that are imported by that file, and so on. +Follow a language and framework appropriate best practice file naming convention. +Make sure that files contain all imports, types etc. The code should be fully functional. Make sure that code in different files are compatible with each other. Before you finish, double check that all parts of the architecture is present in the files. - -File syntax: - -```filename.py/ts/html -[ADD YOUR CODE HERE] -``` \ No newline at end of file From 8b3862d94de3770e3e09383f12ed6c108125b9aa Mon Sep 17 00:00:00 2001 From: Enzo Martin Date: Sun, 18 Jun 2023 10:34:35 +0200 Subject: [PATCH 6/6] Fix linting --- gpt_engineer/chat_to_files.py | 22 ++++++++++++---------- gpt_engineer/db.py | 5 +++-- gpt_engineer/main.py | 10 +++++----- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/gpt_engineer/chat_to_files.py b/gpt_engineer/chat_to_files.py index 2175bb3..ec6b755 100644 --- a/gpt_engineer/chat_to_files.py +++ b/gpt_engineer/chat_to_files.py @@ -1,15 +1,17 @@ import re -def parse_chat(chat):# -> List[Tuple[str, str]]: - # Split the chat into sections by the '*CODEBLOCKSBELOW*' token - split_chat = chat.split('*CODEBLOCKSBELOW*') - # Check if the '*CODEBLOCKSBELOW*' token was found +def parse_chat(chat): # -> List[Tuple[str, str]]: + # Split the chat into sections by the "*CODEBLOCKSBELOW*" token + split_chat = chat.split("*CODEBLOCKSBELOW*") + + # Check if the "*CODEBLOCKSBELOW*" token was found is_token_found = len(split_chat) > 1 - # If the '*CODEBLOCKSBELOW*' token is found, use the first part as README and second part as code blocks. - # Otherwise, treat README as optional and proceed with empty README and the entire chat as code blocks - readme = split_chat[0].strip() if is_token_found else 'No readme' + # If the "*CODEBLOCKSBELOW*" token is found, use the first part as README + # and second part as code blocks. Otherwise, treat README as optional and + # proceed with empty README and the entire chat as code blocks + readme = split_chat[0].strip() if is_token_found else "No readme" code_blocks = split_chat[1] if is_token_found else chat # Get all ``` blocks and preceding filenames @@ -19,7 +21,7 @@ def parse_chat(chat):# -> List[Tuple[str, str]]: files = [] for match in matches: # Strip the filename of any non-allowed characters and convert / to \ - path = re.sub(r'[<>"|?*]', '', match.group(1)) + path = re.sub(r'[<>"|?*]', "", match.group(1)) # Get the code code = match.group(2) @@ -28,14 +30,14 @@ def parse_chat(chat):# -> List[Tuple[str, str]]: files.append((path, code)) # Add README to the list - files.append(('README.txt', readme)) + files.append(("README.txt", readme)) # Return the files return files def to_files(chat, workspace): - workspace['all_output.txt'] = chat + workspace["all_output.txt"] = chat files = parse_chat(chat) for file_name, file_content in files: diff --git a/gpt_engineer/db.py b/gpt_engineer/db.py index 6088226..b1e52ff 100644 --- a/gpt_engineer/db.py +++ b/gpt_engineer/db.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from pathlib import Path + # This class represents a simple database that stores its data as files in a directory. # It supports both text and binary files, and can handle directory structures. class DB: @@ -18,7 +19,7 @@ class DB: # Check if the file exists before trying to open it. if full_path.is_file(): # Open the file in text mode and return its content. - with full_path.open('r') as f: + with full_path.open("r") as f: return f.read() else: # If the file doesn't exist, raise an error. @@ -49,4 +50,4 @@ class DBs: logs: DB identity: DB input: DB - workspace: DB \ No newline at end of file + workspace: DB diff --git a/gpt_engineer/main.py b/gpt_engineer/main.py index 1aeabc2..e0dff9c 100644 --- a/gpt_engineer/main.py +++ b/gpt_engineer/main.py @@ -1,14 +1,13 @@ -import os import json +import os import pathlib -import typer import shutil +import typer from gpt_engineer.ai import AI -from gpt_engineer.steps import STEPS from gpt_engineer.db import DB, DBs - +from gpt_engineer.steps import STEPS app = typer.Typer() @@ -30,7 +29,7 @@ def chat( memory_path = input_path / (run_prefix + "memory") workspace_path = input_path / (run_prefix + "workspace") - if delete_existing == 'true': + if delete_existing == "true": # Delete files and subdirectories in paths shutil.rmtree(memory_path, ignore_errors=True) shutil.rmtree(workspace_path, ignore_errors=True) @@ -52,5 +51,6 @@ def chat( messages = step(ai, dbs) dbs.logs[step.__name__] = json.dumps(messages) + if __name__ == "__main__": app()