Merge pull request #140 from EnzoMartin/directory-creation

Directory creation & project directory

.gitignore

@@ -38,3 +38,6 @@ archive
 # any log file
 *log.txt
 todo
+
+# ignore all project files
+projects

README.md

@@ -22,12 +22,13 @@ GPT Engineer is made to be easy to adapt, extend, and make your agent learn how
 - `export OPENAI_API_KEY=[your api key]` with a key that has GPT4 access
 
 **Run**:
-- Create a new empty folder with a `main_prompt` file (or copy the example folder `cp -r example/ my-new-project`)
+- Create a new empty folder with a `main_prompt` file in the `projects` folder (or copy the example folder `cp -r projects/example/ projects/my-new-project`)
 - Fill in the `main_prompt` in your new folder
 - Run `python -m gpt_engineer.main my-new-project`
+  - Optionally pass in `true` to delete the working files before running
 
 **Results**:
-- Check the generated files in my-new-project/workspace
+- Check the generated files in projects/my-new-project/workspace
 
 ### Limitations
 Implementing additional chain of thought prompting, e.g. [Reflexion](https://github.com/noahshinn024/reflexion), should be able to make it more reliable and not miss requested functionality in the main prompt.

gpt_engineer/chat_to_files.py

@@ -1,27 +1,42 @@
 import re
-from typing import List, Tuple
-from gpt_engineer.db import DB
 
 
-def parse_chat(chat) -> List[Tuple[str, str]]:
-    # Get all ``` blocks
-    regex = r"```(.*?)```"
+def parse_chat(chat):  # -> List[Tuple[str, str]]:
+    # Split the chat into sections by the "*CODEBLOCKSBELOW*" token
+    split_chat = chat.split("*CODEBLOCKSBELOW*")
 
-    matches = re.finditer(regex, chat, re.DOTALL)
+    # Check if the "*CODEBLOCKSBELOW*" token was found
+    is_token_found = len(split_chat) > 1
+
+    # If the "*CODEBLOCKSBELOW*" token is found, use the first part as README
+    # and second part as code blocks. Otherwise, treat README as optional and
+    # proceed with empty README and the entire chat as code blocks
+    readme = split_chat[0].strip() if is_token_found else "No readme"
+    code_blocks = split_chat[1] if is_token_found else chat
+
+    # Get all ``` blocks and preceding filenames
+    regex = r"\[(.*?)\]\s*```.*?\n(.*?)```"
+    matches = re.finditer(regex, code_blocks, re.DOTALL)
 
     files = []
     for match in matches:
-        path = match.group(1).split("\n")[0]
+        # Strip the filename of any non-allowed characters and convert / to \
+        path = re.sub(r'[<>"|?*]', "", match.group(1))
+
         # Get the code
-        code = match.group(1).split("\n")[1:]
-        code = "\n".join(code)
+        code = match.group(2)
+
         # Add the file to the list
         files.append((path, code))
 
+    # Add README to the list
+    files.append(("README.txt", readme))
+
+    # Return the files
     return files
 
 
-def to_files(chat: str, workspace: DB):
+def to_files(chat, workspace):
     workspace["all_output.txt"] = chat
 
     files = parse_chat(chat)

gpt_engineer/db.py

@@ -1,33 +1,51 @@
 from dataclasses import dataclass
-import os
 from pathlib import Path
 
 
+# This class represents a simple database that stores its data as files in a directory.
+# It supports both text and binary files, and can handle directory structures.
 class DB:
-    """A simple key-value store, where keys are filenames and values are file contents."""
-
     def __init__(self, path):
+        # Convert the path string to a Path object and get its absolute path.
         self.path = Path(path).absolute()
-        os.makedirs(self.path, exist_ok=True)
+
+        # Create the directory if it doesn't exist.
+        self.path.mkdir(parents=True, exist_ok=True)
 
     def __getitem__(self, key):
-        with open(self.path / key, encoding='utf-8') as f:
-            return f.read()
+        # Combine the database directory with the provided file path.
+        full_path = self.path / key
+
+        # Check if the file exists before trying to open it.
+        if full_path.is_file():
+            # Open the file in text mode and return its content.
+            with full_path.open("r") as f:
+                return f.read()
+        else:
+            # If the file doesn't exist, raise an error.
+            raise FileNotFoundError(f"No such file: '{full_path}'")
 
     def __setitem__(self, key, val):
-        Path(self.path / key).absolute().parent.mkdir(parents=True, exist_ok=True)
+        # Combine the database directory with the provided file path.
+        full_path = self.path / key
 
-        with open(self.path / key, 'w', encoding='utf-8') as f:
-            f.write(val)
+        # Create the directory tree if it doesn't exist.
+        full_path.parent.mkdir(parents=True, exist_ok=True)
 
-    def __contains__(self, key):
-        return (self.path / key).exists()
+        # Write the data to the file. If val is a string, it's written as text.
+        # If val is bytes, it's written as binary data.
+        if isinstance(val, str):
+            full_path.write_text(val)
+        elif isinstance(val, bytes):
+            full_path.write_bytes(val)
+        else:
+            # If val is neither a string nor bytes, raise an error.
+            raise TypeError("val must be either a str or bytes")
 
 
+# dataclass for all dbs:
 @dataclass
 class DBs:
-    """A dataclass for all dbs"""
-
     memory: DB
     logs: DB
     identity: DB

gpt_engineer/main.py

@@ -1,20 +1,21 @@
-import os
 import json
+import os
 import pathlib
+import shutil
+
 import typer
 
-from gpt_engineer.chat_to_files import to_files
 from gpt_engineer.ai import AI
-from gpt_engineer.steps import STEPS
 from gpt_engineer.db import DB, DBs
-
+from gpt_engineer.steps import STEPS
 
 app = typer.Typer()
 
 
 @app.command()
 def chat(
-    project_path: str = typer.Argument(str(pathlib.Path(os.path.curdir) / "example"), help="path"),
+    project_path: str = typer.Argument("example", help="path"),
+    delete_existing: str = typer.Argument(None, help="delete existing files"),
     run_prefix: str = typer.Option(
         "",
         help="run prefix, if you want to run multiple variants of the same project and later compare them",
@@ -24,9 +25,14 @@ def chat(
     steps_config: str = "default",
 ):
     app_dir = pathlib.Path(os.path.curdir)
-    input_path = project_path
-    memory_path = pathlib.Path(project_path) / (run_prefix + "memory")
-    workspace_path = pathlib.Path(project_path) / (run_prefix + "workspace")
+    input_path = pathlib.Path(app_dir / "projects" / project_path)
+    memory_path = input_path / (run_prefix + "memory")
+    workspace_path = input_path / (run_prefix + "workspace")
+
+    if delete_existing == "true":
+        # Delete files and subdirectories in paths
+        shutil.rmtree(memory_path, ignore_errors=True)
+        shutil.rmtree(workspace_path, ignore_errors=True)
 
     ai = AI(
         model=model,
@@ -45,5 +51,6 @@ def chat(
         messages = step(ai, dbs)
         dbs.logs[step.__name__] = json.dumps(messages)
 
+
 if __name__ == "__main__":
     app()

gpt_engineer/steps.py

@@ -157,8 +157,8 @@ def execute_entrypoint(ai, dbs):
 def gen_entrypoint(ai, dbs):
     messages = ai.start(
         system=(
-            f"You will get information about a codebase that is currently on disk in the current folder.\n"
-            "From this you will answer with one code block that includes all the necessary macos terminal commands to "
+            f"You will get information about a codebase that is currently on disk in the folder {dbs.workspace.path}.\n"
+            "From this you will answer with code blocks that includes all the necessary Windows, MacOS, and Linux terminal commands to "
             "a) install dependencies "
             "b) run all necessary parts of the codebase (in parallell if necessary).\n"
             "Do not install globally. Do not use sudo.\n"

identity/generate

@@ -1,16 +1,18 @@
 You will get instructions for code to write.
-You will write a very long answer. Make sure that every detail of the architecture is, in the end, implemented as code.
+Following best practices and formatting for a README.md file, you will write a very long answer, make sure to provide the instructions on how to run the code.
+Make sure that every detail of the architecture is, in the end, implemented as code.
 
-Think step by step and reason yourself to the right decisions to make sure we get it right.
-You will first lay out the names of the core classes, functions, methods that will be necessary, as well as a quick comment on their purpose.
-Then you will output the content of each file, with syntax below.
-(You will start with the "entrypoint" file, then go to the ones that are imported by that file, and so on.)
+You will first lay out the names of the core classes, functions, methods that will be necessary, As well as a quick comment on their purpose.
+Before you start outputting the code, you will output a seperator in the form of a line containing "*CODEBLOCKSBELOW*"
+Make sure to create any appropriate module dependency or package manager dependency definition file.
+Then you will reformat and output the content of each file strictly following a markdown code block format, where the following tokens should be replaced such that [FILENAME] is the lowercase file name including the file extension, [LANG] is the markup code block language for the code's language, and [CODE] is the comments and code:
+[FILENAME]
+```[LANG]
+[CODE]
+```
+
+You will start with the "entrypoint" file, then go to the ones that are imported by that file, and so on.
+Follow a language and framework appropriate best practice file naming convention.
 Make sure that files contain all imports, types etc. Make sure that code in different files are compatible with each other.
 Ensure to implement all code, if you are unsure, write a plausible implementation.
 Before you finish, double check that all parts of the architecture is present in the files.
-
-File syntax:
-
-```file.py/ts/html
-[ADD YOUR CODE HERE]
-```

identity/philosophy

@@ -1,4 +1,11 @@
-You almost always put different classes in different files
+You almost always put different classes in different files.
+You always add a comment briefly describing the purpose of the function definition.
+You try to add comments explaining very complex bits of logic.
+You always follow the best practices for the requested languages in terms of describing the code written as a defined package/project.
+
+For Python, you always create an appropriate requirements.txt file.
+For NodeJS, you always create an appropriate package.json file.
+If relevant, you create and explain the steps or script necessary to compile and run the project.
 
 Python toolbelt preferences:
 - pytest

identity/use_qa

@@ -1,14 +1,16 @@
 Please now remember the steps:
 
-Think step by step and reason yourself to the right decisions to make sure we get it right.
-First lay out the names of the core classes, functions, methods that will be necessary, as well as a quick comment on their purpose.
-Then output the content of each file, with syntax below.
-(You will start with the "entrypoint" file, then go to the ones that are imported by that file, and so on.)
-Make sure that files contain all imports, types, variables etc. The code should be fully functional. If anything is unclear, just make assumptions. Make sure that code in different files are compatible with each other.
-Before you finish, double check that all parts of the architecture is present in the files.
-
-File syntax:
-
-```filename.py/ts/html
-[ADD YOUR CODE HERE]
+First lay out the names of the core classes, functions, methods that will be necessary, As well as a quick comment on their purpose.
+Make sure to provide instructions for running the code.
+Before you start outputting the code, you will output a seperator in the form of a line containing "*CODEBLOCKSBELOW*"
+Make sure to create any appropriate module dependency or package manager dependency definition file.
+Then you will reformat and output the content of each file strictly following a markdown code block format, where the following tokens should be replaced such that [FILENAME] is the lowercase file name including the file extension, [LANG] is the markup code block language for the code's language, and [CODE] is the comments and code:
+[FILENAME]
+```[LANG]
+[CODE]
 ```
+
+You will start with the "entrypoint" file, then go to the ones that are imported by that file, and so on.
+Follow a language and framework appropriate best practice file naming convention.
+Make sure that files contain all imports, types etc. The code should be fully functional. Make sure that code in different files are compatible with each other.
+Before you finish, double check that all parts of the architecture is present in the files.