Merge pull request #140 from EnzoMartin/directory-creation

Directory creation & project directory

gpt_engineer/chat_to_files.py

@@ -1,27 +1,42 @@
 import re
-from typing import List, Tuple
-from gpt_engineer.db import DB
 
 
-def parse_chat(chat) -> List[Tuple[str, str]]:
-    # Get all ``` blocks
-    regex = r"```(.*?)```"
+def parse_chat(chat):  # -> List[Tuple[str, str]]:
+    # Split the chat into sections by the "*CODEBLOCKSBELOW*" token
+    split_chat = chat.split("*CODEBLOCKSBELOW*")
 
-    matches = re.finditer(regex, chat, re.DOTALL)
+    # Check if the "*CODEBLOCKSBELOW*" token was found
+    is_token_found = len(split_chat) > 1
+
+    # If the "*CODEBLOCKSBELOW*" token is found, use the first part as README
+    # and second part as code blocks. Otherwise, treat README as optional and
+    # proceed with empty README and the entire chat as code blocks
+    readme = split_chat[0].strip() if is_token_found else "No readme"
+    code_blocks = split_chat[1] if is_token_found else chat
+
+    # Get all ``` blocks and preceding filenames
+    regex = r"\[(.*?)\]\s*```.*?\n(.*?)```"
+    matches = re.finditer(regex, code_blocks, re.DOTALL)
 
     files = []
     for match in matches:
-        path = match.group(1).split("\n")[0]
+        # Strip the filename of any non-allowed characters and convert / to \
+        path = re.sub(r'[<>"|?*]', "", match.group(1))
+
         # Get the code
-        code = match.group(1).split("\n")[1:]
-        code = "\n".join(code)
+        code = match.group(2)
+
         # Add the file to the list
         files.append((path, code))
 
+    # Add README to the list
+    files.append(("README.txt", readme))
+
+    # Return the files
     return files
 
 
-def to_files(chat: str, workspace: DB):
+def to_files(chat, workspace):
     workspace["all_output.txt"] = chat
 
     files = parse_chat(chat)

gpt_engineer/db.py

@@ -1,33 +1,51 @@
 from dataclasses import dataclass
-import os
 from pathlib import Path
 
 
+# This class represents a simple database that stores its data as files in a directory.
+# It supports both text and binary files, and can handle directory structures.
 class DB:
-    """A simple key-value store, where keys are filenames and values are file contents."""
-
     def __init__(self, path):
+        # Convert the path string to a Path object and get its absolute path.
         self.path = Path(path).absolute()
-        os.makedirs(self.path, exist_ok=True)
+
+        # Create the directory if it doesn't exist.
+        self.path.mkdir(parents=True, exist_ok=True)
 
     def __getitem__(self, key):
-        with open(self.path / key, encoding='utf-8') as f:
-            return f.read()
+        # Combine the database directory with the provided file path.
+        full_path = self.path / key
+
+        # Check if the file exists before trying to open it.
+        if full_path.is_file():
+            # Open the file in text mode and return its content.
+            with full_path.open("r") as f:
+                return f.read()
+        else:
+            # If the file doesn't exist, raise an error.
+            raise FileNotFoundError(f"No such file: '{full_path}'")
 
     def __setitem__(self, key, val):
-        Path(self.path / key).absolute().parent.mkdir(parents=True, exist_ok=True)
+        # Combine the database directory with the provided file path.
+        full_path = self.path / key
 
-        with open(self.path / key, 'w', encoding='utf-8') as f:
-            f.write(val)
+        # Create the directory tree if it doesn't exist.
+        full_path.parent.mkdir(parents=True, exist_ok=True)
 
-    def __contains__(self, key):
-        return (self.path / key).exists()
+        # Write the data to the file. If val is a string, it's written as text.
+        # If val is bytes, it's written as binary data.
+        if isinstance(val, str):
+            full_path.write_text(val)
+        elif isinstance(val, bytes):
+            full_path.write_bytes(val)
+        else:
+            # If val is neither a string nor bytes, raise an error.
+            raise TypeError("val must be either a str or bytes")
 
 
+# dataclass for all dbs:
 @dataclass
 class DBs:
-    """A dataclass for all dbs"""
-
     memory: DB
     logs: DB
     identity: DB

gpt_engineer/main.py

@@ -1,20 +1,21 @@
-import os
 import json
+import os
 import pathlib
+import shutil
+
 import typer
 
-from gpt_engineer.chat_to_files import to_files
 from gpt_engineer.ai import AI
-from gpt_engineer.steps import STEPS
 from gpt_engineer.db import DB, DBs
-
+from gpt_engineer.steps import STEPS
 
 app = typer.Typer()
 
 
 @app.command()
 def chat(
-    project_path: str = typer.Argument(str(pathlib.Path(os.path.curdir) / "example"), help="path"),
+    project_path: str = typer.Argument("example", help="path"),
+    delete_existing: str = typer.Argument(None, help="delete existing files"),
     run_prefix: str = typer.Option(
         "",
         help="run prefix, if you want to run multiple variants of the same project and later compare them",
@@ -24,9 +25,14 @@ def chat(
     steps_config: str = "default",
 ):
     app_dir = pathlib.Path(os.path.curdir)
-    input_path = project_path
-    memory_path = pathlib.Path(project_path) / (run_prefix + "memory")
-    workspace_path = pathlib.Path(project_path) / (run_prefix + "workspace")
+    input_path = pathlib.Path(app_dir / "projects" / project_path)
+    memory_path = input_path / (run_prefix + "memory")
+    workspace_path = input_path / (run_prefix + "workspace")
+
+    if delete_existing == "true":
+        # Delete files and subdirectories in paths
+        shutil.rmtree(memory_path, ignore_errors=True)
+        shutil.rmtree(workspace_path, ignore_errors=True)
 
     ai = AI(
         model=model,
@@ -45,5 +51,6 @@ def chat(
         messages = step(ai, dbs)
         dbs.logs[step.__name__] = json.dumps(messages)
 
+
 if __name__ == "__main__":
     app()

gpt_engineer/steps.py

@@ -157,8 +157,8 @@ def execute_entrypoint(ai, dbs):
 def gen_entrypoint(ai, dbs):
     messages = ai.start(
         system=(
-            f"You will get information about a codebase that is currently on disk in the current folder.\n"
-            "From this you will answer with one code block that includes all the necessary macos terminal commands to "
+            f"You will get information about a codebase that is currently on disk in the folder {dbs.workspace.path}.\n"
+            "From this you will answer with code blocks that includes all the necessary Windows, MacOS, and Linux terminal commands to "
             "a) install dependencies "
             "b) run all necessary parts of the codebase (in parallell if necessary).\n"
             "Do not install globally. Do not use sudo.\n"