Merge branch 'master' of https://github.com/Torantulino/Auto-GPT

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1,33 @@
-### Background
+<!-- 📢 Announcement
+We've recently noticed an increase in pull requests focusing on combining multiple changes. While the intentions behind these PRs are appreciated, it's essential to maintain a clean and manageable git history. To ensure the quality of our repository, we kindly ask you to adhere to the following guidelines when submitting PRs:
 
-<!-- Provide a brief overview of why this change is being made. Include any relevant context, prior discussions, or links to relevant issues. -->
+Focus on a single, specific change.
+Do not include any unrelated or "extra" modifications.
+Provide clear documentation and explanations of the changes made.
+Ensure diffs are limited to the intended lines — no applying preferred formatting styles or line endings (unless that's what the PR is about).
+For guidance on committing only the specific lines you have changed, refer to this helpful video: https://youtu.be/8-hSNHHbiZg
+
+By following these guidelines, your PRs are more likely to be merged quickly after testing, as long as they align with the project's overall direction. -->
+
+### Background
+<!-- Provide a concise overview of the rationale behind this change. Include relevant context, prior discussions, or links to related issues. Ensure that the change aligns with the project's overall direction. -->
 
 ### Changes
+<!-- Describe the specific, focused change made in this pull request. Detail the modifications clearly and avoid any unrelated or "extra" changes. -->
 
-<!-- Describe the changes made in this pull request. Be specific and detailed. -->
+### Documentation
+<!-- Explain how your changes are documented, such as in-code comments or external documentation. Ensure that the documentation is clear, concise, and easy to understand. -->
 
 ### Test Plan
+<!-- Describe how you tested this functionality. Include steps to reproduce, relevant test cases, and any other pertinent information. -->
 
-<!-- Explain how you tested this functionality. Include the steps to reproduce and any relevant test cases. -->
+### PR Quality Checklist
+- [ ] My pull request is atomic and focuses on a single change.
+- [ ] I have thouroughly tested my changes with multiple different prompts.
+- [ ] I have considered potential risks and mitigations for my changes.
+- [ ] I have documented my changes clearly and comprehensively.
+- [ ] I have not snuck in any "extra" small tweaks changes <!-- Submit these as seperate Pull Reqests, they are the easiest to merge! -->
 
-### Change Safety
+<!-- If you haven't added tests, please explain why. If you have, check the appropriate box. If you've ensured your PR is atomic and well-documented, check the corresponding boxes. -->
 
-- [ ] I have added tests to cover my changes
-- [ ] I have considered potential risks and mitigations for my changes
-
-<!-- If you haven't added tests, please explain why. If you have, check the appropriate box. -->
+<!-- By submitting this, I agree that my pull request should be closed if I do not fill this out or follow the guide lines. -->

.gitignore

@@ -7,5 +7,7 @@ package-lock.json
 auto_gpt_workspace/*
 *.mpeg
 .env
+venv/*
 outputs/*
 ai_settings.yaml
+auto-gpt.json

scripts/json_parser.py

@@ -1,11 +1,13 @@
 import json
+from typing import Any, Dict, Union
 from call_ai_function import call_ai_function
 from config import Config
+from json_utils import correct_json
+
 cfg = Config()
 
-def fix_and_parse_json(json_str: str, try_to_fix_with_gpt: bool = True):
-    json_schema = """
-    {
+JSON_SCHEMA = """
+{
     "command": {
         "name": "command name",
         "args":{
@@ -20,44 +22,68 @@ def fix_and_parse_json(json_str: str, try_to_fix_with_gpt: bool = True):
         "criticism": "constructive self-criticism",
         "speak": "thoughts summary to say to user"
     }
-    }
-    """
+}
+"""
 
+
+def fix_and_parse_json(
+    json_str: str,
+    try_to_fix_with_gpt: bool = True
+) -> Union[str, Dict[Any, Any]]:
     try:
         json_str = json_str.replace('\t', '')
         return json.loads(json_str)
-    except Exception as e:
-        # Let's do something manually - sometimes GPT responds with something BEFORE the braces:
-        # "I'm sorry, I don't understand. Please try again."{"text": "I'm sorry, I don't understand. Please try again.", "confidence": 0.0}
-        # So let's try to find the first brace and then parse the rest of the string
+    except json.JSONDecodeError as _:  # noqa: F841
+        json_str = correct_json(json_str)
         try:
-          brace_index = json_str.index("{")
-          json_str = json_str[brace_index:]
-          last_brace_index = json_str.rindex("}")
-          json_str = json_str[:last_brace_index+1]
-          return json.loads(json_str)
-        except Exception as e:
-          if try_to_fix_with_gpt:
-            print(f"Warning: Failed to parse AI output, attempting to fix.\n If you see this warning frequently, it's likely that your prompt is confusing the AI. Try changing it up slightly.")
+            return json.loads(json_str)
+        except json.JSONDecodeError as _:  # noqa: F841
+            pass
+    # Let's do something manually:
+    # sometimes GPT responds with something BEFORE the braces:
+    # "I'm sorry, I don't understand. Please try again."
+    # {"text": "I'm sorry, I don't understand. Please try again.",
+    #  "confidence": 0.0}
+    # So let's try to find the first brace and then parse the rest
+    #  of the string
+    try:
+        brace_index = json_str.index("{")
+        json_str = json_str[brace_index:]
+        last_brace_index = json_str.rindex("}")
+        json_str = json_str[:last_brace_index+1]
+        return json.loads(json_str)
+    except json.JSONDecodeError as e:  # noqa: F841
+        if try_to_fix_with_gpt:
+            print("Warning: Failed to parse AI output, attempting to fix."
+                  "\n If you see this warning frequently, it's likely that"
+                  " your prompt is confusing the AI. Try changing it up"
+                  " slightly.")
             # Now try to fix this up using the ai_functions
-            ai_fixed_json = fix_json(json_str, json_schema, cfg.debug)
+            ai_fixed_json = fix_json(json_str, JSON_SCHEMA, cfg.debug)
             if ai_fixed_json != "failed":
-              return json.loads(ai_fixed_json)
+                return json.loads(ai_fixed_json)
             else:
-              print(f"Failed to fix ai output, telling the AI.") # This allows the AI to react to the error message, which usually results in it correcting its ways.
-              return json_str
-          else:
+                # This allows the AI to react to the error message,
+                #   which usually results in it correcting its ways.
+                print("Failed to fix ai output, telling the AI.")
+                return json_str
+        else:
             raise e
 
+
 def fix_json(json_str: str, schema: str, debug=False) -> str:
     # Try to fix the JSON using gpt:
     function_string = "def fix_json(json_str: str, schema:str=None) -> str:"
     args = [f"'''{json_str}'''", f"'''{schema}'''"]
-    description_string = """Fixes the provided JSON string to make it parseable and fully complient with the provided schema.\n If an object or field specifed in the schema isn't contained within the correct JSON, it is ommited.\n This function is brilliant at guessing when the format is incorrect."""
+    description_string = "Fixes the provided JSON string to make it parseable"\
+        " and fully complient with the provided schema.\n If an object or"\
+        " field specifed in the schema isn't contained within the correct"\
+        " JSON, it is ommited.\n This function is brilliant at guessing"\
+        " when the format is incorrect."
 
     # If it doesn't already start with a "`", add one:
     if not json_str.startswith("`"):
-      json_str = "```json\n" + json_str + "\n```"
+        json_str = "```json\n" + json_str + "\n```"
     result_string = call_ai_function(
         function_string, args, description_string, model=cfg.fast_llm_model
     )
@@ -68,9 +94,9 @@ def fix_json(json_str: str, schema: str, debug=False) -> str:
         print(f"Fixed JSON: {result_string}")
         print("----------- END OF FIX ATTEMPT ----------------")
     try:
-        json.loads(result_string) # just check the validity
+        json.loads(result_string)  # just check the validity
         return result_string
-    except:
+    except:  # noqa: E722
         # Get the call stack:
         # import traceback
         # call_stack = traceback.format_exc()

scripts/json_utils.py

@@ -0,0 +1,127 @@
+import re
+import json
+from config import Config
+
+cfg = Config()
+
+
+def extract_char_position(error_message: str) -> int:
+    """Extract the character position from the JSONDecodeError message.
+
+    Args:
+        error_message (str): The error message from the JSONDecodeError
+          exception.
+
+    Returns:
+        int: The character position.
+    """
+    import re
+
+    char_pattern = re.compile(r'\(char (\d+)\)')
+    if match := char_pattern.search(error_message):
+        return int(match[1])
+    else:
+        raise ValueError("Character position not found in the error message.")
+
+
+def add_quotes_to_property_names(json_string: str) -> str:
+    """
+    Add quotes to property names in a JSON string.
+
+    Args:
+        json_string (str): The JSON string.
+
+    Returns:
+        str: The JSON string with quotes added to property names.
+    """
+
+    def replace_func(match):
+        return f'"{match.group(1)}":'
+
+    property_name_pattern = re.compile(r'(\w+):')
+    corrected_json_string = property_name_pattern.sub(
+        replace_func,
+        json_string)
+
+    try:
+        json.loads(corrected_json_string)
+        return corrected_json_string
+    except json.JSONDecodeError as e:
+        raise e
+
+
+def balance_braces(json_string: str) -> str:
+    """
+    Balance the braces in a JSON string.
+
+    Args:
+        json_string (str): The JSON string.
+
+    Returns:
+        str: The JSON string with braces balanced.
+    """
+
+    open_braces_count = json_string.count('{')
+    close_braces_count = json_string.count('}')
+
+    while open_braces_count > close_braces_count:
+        json_string += '}'
+        close_braces_count += 1
+
+    while close_braces_count > open_braces_count:
+        json_string = json_string.rstrip('}')
+        close_braces_count -= 1
+
+    try:
+        json.loads(json_string)
+        return json_string
+    except json.JSONDecodeError as e:
+        raise e
+
+
+def fix_invalid_escape(json_str: str, error_message: str) -> str:
+    while error_message.startswith('Invalid \\escape'):
+        bad_escape_location = extract_char_position(error_message)
+        json_str = json_str[:bad_escape_location] + \
+            json_str[bad_escape_location + 1:]
+        try:
+            json.loads(json_str)
+            return json_str
+        except json.JSONDecodeError as e:
+            if cfg.debug:
+                print('json loads error - fix invalid escape', e)
+            error_message = str(e)
+    return json_str
+
+
+def correct_json(json_str: str) -> str:
+    """
+    Correct common JSON errors.
+
+    Args:
+        json_str (str): The JSON string.
+    """
+
+    try:
+        if cfg.debug:
+            print("json", json_str)
+        json.loads(json_str)
+        return json_str
+    except json.JSONDecodeError as e:
+        if cfg.debug:
+            print('json loads error', e)
+        error_message = str(e)
+        if error_message.startswith('Invalid \\escape'):
+            json_str = fix_invalid_escape(json_str, error_message)
+        if error_message.startswith('Expecting property name enclosed in double quotes'):
+            json_str = add_quotes_to_property_names(json_str)
+            try:
+                json.loads(json_str)
+                return json_str
+            except json.JSONDecodeError as e:
+                if cfg.debug:
+                    print('json loads error - add quotes', e)
+                error_message = str(e)
+        if balanced_str := balance_braces(json_str):
+            return balanced_str
+    return json_str