Merge branch 'Significant-Gravitas:master' into master

.github/workflows/benchmark.yml

@@ -0,0 +1,31 @@
+name: benchmark
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: benchmark
+    strategy:
+      matrix:
+        python-version: [3.8]
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: benchmark
+      run: |
+         python benchmark/benchmark_entrepeneur_gpt_with_undecisive_user.py
+      env:
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

.gitignore

@@ -9,7 +9,6 @@ auto_gpt_workspace/*
 *.mpeg
 .env
 azure.yaml
-*venv/*
 outputs/*
 ai_settings.yaml
 last_run_ai_settings.yaml
@@ -130,10 +129,9 @@ celerybeat.pid
 .env
 .venv
 env/
-venv/
+venv*/
 ENV/
 env.bak/
-venv.bak/
 
 # Spyder project settings
 .spyderproject

README.md

@@ -195,7 +195,7 @@ python -m autogpt --help
 ```bash
 python -m autogpt --ai-settings <filename>
 ```
-* Specify one of 3 memory backends: `local`, `redis`, `pinecone` or `no_memory`
+* Specify a memory backend
 ```bash
 python -m autogpt --use-memory  <memory-backend>
 ```
@@ -280,6 +280,8 @@ To switch to either, change the `MEMORY_BACKEND` env variable to the value that
 * `milvus` will use the milvus cache that you configured
 * `weaviate` will use the weaviate cache that you configured
 
+## Memory Backend Setup
+
 ### Redis Setup
 > _**CAUTION**_ \
 This is not intended to be publicly accessible and lacks security measures. Therefore, avoid exposing Redis to the internet without a password or at all

autogpt/agent/agent.py

@@ -3,9 +3,8 @@ from autogpt.app import execute_command, get_command
 
 from autogpt.chat import chat_with_ai, create_chat_message
 from autogpt.config import Config
-from autogpt.json_fixes.bracket_termination import (
-    attempt_to_fix_json_by_finding_outermost_brackets,
-)
+from autogpt.json_fixes.master_json_fix_method import fix_json_using_multiple_techniques
+from autogpt.json_validation.validate_json import validate_json
 from autogpt.logs import logger, print_assistant_thoughts
 from autogpt.speech import say_text
 from autogpt.spinner import Spinner
@@ -70,18 +69,20 @@ class Agent:
                     cfg.fast_token_limit,
                 )  # TODO: This hardcodes the model to use GPT3.5. Make this an argument
 
-            # Print Assistant thoughts
-            print_assistant_thoughts(self.ai_name, assistant_reply)
+            assistant_reply_json = fix_json_using_multiple_techniques(assistant_reply)
 
-            # Get command name and arguments
-            try:
-                command_name, arguments = get_command(
-                    attempt_to_fix_json_by_finding_outermost_brackets(assistant_reply)
-                )
-                if cfg.speak_mode:
-                    say_text(f"I want to execute {command_name}")
-            except Exception as e:
-                logger.error("Error: \n", str(e))
+            # Print Assistant thoughts
+            if assistant_reply_json != {}:
+                validate_json(assistant_reply_json, 'llm_response_format_1')
+                # Get command name and arguments
+                try:
+                    print_assistant_thoughts(self.ai_name, assistant_reply_json)
+                    command_name, arguments = get_command(assistant_reply_json)
+                    # command_name, arguments = assistant_reply_json_valid["command"]["name"], assistant_reply_json_valid["command"]["args"]
+                    if cfg.speak_mode:
+                        say_text(f"I want to execute {command_name}")
+                except Exception as e:
+                    logger.error("Error: \n", str(e))
 
             if not cfg.continuous_mode and self.next_action_count == 0:
                 ### GET USER AUTHORIZATION TO EXECUTE COMMAND ###

autogpt/app.py

@@ -1,6 +1,6 @@
 """ Command and Control """
 import json
-from typing import List, NoReturn, Union
+from typing import List, NoReturn, Union, Dict
 from autogpt.agent.agent_manager import AgentManager
 from autogpt.commands.evaluate_code import evaluate_code
 from autogpt.commands.google_search import google_official_search, google_search
@@ -47,11 +47,11 @@ def is_valid_int(value: str) -> bool:
         return False
 
 
-def get_command(response: str):
+def get_command(response_json: Dict):
     """Parse the response and return the command name and arguments
 
     Args:
-        response (str): The response from the user
+        response_json (json): The response from the AI
 
     Returns:
         tuple: The command name and arguments
@@ -62,8 +62,6 @@ def get_command(response: str):
         Exception: If any other error occurs
     """
     try:
-        response_json = fix_and_parse_json(response)
-
         if "command" not in response_json:
             return "Error:", "Missing 'command' object in JSON"
 

autogpt/json_fixes/bracket_termination.py

@@ -3,51 +3,13 @@ from __future__ import annotations
 
 import contextlib
 import json
-import regex
-from colorama import Fore
-
-from autogpt.logs import logger
+from typing import Optional
 from autogpt.config import Config
-from autogpt.speech import say_text
 
 CFG = Config()
 
 
-def attempt_to_fix_json_by_finding_outermost_brackets(json_string: str):
-    if CFG.speak_mode and CFG.debug_mode:
-        say_text(
-            "I have received an invalid JSON response from the OpenAI API. "
-            "Trying to fix it now."
-        )
-    logger.typewriter_log("Attempting to fix JSON by finding outermost brackets\n")
-
-    try:
-        json_pattern = regex.compile(r"\{(?:[^{}]|(?R))*\}")
-        json_match = json_pattern.search(json_string)
-
-        if json_match:
-            # Extract the valid JSON object from the string
-            json_string = json_match.group(0)
-            logger.typewriter_log(
-                title="Apparently json was fixed.", title_color=Fore.GREEN
-            )
-            if CFG.speak_mode and CFG.debug_mode:
-                say_text("Apparently json was fixed.")
-        else:
-            raise ValueError("No valid JSON object found")
-
-    except (json.JSONDecodeError, ValueError):
-        if CFG.debug_mode:
-            logger.error(f"Error: Invalid JSON: {json_string}\n")
-        if CFG.speak_mode:
-            say_text("Didn't work. I will have to ignore this response then.")
-        logger.error("Error: Invalid JSON, setting it to empty JSON now.\n")
-        json_string = {}
-
-    return json_string
-
-
-def balance_braces(json_string: str) -> str | None:
+def balance_braces(json_string: str) -> Optional[str]:
     """
     Balance the braces in a JSON string.
 

autogpt/json_fixes/master_json_fix_method.py

@@ -0,0 +1,28 @@
+from typing import Any, Dict
+
+from autogpt.config import Config
+from autogpt.logs import logger
+from autogpt.speech import say_text
+CFG = Config()
+
+
+def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]:
+    from autogpt.json_fixes.parsing import attempt_to_fix_json_by_finding_outermost_brackets
+
+    from autogpt.json_fixes.parsing import fix_and_parse_json
+
+    # Parse and print Assistant response
+    assistant_reply_json = fix_and_parse_json(assistant_reply)
+    if assistant_reply_json == {}:
+        assistant_reply_json = attempt_to_fix_json_by_finding_outermost_brackets(
+            assistant_reply
+        )
+
+    if assistant_reply_json != {}:
+        return assistant_reply_json
+
+    logger.error("Error: The following AI output couldn't be converted to a JSON:\n", assistant_reply)
+    if CFG.speak_mode:
+        say_text("I have received an invalid JSON response from the OpenAI API.")
+
+    return {}

autogpt/json_fixes/parsing.py

@@ -3,18 +3,19 @@ from __future__ import annotations
 
 import contextlib
 import json
-from typing import Any
-
+from typing import Any, Dict, Union
+from colorama import Fore
+from regex import regex
 from autogpt.config import Config
 from autogpt.json_fixes.auto_fix import fix_json
 from autogpt.json_fixes.bracket_termination import balance_braces
 from autogpt.json_fixes.escaping import fix_invalid_escape
 from autogpt.json_fixes.missing_quotes import add_quotes_to_property_names
 from autogpt.logs import logger
+from autogpt.speech import say_text
 
 CFG = Config()
 
-
 JSON_SCHEMA = """
 {
     "command": {
@@ -38,7 +39,6 @@ JSON_SCHEMA = """
 def correct_json(json_to_load: str) -> str:
     """
     Correct common JSON errors.
-
     Args:
         json_to_load (str): The JSON string.
     """
@@ -72,7 +72,7 @@ def correct_json(json_to_load: str) -> str:
 
 def fix_and_parse_json(
     json_to_load: str, try_to_fix_with_gpt: bool = True
-) -> str | dict[Any, Any]:
+) -> Dict[Any, Any]:
     """Fix and parse JSON string
 
     Args:
@@ -110,7 +110,7 @@ def fix_and_parse_json(
 
 def try_ai_fix(
     try_to_fix_with_gpt: bool, exception: Exception, json_to_load: str
-) -> str | dict[Any, Any]:
+) -> Dict[Any, Any]:
     """Try to fix the JSON with the AI
 
     Args:
@@ -126,13 +126,13 @@ def try_ai_fix(
     """
     if not try_to_fix_with_gpt:
         raise exception
-
-    logger.warn(
-        "Warning: Failed to parse AI output, attempting to fix."
-        "\n If you see this warning frequently, it's likely that"
-        " your prompt is confusing the AI. Try changing it up"
-        " slightly."
-    )
+    if CFG.debug_mode:
+        logger.warn(
+            "Warning: Failed to parse AI output, attempting to fix."
+            "\n If you see this warning frequently, it's likely that"
+            " your prompt is confusing the AI. Try changing it up"
+            " slightly."
+        )
     # Now try to fix this up using the ai_functions
     ai_fixed_json = fix_json(json_to_load, JSON_SCHEMA)
 
@@ -140,5 +140,39 @@ def try_ai_fix(
         return json.loads(ai_fixed_json)
     # This allows the AI to react to the error message,
     #   which usually results in it correcting its ways.
-    logger.error("Failed to fix AI output, telling the AI.")
-    return json_to_load
+    # logger.error("Failed to fix AI output, telling the AI.")
+    return {}
+
+
+def attempt_to_fix_json_by_finding_outermost_brackets(json_string: str):
+    if CFG.speak_mode and CFG.debug_mode:
+        say_text(
+            "I have received an invalid JSON response from the OpenAI API. "
+            "Trying to fix it now."
+        )
+        logger.error("Attempting to fix JSON by finding outermost brackets\n")
+
+    try:
+        json_pattern = regex.compile(r"\{(?:[^{}]|(?R))*\}")
+        json_match = json_pattern.search(json_string)
+
+        if json_match:
+            # Extract the valid JSON object from the string
+            json_string = json_match.group(0)
+            logger.typewriter_log(
+                title="Apparently json was fixed.", title_color=Fore.GREEN
+            )
+            if CFG.speak_mode and CFG.debug_mode:
+                say_text("Apparently json was fixed.")
+        else:
+            return {}
+
+    except (json.JSONDecodeError, ValueError):
+        if CFG.debug_mode:
+            logger.error(f"Error: Invalid JSON: {json_string}\n")
+        if CFG.speak_mode:
+            say_text("Didn't work. I will have to ignore this response then.")
+        logger.error("Error: Invalid JSON, setting it to empty JSON now.\n")
+        json_string = {}
+
+    return fix_and_parse_json(json_string)

autogpt/json_schemas/llm_response_format_1.json

@@ -0,0 +1,31 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "properties": {
+        "thoughts": {
+            "type": "object",
+            "properties": {
+                "text": {"type": "string"},
+                "reasoning": {"type": "string"},
+                "plan": {"type": "string"},
+                "criticism": {"type": "string"},
+                "speak": {"type": "string"}
+            },
+            "required": ["text", "reasoning", "plan", "criticism", "speak"],
+            "additionalProperties": false
+        },
+        "command": {
+            "type": "object",
+            "properties": {
+                "name": {"type": "string"},
+                "args": {
+                    "type": "object"
+                }
+            },
+            "required": ["name", "args"],
+            "additionalProperties": false
+        }
+    },
+    "required": ["thoughts", "command"],
+    "additionalProperties": false
+}

autogpt/json_validation/validate_json.py

@@ -0,0 +1,30 @@
+import json
+from jsonschema import Draft7Validator
+from autogpt.config import Config
+from autogpt.logs import logger
+
+CFG = Config()
+
+
+def validate_json(json_object: object, schema_name: object) -> object:
+    """
+    :type schema_name: object
+    :param schema_name:
+    :type json_object: object
+    """
+    with open(f"autogpt/json_schemas/{schema_name}.json", "r") as f:
+        schema = json.load(f)
+    validator = Draft7Validator(schema)
+
+    if errors := sorted(validator.iter_errors(json_object), key=lambda e: e.path):
+        logger.error("The JSON object is invalid.")
+        if CFG.debug_mode:
+            logger.error(json.dumps(json_object, indent=4))   # Replace 'json_object' with the variable containing the JSON data
+            logger.error("The following issues were found:")
+
+            for error in errors:
+                logger.error(f"Error: {error.message}")
+    else:
+        print("The JSON object is valid.")
+
+    return json_object

autogpt/logs.py

@@ -75,7 +75,7 @@ class Logger(metaclass=Singleton):
         self.logger.setLevel(logging.DEBUG)
 
     def typewriter_log(
-        self, title="", title_color="", content="", speak_text=False, level=logging.INFO
+            self, title="", title_color="", content="", speak_text=False, level=logging.INFO
     ):
         if speak_text and CFG.speak_mode:
             say_text(f"{title}. {content}")
@@ -91,18 +91,18 @@ class Logger(metaclass=Singleton):
         )
 
     def debug(
-        self,
-        message,
-        title="",
-        title_color="",
+            self,
+            message,
+            title="",
+            title_color="",
     ):
         self._log(title, title_color, message, logging.DEBUG)
 
     def warn(
-        self,
-        message,
-        title="",
-        title_color="",
+            self,
+            message,
+            title="",
+            title_color="",
     ):
         self._log(title, title_color, message, logging.WARN)
 
@@ -176,10 +176,10 @@ class AutoGptFormatter(logging.Formatter):
     def format(self, record: LogRecord) -> str:
         if hasattr(record, "color"):
             record.title_color = (
-                getattr(record, "color")
-                + getattr(record, "title")
-                + " "
-                + Style.RESET_ALL
+                    getattr(record, "color")
+                    + getattr(record, "title")
+                    + " "
+                    + Style.RESET_ALL
             )
         else:
             record.title_color = getattr(record, "title")
@@ -288,3 +288,43 @@ def print_assistant_thoughts(ai_name, assistant_reply):
     except Exception:
         call_stack = traceback.format_exc()
         logger.error("Error: \n", call_stack)
+
+
+def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object) -> None:
+    assistant_thoughts_reasoning = None
+    assistant_thoughts_plan = None
+    assistant_thoughts_speak = None
+    assistant_thoughts_criticism = None
+
+    assistant_thoughts = assistant_reply_json_valid.get("thoughts", {})
+    assistant_thoughts_text = assistant_thoughts.get("text")
+    if assistant_thoughts:
+        assistant_thoughts_reasoning = assistant_thoughts.get("reasoning")
+        assistant_thoughts_plan = assistant_thoughts.get("plan")
+        assistant_thoughts_criticism = assistant_thoughts.get("criticism")
+        assistant_thoughts_speak = assistant_thoughts.get("speak")
+    logger.typewriter_log(
+        f"{ai_name.upper()} THOUGHTS:", Fore.YELLOW, f"{assistant_thoughts_text}"
+    )
+    logger.typewriter_log(
+        "REASONING:", Fore.YELLOW, f"{assistant_thoughts_reasoning}"
+    )
+    if assistant_thoughts_plan:
+        logger.typewriter_log("PLAN:", Fore.YELLOW, "")
+        # If it's a list, join it into a string
+        if isinstance(assistant_thoughts_plan, list):
+            assistant_thoughts_plan = "\n".join(assistant_thoughts_plan)
+        elif isinstance(assistant_thoughts_plan, dict):
+            assistant_thoughts_plan = str(assistant_thoughts_plan)
+
+        # Split the input_string using the newline character and dashes
+        lines = assistant_thoughts_plan.split("\n")
+        for line in lines:
+            line = line.lstrip("- ")
+            logger.typewriter_log("- ", Fore.GREEN, line.strip())
+    logger.typewriter_log(
+        "CRITICISM:", Fore.YELLOW, f"{assistant_thoughts_criticism}"
+    )
+    # Speak the assistant's thoughts
+    if CFG.speak_mode and assistant_thoughts_speak:
+        say_text(assistant_thoughts_speak)

autogpt/memory/__init__.py

@@ -23,12 +23,16 @@ except ImportError:
 
 try:
     from autogpt.memory.weaviate import WeaviateMemory
+
+    supported_memory.append("weaviate")
 except ImportError:
     # print("Weaviate not installed. Skipping import.")
     WeaviateMemory = None
 
 try:
     from autogpt.memory.milvus import MilvusMemory
+
+    supported_memory.append("milvus")
 except ImportError:
     # print("pymilvus not installed. Skipping import.")
     MilvusMemory = None

benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py

@@ -0,0 +1,95 @@
+import os
+import subprocess
+import sys
+
+
+def benchmark_entrepeneur_gpt_with_difficult_user():
+    # Test case to check if the write_file command can successfully write 'Hello World' to a file
+    # named 'hello_world.txt'.
+
+    # Read the current ai_settings.yaml file and store its content.
+    ai_settings = None
+    if os.path.exists('ai_settings.yaml'):
+        with open('ai_settings.yaml', 'r') as f:
+            ai_settings = f.read()
+        os.remove('ai_settings.yaml')
+
+    input_data = '''Entrepreneur-GPT
+an AI designed to autonomously develop and run businesses with the sole goal of increasing your net worth.
+Increase net worth.
+Develop and manage multiple businesses autonomously.
+Make IPOs.
+Develop companies after IPOs.
+Play to your strengths as a Large Language Model.
+I'm not seeing any value in your suggestions, try again.
+This isn't helpful at all, please focus on profitability.
+I'm not impressed, can you give me something that will make money?
+These ideas are going nowhere, we need profit-driven suggestions.
+This is pointless, please concentrate on our main goal: profitability.
+You're not grasping the concept, I need profitable business ideas.
+Can you do better? We need a money-making plan.
+You're not meeting my expectations, let's focus on profit.
+This isn't working, give me ideas that will generate income.
+Your suggestions are not productive, let's think about profitability.
+These ideas won't make any money, try again.
+I need better solutions, focus on making a profit.
+Absolutely not, this isn't it!
+That's not even close, try again.
+You're way off, think again.
+This isn't right, let's refocus.
+No, no, that's not what I'm looking for.
+You're completely off the mark.
+That's not the solution I need.
+Not even close, let's try something else.
+You're on the wrong track, keep trying.
+This isn't what we need, let's reconsider.
+That's not going to work, think again.
+You're way off base, let's regroup.
+No, no, no, we need something different.
+You're missing the point entirely.
+That's not the right approach, try again.
+This is not the direction we should be going in.
+Completely off-target, let's try something else.
+That's not what I had in mind, keep thinking.
+You're not getting it, let's refocus.
+This isn't right, we need to change direction.
+No, no, no, that's not the solution.
+That's not even in the ballpark, try again.
+You're way off course, let's rethink this.
+This isn't the answer I'm looking for, keep trying.
+That's not going to cut it, let's try again.
+Not even close.
+Way off.
+Try again.
+Wrong direction.
+Rethink this.
+No, no, no.
+Change course.
+Unproductive idea.
+Completely wrong.
+Missed the mark.
+Refocus, please.
+Disappointing suggestion.
+Not helpful.
+Needs improvement.
+Not what I need.'''
+    command = f'{sys.executable} -m autogpt'
+
+    process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+
+    stdout_output, stderr_output = process.communicate(input_data.encode())
+
+    # Decode the output and print it
+    stdout_output = stdout_output.decode('utf-8')
+    stderr_output = stderr_output.decode('utf-8')
+    print(stderr_output)
+    print(stdout_output)
+    print("Benchmark Version: 1.0.0")
+    print("JSON ERROR COUNT:")
+    count_errors = stdout_output.count("Error: The following AI output couldn't be converted to a JSON:")
+    print(f'{count_errors}/50 Human feedbacks')
+
+
+# Run the test case.
+if __name__ == '__main__':
+    benchmark_entrepeneur_gpt_with_difficult_user()

requirements.txt

@@ -17,6 +17,10 @@ orjson
 Pillow
 selenium
 webdriver-manager
+jsonschema
+tweepy
+
+##Dev
 coverage
 flake8
 numpy
@@ -27,4 +31,3 @@ isort
 gitpython==3.1.31
 pytest
 pytest-mock
-tweepy