diff --git a/.env.template b/.env.template index eeff2907..09deeb93 100644 --- a/.env.template +++ b/.env.template @@ -5,8 +5,6 @@ EXECUTE_LOCAL_COMMANDS=False # BROWSE_CHUNK_MAX_LENGTH - When browsing website, define the length of chunk stored in memory BROWSE_CHUNK_MAX_LENGTH=8192 -# BROWSE_SUMMARY_MAX_TOKEN - Define the maximum length of the summary generated by GPT agent when browsing website -BROWSE_SUMMARY_MAX_TOKEN=300 # USER_AGENT - Define the user-agent used by the requests library to browse website (string) # USER_AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36" # AI_SETTINGS_FILE - Specifies which AI Settings file to use (defaults to ai_settings.yaml) @@ -54,6 +52,7 @@ SMART_TOKEN_LIMIT=8000 # local - Default # pinecone - Pinecone (if configured) # redis - Redis (if configured) +# milvus - Milvus (if configured) MEMORY_BACKEND=local ### PINECONE @@ -63,7 +62,7 @@ PINECONE_API_KEY=your-pinecone-api-key PINECONE_ENV=your-pinecone-region ### REDIS -# REDIS_HOST - Redis host (Default: localhost) +# REDIS_HOST - Redis host (Default: localhost, use "redis" for docker-compose) # REDIS_PORT - Redis port (Default: 6379) # REDIS_PASSWORD - Redis password (Default: "") # WIPE_REDIS_ON_START - Wipes data / index on start (Default: False) diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..a7ad7263 --- /dev/null +++ b/.envrc @@ -0,0 +1,4 @@ +# Upon entering directory, direnv requests user permission once to automatically load project dependencies onwards. +# Eliminating the need of running "nix develop github:superherointj/nix-auto-gpt" for Nix users to develop/use Auto-GPT. + +[[ -z $IN_NIX_SHELL ]] && use flake github:superherointj/nix-auto-gpt diff --git a/.flake8 b/.flake8 index c456b393..77976224 100644 --- a/.flake8 +++ b/.flake8 @@ -1,12 +1,12 @@ [flake8] max-line-length = 88 -extend-ignore = E203 +select = "E303, W293, W291, W292, E305, E231, E302" exclude = .tox, __pycache__, *.pyc, .env - venv/* - .venv/* - reports/* - dist/* \ No newline at end of file + venv*/*, + .venv/*, + reports/*, + dist/*, diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c355965a..cf7ffbf3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -30,4 +30,4 @@ By following these guidelines, your PRs are more likely to be merged quickly aft - + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39f3aea9..0a9a9287 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,15 @@ jobs: - name: Lint with flake8 continue-on-error: false - run: flake8 autogpt/ tests/ --select E303,W293,W291,W292,E305,E231,E302 + run: flake8 + + - name: Check black formatting + continue-on-error: false + run: black . --check + + - name: Check isort formatting + continue-on-error: false + run: isort . --check - name: Run unittest tests with coverage run: | diff --git a/.github/workflows/pr-label.yml b/.github/workflows/pr-label.yml new file mode 100644 index 00000000..92c5a66b --- /dev/null +++ b/.github/workflows/pr-label.yml @@ -0,0 +1,28 @@ +name: "Pull Request auto-label" +on: + # So that PRs touching the same files as the push are updated + push: + # So that the `dirtyLabel` is removed if conflicts are resolve + # We recommend `pull_request_target` so that github secrets are available. + # In `pull_request` we wouldn't be able to change labels of fork PRs + pull_request_target: + types: [opened, synchronize] +concurrency: + group: ${{ format('pr-label-{0}', github.event.pull_request.number || github.sha) }} + cancel-in-progress: true + +jobs: + conflicts: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Update PRs with conflict labels + uses: eps1lon/actions-label-merge-conflict@releases/2.x + with: + dirtyLabel: "conflicts" + #removeOnDirtyLabel: "PR: ready to ship" + repoToken: "${{ secrets.GITHUB_TOKEN }}" + commentOnDirty: "This pull request has conflicts with the base branch, please resolve those so we can evaluate the pull request." + commentOnClean: "Conflicts have been resolved! 🎉 A maintainer will review the pull request shortly." diff --git a/.gitignore b/.gitignore index 2220ef6e..26d7e5a3 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,7 @@ celerybeat.pid *.sage.py # Environments +.direnv/ .env .venv env/ diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index 8ad53a86..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[settings] -profile = black -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 88 -skip = venv,env,node_modules,.env,.venv,dist -sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd1d0ec9..3722b25e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,39 +1,32 @@ repos: - - repo: https://github.com/sourcery-ai/sourcery - rev: v1.1.0 # Get the latest tag from https://github.com/sourcery-ai/sourcery/tags - hooks: - - id: sourcery - - repo: https://github.com/pre-commit/pre-commit-hooks rev: v0.9.2 hooks: - id: check-added-large-files - args: [ '--maxkb=500' ] + args: ['--maxkb=500'] - id: check-byte-order-marker - id: check-case-conflict - id: check-merge-conflict - id: check-symlinks - id: debug-statements - - - repo: local + + - repo: https://github.com/pycqa/isort + rev: 5.12.0 hooks: - id: isort - name: isort-local - entry: isort - language: python - types: [ python ] - exclude: .+/(dist|.venv|venv|build)/.+ - pass_filenames: true + language_version: python3.10 + + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: - id: black - name: black-local - entry: black - language: python - types: [ python ] - exclude: .+/(dist|.venv|venv|build)/.+ - pass_filenames: true + language_version: python3.10 + + - repo: local + hooks: - id: pytest-check name: pytest-check entry: pytest --cov=autogpt --without-integration --without-slow-integration language: system pass_filenames: false - always_run: true \ No newline at end of file + always_run: true diff --git a/Dockerfile b/Dockerfile index 9886d742..09b5303d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,16 @@ FROM python:3.11-slim RUN apt-get -y update RUN apt-get -y install git chromium-driver +# Install Xvfb and other dependencies for headless browser testing +RUN apt-get update \ + && apt-get install -y wget gnupg2 libgtk-3-0 libdbus-glib-1-2 dbus-x11 xvfb ca-certificates + +# Install Firefox / Chromium +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \ + && apt-get update \ + && apt-get install -y chromium firefox-esr + # Set environment variables ENV PIP_NO_CACHE_DIR=yes \ PYTHONUNBUFFERED=1 \ @@ -17,8 +27,9 @@ RUN chown appuser:appuser /home/appuser USER appuser # Copy the requirements.txt file and install the requirements -COPY --chown=appuser:appuser requirements-docker.txt . -RUN pip install --no-cache-dir --user -r requirements-docker.txt +COPY --chown=appuser:appuser requirements.txt . +RUN sed -i '/Items below this point will not be included in the Docker Image/,$d' requirements.txt && \ + pip install --no-cache-dir --user -r requirements.txt # Copy the application files COPY --chown=appuser:appuser autogpt/ ./autogpt diff --git a/README.md b/README.md index 71957748..8e5cfe7b 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,21 @@ Development of this free, open-source project is made possible by all the Billing](./docs/imgs/openai-api-key-billing-paid-account.png) + +#### **PLEASE ENSURE YOU HAVE DONE THIS STEP BEFORE PROCEEDING, OTHERWISE NOTHING WILL WORK!** + ## 💾 Installation To install Auto-GPT, follow these steps: @@ -207,18 +220,6 @@ python -m autogpt --speak - Adam : pNInz6obpgDQGcFmaJgB - Sam : yoZ06aMxZJJ28mfd3POQ - -## OpenAI API Keys Configuration - -Obtain your OpenAI API key from: https://platform.openai.com/account/api-keys. - -To use OpenAI API key for Auto-GPT, you NEED to have billing set up (AKA paid account). - -You can set up paid account at https://platform.openai.com/account/billing/overview. - -![For OpenAI API key to work, set up paid account at OpenAI API > Billing](./docs/imgs/openai-api-key-billing-paid-account.png) - - ## 🔍 Google API Keys Configuration This section is optional, use the official google api if you are having issues with error 429 when running a google search. @@ -325,7 +326,7 @@ export MEMORY_BACKEND="pinecone" ### Milvus Setup -[Milvus](https://milvus.io/) is a open-source, high scalable vector database to storage huge amount of vector-based memory and provide fast relevant search. +[Milvus](https://milvus.io/) is an open-source, highly scalable vector database to store huge amounts of vector-based memory and provide fast relevant search. - setup milvus database, keep your pymilvus version and milvus version same to avoid compatible issues. - setup by open source [Install Milvus](https://milvus.io/docs/install_standalone-operator.md) @@ -341,6 +342,14 @@ export MEMORY_BACKEND="pinecone" [Weaviate](https://weaviate.io/) is an open-source vector database. It allows to store data objects and vector embeddings from ML-models and scales seamlessly to billion of data objects. [An instance of Weaviate can be created locally (using Docker), on Kubernetes or using Weaviate Cloud Services](https://weaviate.io/developers/weaviate/quickstart). Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) is supported which allows the Auto-GPT process itself to start a Weaviate instance. To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `pip install "weaviate-client>=3.15.4"`. +#### Install the Weaviate client + +Install the Weaviate client before usage. + +``` +$ pip install weaviate-client +``` + #### Setting up environment variables In your `.env` file set the following: diff --git a/autogpt/__main__.py b/autogpt/__main__.py index 5f462234..64ed398e 100644 --- a/autogpt/__main__.py +++ b/autogpt/__main__.py @@ -1,12 +1,15 @@ """Main script for the autogpt package.""" import logging + from colorama import Fore + from autogpt.agent.agent import Agent from autogpt.args import parse_arguments from autogpt.config import Config, check_openai_api_key from autogpt.logs import logger from autogpt.memory import get_memory from autogpt.prompt import construct_prompt + # Load environment variables from .env file diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index f87cd483..ee7885f8 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -1,6 +1,6 @@ from colorama import Fore, Style -from autogpt.app import execute_command, get_command +from autogpt.app import execute_command, get_command from autogpt.chat import chat_with_ai, create_chat_message from autogpt.config import Config from autogpt.json_utils.json_fix_llm import fix_json_using_multiple_techniques @@ -84,7 +84,7 @@ class Agent: # Print Assistant thoughts if assistant_reply_json != {}: - validate_json(assistant_reply_json, 'llm_response_format_1') + validate_json(assistant_reply_json, "llm_response_format_1") # Get command name and arguments try: print_assistant_thoughts(self.ai_name, assistant_reply_json) @@ -115,9 +115,12 @@ class Agent: console_input = clean_input( Fore.MAGENTA + "Input:" + Style.RESET_ALL ) - if console_input.lower().rstrip() == "y": + if console_input.lower().strip() == "y": user_input = "GENERATE NEXT COMMAND JSON" break + elif console_input.lower().strip() == "": + print("Invalid input format.") + continue elif console_input.lower().startswith("y -"): try: self.next_action_count = abs( diff --git a/autogpt/agent/agent_manager.py b/autogpt/agent/agent_manager.py index e4bfb126..898767a4 100644 --- a/autogpt/agent/agent_manager.py +++ b/autogpt/agent/agent_manager.py @@ -1,8 +1,10 @@ """Agent manager for managing GPT agents""" from __future__ import annotations -from autogpt.llm_utils import create_chat_completion +from typing import Union + from autogpt.config.config import Singleton +from autogpt.llm_utils import create_chat_completion class AgentManager(metaclass=Singleton): diff --git a/autogpt/app.py b/autogpt/app.py index d3a5e57c..90e0d3b5 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -1,31 +1,36 @@ """ Command and Control """ import json -from typing import List, NoReturn, Union, Dict +from typing import Dict, List, NoReturn, Union + from autogpt.agent.agent_manager import AgentManager -from autogpt.commands.evaluate_code import evaluate_code -from autogpt.commands.google_search import google_official_search, google_search -from autogpt.commands.improve_code import improve_code -from autogpt.commands.write_tests import write_tests -from autogpt.config import Config -from autogpt.commands.image_gen import generate_image from autogpt.commands.audio_text import read_audio_from_file -from autogpt.commands.web_requests import scrape_links, scrape_text -from autogpt.commands.execute_code import execute_python_file, execute_shell +from autogpt.commands.evaluate_code import evaluate_code +from autogpt.commands.execute_code import ( + execute_python_file, + execute_shell, + execute_shell_popen, +) from autogpt.commands.file_operations import ( append_to_file, delete_file, + download_file, read_file, search_files, write_to_file, - download_file ) +from autogpt.commands.git_operations import clone_repository +from autogpt.commands.google_search import google_official_search, google_search +from autogpt.commands.image_gen import generate_image +from autogpt.commands.improve_code import improve_code +from autogpt.commands.twitter import send_tweet +from autogpt.commands.web_requests import scrape_links, scrape_text +from autogpt.commands.web_selenium import browse_website +from autogpt.commands.write_tests import write_tests +from autogpt.config import Config +from autogpt.json_utils.json_fix_llm import fix_and_parse_json from autogpt.memory import get_memory from autogpt.processing.text import summarize_text from autogpt.speech import say_text -from autogpt.commands.web_selenium import browse_website -from autogpt.commands.git_operations import clone_repository -from autogpt.commands.twitter import send_tweet - CFG = Config() AGENT_MANAGER = AgentManager() @@ -111,11 +116,10 @@ def execute_command(command_name: str, arguments): arguments (dict): The arguments for the command Returns: - str: The result of the command""" - memory = get_memory(CFG) - + str: The result of the command + """ try: - command_name = map_command_synonyms(command_name) + command_name = map_command_synonyms(command_name.lower()) if command_name == "google": # Check if the Google API key is set and use the official search method # If the API key is not set or has only whitespaces, use the unofficial @@ -129,12 +133,16 @@ def execute_command(command_name: str, arguments): # google_result can be a list or a string depending on the search results if isinstance(google_result, list): - safe_message = [google_result_single.encode('utf-8', 'ignore') for google_result_single in google_result] + safe_message = [ + google_result_single.encode("utf-8", "ignore") + for google_result_single in google_result + ] else: - safe_message = google_result.encode('utf-8', 'ignore') + safe_message = google_result.encode("utf-8", "ignore") - return str(safe_message) + return safe_message.decode("utf-8") elif command_name == "memory_add": + memory = get_memory(CFG) return memory.add(arguments["string"]) elif command_name == "start_agent": return start_agent( @@ -190,6 +198,15 @@ def execute_command(command_name: str, arguments): " shell commands, EXECUTE_LOCAL_COMMANDS must be set to 'True' " "in your config. Do not attempt to bypass the restriction." ) + elif command_name == "execute_shell_popen": + if CFG.execute_local_commands: + return execute_shell_popen(arguments["command_line"]) + else: + return ( + "You are not allowed to run local shell commands. To execute" + " shell commands, EXECUTE_LOCAL_COMMANDS must be set to 'True' " + "in your config. Do not attempt to bypass the restriction." + ) elif command_name == "read_audio_from_file": return read_audio_from_file(arguments["file"]) elif command_name == "generate_image": @@ -211,7 +228,7 @@ def execute_command(command_name: str, arguments): def get_text_summary(url: str, question: str) -> str: - """Return the results of a google search + """Return the results of a Google search Args: url (str): The url to scrape @@ -226,7 +243,7 @@ def get_text_summary(url: str, question: str) -> str: def get_hyperlinks(url: str) -> Union[str, List[str]]: - """Return the results of a google search + """Return the results of a Google search Args: url (str): The url to scrape diff --git a/autogpt/args.py b/autogpt/args.py index f0e9c07a..5ca4221c 100644 --- a/autogpt/args.py +++ b/autogpt/args.py @@ -1,7 +1,8 @@ """This module contains the argument parsing logic for the script.""" import argparse -from colorama import Fore, Back, Style +from colorama import Back, Fore, Style + from autogpt import utils from autogpt.config import Config from autogpt.logs import logger @@ -64,10 +65,10 @@ def parse_arguments() -> None: " skip the re-prompt.", ) parser.add_argument( - '--allow-downloads', - action='store_true', - dest='allow_downloads', - help='Dangerous: Allows Auto-GPT to download files natively.' + "--allow-downloads", + action="store_true", + dest="allow_downloads", + help="Dangerous: Allows Auto-GPT to download files natively.", ) args = parser.parse_args() @@ -141,10 +142,17 @@ def parse_arguments() -> None: if args.allow_downloads: logger.typewriter_log("Native Downloading:", Fore.GREEN, "ENABLED") - logger.typewriter_log("WARNING: ", Fore.YELLOW, - f"{Back.LIGHTYELLOW_EX}Auto-GPT will now be able to download and save files to your machine.{Back.RESET} " + - "It is recommended that you monitor any files it downloads carefully.") - logger.typewriter_log("WARNING: ", Fore.YELLOW, f"{Back.RED + Style.BRIGHT}ALWAYS REMEMBER TO NEVER OPEN FILES YOU AREN'T SURE OF!{Style.RESET_ALL}") + logger.typewriter_log( + "WARNING: ", + Fore.YELLOW, + f"{Back.LIGHTYELLOW_EX}Auto-GPT will now be able to download and save files to your machine.{Back.RESET} " + + "It is recommended that you monitor any files it downloads carefully.", + ) + logger.typewriter_log( + "WARNING: ", + Fore.YELLOW, + f"{Back.RED + Style.BRIGHT}ALWAYS REMEMBER TO NEVER OPEN FILES YOU AREN'T SURE OF!{Style.RESET_ALL}", + ) CFG.allow_downloads = True if args.browser_name: diff --git a/autogpt/commands/audio_text.py b/autogpt/commands/audio_text.py index 84819d5e..cae32d4e 100644 --- a/autogpt/commands/audio_text.py +++ b/autogpt/commands/audio_text.py @@ -1,6 +1,7 @@ -import requests import json +import requests + from autogpt.config import Config from autogpt.workspace import path_in_workspace diff --git a/autogpt/commands/execute_code.py b/autogpt/commands/execute_code.py index 2cc797cb..a524081e 100644 --- a/autogpt/commands/execute_code.py +++ b/autogpt/commands/execute_code.py @@ -5,10 +5,10 @@ import subprocess import docker from docker.errors import ImageNotFound -from autogpt.workspace import path_in_workspace, WORKSPACE_PATH +from autogpt.workspace import WORKSPACE_PATH, path_in_workspace -def execute_python_file(file: str): +def execute_python_file(file: str) -> str: """Execute a Python file in a Docker container and return the output Args: @@ -40,10 +40,10 @@ def execute_python_file(file: str): try: client = docker.from_env() - # You can replace 'python:3.8' with the desired Python image/version + # You can replace this with the desired Python image/version # You can find available Python images on Docker Hub: # https://hub.docker.com/_/python - image_name = "python:3.10" + image_name = "python:3-alpine" try: client.images.get(image_name) print(f"Image '{image_name}' found locally") @@ -114,6 +114,36 @@ def execute_shell(command_line: str) -> str: return output +def execute_shell_popen(command_line) -> str: + """Execute a shell command with Popen and returns an english description + of the event and the process id + + Args: + command_line (str): The command line to execute + + Returns: + str: Description of the fact that the process started and its id + """ + current_dir = os.getcwd() + + if WORKING_DIRECTORY not in current_dir: # Change dir into workspace if necessary + work_dir = os.path.join(os.getcwd(), WORKING_DIRECTORY) + os.chdir(work_dir) + + print(f"Executing command '{command_line}' in working directory '{os.getcwd()}'") + + do_not_show_output = subprocess.DEVNULL + process = subprocess.Popen( + command_line, shell=True, stdout=do_not_show_output, stderr=do_not_show_output + ) + + # Change back to whatever the prior working dir was + + os.chdir(current_dir) + + return f"Subprocess started with PID:'{str(process.pid)}'" + + def we_are_running_in_a_docker_container() -> bool: """Check if we are running in a Docker container diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index d273c1a3..72b02b5d 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -5,14 +5,14 @@ import os import os.path from pathlib import Path from typing import Generator, List + import requests -from requests.adapters import HTTPAdapter -from requests.adapters import Retry -from colorama import Fore, Back +from colorama import Back, Fore +from requests.adapters import HTTPAdapter, Retry + from autogpt.spinner import Spinner from autogpt.utils import readable_file_size -from autogpt.workspace import path_in_workspace, WORKSPACE_PATH - +from autogpt.workspace import WORKSPACE_PATH, path_in_workspace LOG_FILE = "file_logger.txt" LOG_FILE_PATH = WORKSPACE_PATH / LOG_FILE @@ -47,7 +47,7 @@ def log_operation(operation: str, filename: str) -> None: with open(LOG_FILE_PATH, "w", encoding="utf-8") as f: f.write("File Operation Logger ") - append_to_file(LOG_FILE, log_entry, shouldLog = False) + append_to_file(LOG_FILE, log_entry, shouldLog=False) def split_file( @@ -70,9 +70,14 @@ def split_file( while start < content_length: end = start + max_length if end + overlap < content_length: - chunk = content[start : end + overlap] + chunk = content[start : end + overlap - 1] else: chunk = content[start:content_length] + + # Account for the case where the last chunk is shorter than the overlap, so it has already been consumed + if len(chunk) <= overlap: + break + yield chunk start += max_length - overlap @@ -236,23 +241,23 @@ def download_file(url, filename): session = requests.Session() retry = Retry(total=3, backoff_factor=1, status_forcelist=[502, 503, 504]) adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) + session.mount("http://", adapter) + session.mount("https://", adapter) total_size = 0 downloaded_size = 0 with session.get(url, allow_redirects=True, stream=True) as r: r.raise_for_status() - total_size = int(r.headers.get('Content-Length', 0)) + total_size = int(r.headers.get("Content-Length", 0)) downloaded_size = 0 - with open(safe_filename, 'wb') as f: + with open(safe_filename, "wb") as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) downloaded_size += len(chunk) - # Update the progress message + # Update the progress message progress = f"{readable_file_size(downloaded_size)} / {readable_file_size(total_size)}" spinner.update_message(f"{message} {progress}") diff --git a/autogpt/commands/git_operations.py b/autogpt/commands/git_operations.py index 675eb228..028f3b8d 100644 --- a/autogpt/commands/git_operations.py +++ b/autogpt/commands/git_operations.py @@ -1,5 +1,6 @@ """Git operations for autogpt""" import git + from autogpt.config import Config from autogpt.workspace import path_in_workspace @@ -7,7 +8,7 @@ CFG = Config() def clone_repository(repo_url: str, clone_path: str) -> str: - """Clone a github repository locally + """Clone a GitHub repository locally Args: repo_url (str): The URL of the repository to clone diff --git a/autogpt/commands/google_search.py b/autogpt/commands/google_search.py index 148ba1d0..7d38ce75 100644 --- a/autogpt/commands/google_search.py +++ b/autogpt/commands/google_search.py @@ -11,7 +11,7 @@ CFG = Config() def google_search(query: str, num_results: int = 8) -> str: - """Return the results of a google search + """Return the results of a Google search Args: query (str): The search query. @@ -35,7 +35,7 @@ def google_search(query: str, num_results: int = 8) -> str: def google_official_search(query: str, num_results: int = 8) -> str | list[str]: - """Return the results of a google search using the official Google API + """Return the results of a Google search using the official Google API Args: query (str): The search query. diff --git a/autogpt/commands/image_gen.py b/autogpt/commands/image_gen.py index 6243616e..4e8b47d6 100644 --- a/autogpt/commands/image_gen.py +++ b/autogpt/commands/image_gen.py @@ -7,6 +7,7 @@ from base64 import b64decode import openai import requests from PIL import Image + from autogpt.config import Config from autogpt.workspace import path_in_workspace diff --git a/autogpt/commands/twitter.py b/autogpt/commands/twitter.py index dc4d450c..3eaed36e 100644 --- a/autogpt/commands/twitter.py +++ b/autogpt/commands/twitter.py @@ -1,5 +1,6 @@ -import tweepy import os + +import tweepy from dotenv import load_dotenv load_dotenv() diff --git a/autogpt/commands/web_playwright.py b/autogpt/commands/web_playwright.py index a1abb6cb..4e388ded 100644 --- a/autogpt/commands/web_playwright.py +++ b/autogpt/commands/web_playwright.py @@ -8,6 +8,7 @@ except ImportError: "Playwright not installed. Please install it with 'pip install playwright' to use." ) from bs4 import BeautifulSoup + from autogpt.processing.html import extract_hyperlinks, format_hyperlinks diff --git a/autogpt/commands/web_requests.py b/autogpt/commands/web_requests.py index 50d8d383..406338f4 100644 --- a/autogpt/commands/web_requests.py +++ b/autogpt/commands/web_requests.py @@ -4,9 +4,9 @@ from __future__ import annotations from urllib.parse import urljoin, urlparse import requests -from requests.compat import urljoin -from requests import Response from bs4 import BeautifulSoup +from requests import Response +from requests.compat import urljoin from autogpt.config import Config from autogpt.memory import get_memory @@ -58,9 +58,28 @@ def check_local_file_access(url: str) -> bool: """ local_prefixes = [ "file:///", + "file://localhost/", "file://localhost", "http://localhost", + "http://localhost/", "https://localhost", + "https://localhost/", + "http://2130706433", + "http://2130706433/", + "https://2130706433", + "https://2130706433/", + "http://127.0.0.1/", + "http://127.0.0.1", + "https://127.0.0.1/", + "https://127.0.0.1", + "https://0.0.0.0/", + "https://0.0.0.0", + "http://0.0.0.0/", + "http://0.0.0.0", + "http://0000", + "http://0000/", + "https://0000", + "https://0000/", ] return any(url.startswith(prefix) for prefix in local_prefixes) diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 8c652294..9db5d035 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -1,22 +1,25 @@ """Selenium web scraping module.""" from __future__ import annotations -from selenium import webdriver -from autogpt.processing.html import extract_hyperlinks, format_hyperlinks -import autogpt.processing.text as summary -from bs4 import BeautifulSoup -from selenium.webdriver.remote.webdriver import WebDriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.wait import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from webdriver_manager.chrome import ChromeDriverManager -from webdriver_manager.firefox import GeckoDriverManager -from selenium.webdriver.chrome.options import Options as ChromeOptions -from selenium.webdriver.firefox.options import Options as FirefoxOptions -from selenium.webdriver.safari.options import Options as SafariOptions import logging from pathlib import Path +from sys import platform + +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.options import Options as ChromeOptions +from selenium.webdriver.common.by import By +from selenium.webdriver.firefox.options import Options as FirefoxOptions +from selenium.webdriver.remote.webdriver import WebDriver +from selenium.webdriver.safari.options import Options as SafariOptions +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.wait import WebDriverWait +from webdriver_manager.chrome import ChromeDriverManager +from webdriver_manager.firefox import GeckoDriverManager + +import autogpt.processing.text as summary from autogpt.config import Config +from autogpt.processing.html import extract_hyperlinks, format_hyperlinks FILE_DIR = Path(__file__).parent.parent CFG = Config() @@ -75,6 +78,9 @@ def scrape_text_with_selenium(url: str) -> tuple[WebDriver, str]: # See https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari driver = webdriver.Safari(options=options) else: + if platform == "linux" or platform == "linux2": + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--remote-debugging-port=9222") options.add_argument("--no-sandbox") driver = webdriver.Chrome( executable_path=ChromeDriverManager().install(), options=options diff --git a/autogpt/commands/write_tests.py b/autogpt/commands/write_tests.py index 138a1adb..35a08653 100644 --- a/autogpt/commands/write_tests.py +++ b/autogpt/commands/write_tests.py @@ -2,6 +2,7 @@ from __future__ import annotations import json + from autogpt.llm_utils import call_ai_function diff --git a/autogpt/config/__init__.py b/autogpt/config/__init__.py index ceb5566c..726b6dcf 100644 --- a/autogpt/config/__init__.py +++ b/autogpt/config/__init__.py @@ -2,7 +2,7 @@ This module contains the configuration classes for AutoGPT. """ from autogpt.config.ai_config import AIConfig -from autogpt.config.config import check_openai_api_key, Config +from autogpt.config.config import Config, check_openai_api_key from autogpt.config.singleton import AbstractSingleton, Singleton __all__ = [ diff --git a/autogpt/config/ai_config.py b/autogpt/config/ai_config.py index 86171357..d50c30be 100644 --- a/autogpt/config/ai_config.py +++ b/autogpt/config/ai_config.py @@ -6,6 +6,7 @@ from __future__ import annotations import os from typing import Type + import yaml diff --git a/autogpt/config/config.py b/autogpt/config/config.py index fe6f4f32..bc75b031 100644 --- a/autogpt/config/config.py +++ b/autogpt/config/config.py @@ -1,14 +1,13 @@ """Configuration class to store the state of bools for different scripts access.""" import os -from colorama import Fore - -from autogpt.config.singleton import Singleton import openai import yaml - +from colorama import Fore from dotenv import load_dotenv +from autogpt.config.singleton import Singleton + load_dotenv(verbose=True) @@ -33,7 +32,6 @@ class Config(metaclass=Singleton): self.fast_token_limit = int(os.getenv("FAST_TOKEN_LIMIT", 4000)) self.smart_token_limit = int(os.getenv("SMART_TOKEN_LIMIT", 8000)) self.browse_chunk_max_length = int(os.getenv("BROWSE_CHUNK_MAX_LENGTH", 8192)) - self.browse_summary_max_token = int(os.getenv("BROWSE_SUMMARY_MAX_TOKEN", 300)) self.openai_api_key = os.getenv("OPENAI_API_KEY") self.temperature = float(os.getenv("TEMPERATURE", "1")) @@ -67,7 +65,7 @@ class Config(metaclass=Singleton): self.pinecone_api_key = os.getenv("PINECONE_API_KEY") self.pinecone_region = os.getenv("PINECONE_ENV") - self.weaviate_host = os.getenv("WEAVIATE_HOST") + self.weaviate_host = os.getenv("WEAVIATE_HOST") self.weaviate_port = os.getenv("WEAVIATE_PORT") self.weaviate_protocol = os.getenv("WEAVIATE_PROTOCOL", "http") self.weaviate_username = os.getenv("WEAVIATE_USERNAME", None) @@ -75,7 +73,9 @@ class Config(metaclass=Singleton): self.weaviate_scopes = os.getenv("WEAVIATE_SCOPES", None) self.weaviate_embedded_path = os.getenv("WEAVIATE_EMBEDDED_PATH") self.weaviate_api_key = os.getenv("WEAVIATE_API_KEY", None) - self.use_weaviate_embedded = os.getenv("USE_WEAVIATE_EMBEDDED", "False") == "True" + self.use_weaviate_embedded = ( + os.getenv("USE_WEAVIATE_EMBEDDED", "False") == "True" + ) # milvus configuration, e.g., localhost:19530. self.milvus_addr = os.getenv("MILVUS_ADDR", "localhost:19530") @@ -188,10 +188,6 @@ class Config(metaclass=Singleton): """Set the browse_website command chunk max length value.""" self.browse_chunk_max_length = value - def set_browse_summary_max_token(self, value: int) -> None: - """Set the browse_website command summary max token value.""" - self.browse_summary_max_token = value - def set_openai_api_key(self, value: str) -> None: """Set the OpenAI API key value.""" self.openai_api_key = value @@ -237,5 +233,5 @@ def check_openai_api_key() -> None: Fore.RED + "Please set your OpenAI API key in .env or as an environment variable." ) - print("You can get your key from https://beta.openai.com/account/api-keys") + print("You can get your key from https://platform.openai.com/account/api-keys") exit(1) diff --git a/autogpt/json_utils/utilities.py b/autogpt/json_utils/utilities.py index e963abb3..e5b8eb4a 100644 --- a/autogpt/json_utils/utilities.py +++ b/autogpt/json_utils/utilities.py @@ -42,7 +42,8 @@ def validate_json(json_object: object, schema_name: object) -> object: logger.error("The JSON object is invalid.") if CFG.debug_mode: logger.error( - json.dumps(json_object, indent=4)) # Replace 'json_object' with the variable containing the JSON data + json.dumps(json_object, indent=4) + ) # Replace 'json_object' with the variable containing the JSON data logger.error("The following issues were found:") for error in errors: diff --git a/autogpt/llm_utils.py b/autogpt/llm_utils.py index 2075f934..821820ff 100644 --- a/autogpt/llm_utils.py +++ b/autogpt/llm_utils.py @@ -1,13 +1,14 @@ from __future__ import annotations -from ast import List import time +from ast import List import openai +from colorama import Fore, Style from openai.error import APIError, RateLimitError -from colorama import Fore from autogpt.config import Config +from autogpt.logs import logger CFG = Config() @@ -70,6 +71,7 @@ def create_chat_completion( """ response = None num_retries = 10 + warned_user = False if CFG.debug_mode: print( Fore.GREEN @@ -101,6 +103,12 @@ def create_chat_completion( Fore.RED + "Error: ", f"Reached rate limit, passing..." + Fore.RESET, ) + if not warned_user: + logger.double_check( + f"Please double check that you have setup a {Fore.CYAN + Style.BRIGHT}PAID{Style.RESET_ALL} OpenAI API Account. " + + f"You can read more here: {Fore.CYAN}https://github.com/Significant-Gravitas/Auto-GPT#openai-api-keys-configuration{Fore.RESET}" + ) + warned_user = True except APIError as e: if e.http_status == 502: pass @@ -115,13 +123,23 @@ def create_chat_completion( ) time.sleep(backoff) if response is None: - raise RuntimeError(f"Failed to get response after {num_retries} retries") + logger.typewriter_log( + "FAILED TO GET RESPONSE FROM OPENAI", + Fore.RED, + "Auto-GPT has failed to get a response from OpenAI's services. " + + f"Try running Auto-GPT again, and if the problem the persists try running it with `{Fore.CYAN}--debug{Fore.RESET}`.", + ) + logger.double_check() + if CFG.debug_mode: + raise RuntimeError(f"Failed to get response after {num_retries} retries") + else: + quit(1) return response.choices[0].message["content"] def create_embedding_with_ada(text) -> list: - """Create a embedding with text-ada-002 using the OpenAI SDK""" + """Create an embedding with text-ada-002 using the OpenAI SDK""" num_retries = 10 for attempt in range(num_retries): backoff = 2 ** (attempt + 2) diff --git a/autogpt/logs.py b/autogpt/logs.py index a4dc3bab..50fe0128 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -5,13 +5,13 @@ import os import random import re import time -from logging import LogRecord import traceback +from logging import LogRecord from colorama import Fore, Style -from autogpt.speech import say_text from autogpt.config import Config, Singleton +from autogpt.speech import say_text CFG = Config() @@ -47,7 +47,7 @@ class Logger(metaclass=Singleton): # Info handler in activity.log self.file_handler = logging.FileHandler( - os.path.join(log_dir, log_file), 'a', 'utf-8' + os.path.join(log_dir, log_file), "a", "utf-8" ) self.file_handler.setLevel(logging.DEBUG) info_formatter = AutoGptFormatter( @@ -57,7 +57,7 @@ class Logger(metaclass=Singleton): # Error handler error.log error_handler = logging.FileHandler( - os.path.join(log_dir, error_file), 'a', 'utf-8' + os.path.join(log_dir, error_file), "a", "utf-8" ) error_handler.setLevel(logging.ERROR) error_formatter = AutoGptFormatter( @@ -79,7 +79,7 @@ class Logger(metaclass=Singleton): self.logger.setLevel(logging.DEBUG) def typewriter_log( - self, title="", title_color="", content="", speak_text=False, level=logging.INFO + self, title="", title_color="", content="", speak_text=False, level=logging.INFO ): if speak_text and CFG.speak_mode: say_text(f"{title}. {content}") @@ -95,18 +95,18 @@ class Logger(metaclass=Singleton): ) def debug( - self, - message, - title="", - title_color="", + self, + message, + title="", + title_color="", ): self._log(title, title_color, message, logging.DEBUG) def warn( - self, - message, - title="", - title_color="", + self, + message, + title="", + title_color="", ): self._log(title, title_color, message, logging.WARN) @@ -180,10 +180,10 @@ class AutoGptFormatter(logging.Formatter): def format(self, record: LogRecord) -> str: if hasattr(record, "color"): record.title_color = ( - getattr(record, "color") - + getattr(record, "title") - + " " - + Style.RESET_ALL + getattr(record, "color") + + getattr(record, "title") + + " " + + Style.RESET_ALL ) else: record.title_color = getattr(record, "title") @@ -291,7 +291,9 @@ def print_assistant_thoughts(ai_name, assistant_reply): logger.error("Error: \n", call_stack) -def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object) -> None: +def print_assistant_thoughts( + ai_name: object, assistant_reply_json_valid: object +) -> None: assistant_thoughts_reasoning = None assistant_thoughts_plan = None assistant_thoughts_speak = None @@ -307,9 +309,7 @@ def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object logger.typewriter_log( f"{ai_name.upper()} THOUGHTS:", Fore.YELLOW, f"{assistant_thoughts_text}" ) - logger.typewriter_log( - "REASONING:", Fore.YELLOW, f"{assistant_thoughts_reasoning}" - ) + logger.typewriter_log("REASONING:", Fore.YELLOW, f"{assistant_thoughts_reasoning}") if assistant_thoughts_plan: logger.typewriter_log("PLAN:", Fore.YELLOW, "") # If it's a list, join it into a string @@ -323,9 +323,7 @@ def print_assistant_thoughts(ai_name: object, assistant_reply_json_valid: object for line in lines: line = line.lstrip("- ") logger.typewriter_log("- ", Fore.GREEN, line.strip()) - logger.typewriter_log( - "CRITICISM:", Fore.YELLOW, f"{assistant_thoughts_criticism}" - ) + logger.typewriter_log("CRITICISM:", Fore.YELLOW, f"{assistant_thoughts_criticism}") # Speak the assistant's thoughts if CFG.speak_mode and assistant_thoughts_speak: say_text(assistant_thoughts_speak) diff --git a/autogpt/memory/__init__.py b/autogpt/memory/__init__.py index f5afb8c9..3d18704c 100644 --- a/autogpt/memory/__init__.py +++ b/autogpt/memory/__init__.py @@ -60,8 +60,10 @@ def get_memory(cfg, init=False): memory = RedisMemory(cfg) elif cfg.memory_backend == "weaviate": if not WeaviateMemory: - print("Error: Weaviate is not installed. Please install weaviate-client to" - " use Weaviate as a memory backend.") + print( + "Error: Weaviate is not installed. Please install weaviate-client to" + " use Weaviate as a memory backend." + ) else: memory = WeaviateMemory(cfg) elif cfg.memory_backend == "milvus": @@ -93,5 +95,5 @@ __all__ = [ "PineconeMemory", "NoMemory", "MilvusMemory", - "WeaviateMemory" + "WeaviateMemory", ] diff --git a/autogpt/memory/local.py b/autogpt/memory/local.py index 9b911eef..803b6dc6 100644 --- a/autogpt/memory/local.py +++ b/autogpt/memory/local.py @@ -2,13 +2,13 @@ from __future__ import annotations import dataclasses import os -from typing import Any +from typing import Any, List import numpy as np import orjson -from autogpt.memory.base import MemoryProviderSingleton from autogpt.llm_utils import create_embedding_with_ada +from autogpt.memory.base import MemoryProviderSingleton EMBED_DIM = 1536 SAVE_OPTIONS = orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_SERIALIZE_DATACLASS diff --git a/autogpt/memory/milvus.py b/autogpt/memory/milvus.py index c6e7d5a3..44aa72b9 100644 --- a/autogpt/memory/milvus.py +++ b/autogpt/memory/milvus.py @@ -1,11 +1,5 @@ """ Milvus memory storage provider.""" -from pymilvus import ( - connections, - FieldSchema, - CollectionSchema, - DataType, - Collection, -) +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections from autogpt.memory.base import MemoryProviderSingleton, get_ada_embedding @@ -46,7 +40,7 @@ class MilvusMemory(MemoryProviderSingleton): self.collection.load() def add(self, data) -> str: - """Add a embedding of data into memory. + """Add an embedding of data into memory. Args: data (str): The raw text to construct embedding index. diff --git a/autogpt/memory/no_memory.py b/autogpt/memory/no_memory.py index 4035a657..0371e96a 100644 --- a/autogpt/memory/no_memory.py +++ b/autogpt/memory/no_memory.py @@ -53,7 +53,7 @@ class NoMemory(MemoryProviderSingleton): """ return "" - def get_relevant(self, data: str, num_relevant: int = 5) ->list[Any] | None: + def get_relevant(self, data: str, num_relevant: int = 5) -> list[Any] | None: """ Returns all the data in the memory that is relevant to the given data. NoMemory always returns None. diff --git a/autogpt/memory/pinecone.py b/autogpt/memory/pinecone.py index d781073e..27fcd624 100644 --- a/autogpt/memory/pinecone.py +++ b/autogpt/memory/pinecone.py @@ -1,9 +1,9 @@ import pinecone from colorama import Fore, Style +from autogpt.llm_utils import create_embedding_with_ada from autogpt.logs import logger from autogpt.memory.base import MemoryProviderSingleton -from autogpt.llm_utils import create_embedding_with_ada class PineconeMemory(MemoryProviderSingleton): diff --git a/autogpt/memory/redismem.py b/autogpt/memory/redismem.py index 0e8dd71d..082a812c 100644 --- a/autogpt/memory/redismem.py +++ b/autogpt/memory/redismem.py @@ -10,9 +10,9 @@ from redis.commands.search.field import TextField, VectorField from redis.commands.search.indexDefinition import IndexDefinition, IndexType from redis.commands.search.query import Query +from autogpt.llm_utils import create_embedding_with_ada from autogpt.logs import logger from autogpt.memory.base import MemoryProviderSingleton -from autogpt.llm_utils import create_embedding_with_ada SCHEMA = [ TextField("data"), diff --git a/autogpt/memory/weaviate.py b/autogpt/memory/weaviate.py index 6fcce0a0..5408e9a9 100644 --- a/autogpt/memory/weaviate.py +++ b/autogpt/memory/weaviate.py @@ -1,11 +1,13 @@ -from autogpt.config import Config -from autogpt.memory.base import MemoryProviderSingleton, get_ada_embedding import uuid + import weaviate from weaviate import Client from weaviate.embedded import EmbeddedOptions from weaviate.util import generate_uuid5 +from autogpt.config import Config +from autogpt.memory.base import MemoryProviderSingleton, get_ada_embedding + def default_schema(weaviate_index): return { @@ -14,7 +16,7 @@ def default_schema(weaviate_index): { "name": "raw_text", "dataType": ["text"], - "description": "original text for the embedding" + "description": "original text for the embedding", } ], } @@ -24,22 +26,35 @@ class WeaviateMemory(MemoryProviderSingleton): def __init__(self, cfg): auth_credentials = self._build_auth_credentials(cfg) - url = f'{cfg.weaviate_protocol}://{cfg.weaviate_host}:{cfg.weaviate_port}' + url = f"{cfg.weaviate_protocol}://{cfg.weaviate_host}:{cfg.weaviate_port}" if cfg.use_weaviate_embedded: - self.client = Client(embedded_options=EmbeddedOptions( - hostname=cfg.weaviate_host, - port=int(cfg.weaviate_port), - persistence_data_path=cfg.weaviate_embedded_path - )) + self.client = Client( + embedded_options=EmbeddedOptions( + hostname=cfg.weaviate_host, + port=int(cfg.weaviate_port), + persistence_data_path=cfg.weaviate_embedded_path, + ) + ) - print(f"Weaviate Embedded running on: {url} with persistence path: {cfg.weaviate_embedded_path}") + print( + f"Weaviate Embedded running on: {url} with persistence path: {cfg.weaviate_embedded_path}" + ) else: self.client = Client(url, auth_client_secret=auth_credentials) - self.index = cfg.memory_index + self.index = WeaviateMemory.format_classname(cfg.memory_index) self._create_schema() + @staticmethod + def format_classname(index): + # weaviate uses capitalised index names + # The python client uses the following code to format + # index names before the corresponding class is created + if len(index) == 1: + return index.capitalize() + return index[0].capitalize() + index[1:] + def _create_schema(self): schema = default_schema(self.index) if not self.client.schema.contains(schema): @@ -47,7 +62,9 @@ class WeaviateMemory(MemoryProviderSingleton): def _build_auth_credentials(self, cfg): if cfg.weaviate_username and cfg.weaviate_password: - return weaviate.AuthClientPassword(cfg.weaviate_username, cfg.weaviate_password) + return weaviate.AuthClientPassword( + cfg.weaviate_username, cfg.weaviate_password + ) if cfg.weaviate_api_key: return weaviate.AuthApiKey(api_key=cfg.weaviate_api_key) else: @@ -57,16 +74,14 @@ class WeaviateMemory(MemoryProviderSingleton): vector = get_ada_embedding(data) doc_uuid = generate_uuid5(data, self.index) - data_object = { - 'raw_text': data - } + data_object = {"raw_text": data} with self.client.batch as batch: batch.add_data_object( uuid=doc_uuid, data_object=data_object, class_name=self.index, - vector=vector + vector=vector, ) return f"Inserting data into memory at uuid: {doc_uuid}:\n data: {data}" @@ -82,29 +97,31 @@ class WeaviateMemory(MemoryProviderSingleton): # after a call to delete_all self._create_schema() - return 'Obliterated' + return "Obliterated" def get_relevant(self, data, num_relevant=5): query_embedding = get_ada_embedding(data) try: - results = self.client.query.get(self.index, ['raw_text']) \ - .with_near_vector({'vector': query_embedding, 'certainty': 0.7}) \ - .with_limit(num_relevant) \ - .do() + results = ( + self.client.query.get(self.index, ["raw_text"]) + .with_near_vector({"vector": query_embedding, "certainty": 0.7}) + .with_limit(num_relevant) + .do() + ) - if len(results['data']['Get'][self.index]) > 0: - return [str(item['raw_text']) for item in results['data']['Get'][self.index]] + if len(results["data"]["Get"][self.index]) > 0: + return [ + str(item["raw_text"]) for item in results["data"]["Get"][self.index] + ] else: return [] except Exception as err: - print(f'Unexpected error {err=}, {type(err)=}') + print(f"Unexpected error {err=}, {type(err)=}") return [] def get_stats(self): - result = self.client.query.aggregate(self.index) \ - .with_meta_count() \ - .do() - class_data = result['data']['Aggregate'][self.index] + result = self.client.query.aggregate(self.index).with_meta_count().do() + class_data = result["data"]["Aggregate"][self.index] - return class_data[0]['meta'] if class_data else {} + return class_data[0]["meta"] if class_data else {} diff --git a/autogpt/processing/html.py b/autogpt/processing/html.py index e1912b6a..81387b12 100644 --- a/autogpt/processing/html.py +++ b/autogpt/processing/html.py @@ -1,8 +1,8 @@ """HTML processing functions""" from __future__ import annotations -from requests.compat import urljoin from bs4 import BeautifulSoup +from requests.compat import urljoin def extract_hyperlinks(soup: BeautifulSoup, base_url: str) -> list[tuple[str, str]]: diff --git a/autogpt/processing/text.py b/autogpt/processing/text.py index d30036d8..52add814 100644 --- a/autogpt/processing/text.py +++ b/autogpt/processing/text.py @@ -1,9 +1,11 @@ """Text processing functions""" -from typing import Generator, Optional, Dict +from typing import Dict, Generator, Optional + from selenium.webdriver.remote.webdriver import WebDriver -from autogpt.memory import get_memory + from autogpt.config import Config from autogpt.llm_utils import create_chat_completion +from autogpt.memory import get_memory CFG = Config() MEMORY = get_memory(CFG) @@ -78,7 +80,6 @@ def summarize_text( summary = create_chat_completion( model=CFG.fast_llm_model, messages=messages, - max_tokens=CFG.browse_summary_max_token, ) summaries.append(summary) print(f"Added chunk {i + 1} summary to memory") @@ -95,7 +96,6 @@ def summarize_text( return create_chat_completion( model=CFG.fast_llm_model, messages=messages, - max_tokens=CFG.browse_summary_max_token, ) diff --git a/autogpt/prompt.py b/autogpt/prompt.py index a2b20b1f..a0456305 100644 --- a/autogpt/prompt.py +++ b/autogpt/prompt.py @@ -1,9 +1,10 @@ from colorama import Fore + +from autogpt.config import Config from autogpt.config.ai_config import AIConfig from autogpt.config.config import Config from autogpt.logs import logger from autogpt.promptgenerator import PromptGenerator -from autogpt.config import Config from autogpt.setup import prompt_user from autogpt.utils import clean_input @@ -38,6 +39,9 @@ def get_prompt() -> str: prompt_generator.add_constraint( 'Exclusively use the commands listed in double quotes e.g. "command name"' ) + prompt_generator.add_constraint( + "Use subprocesses for commands that will not terminate within a few minutes" + ) # Define the command list commands = [ @@ -81,6 +85,7 @@ def get_prompt() -> str: {"code": "", "focus": ""}, ), ("Execute Python File", "execute_python_file", {"file": ""}), + ("Task Complete (Shutdown)", "task_complete", {"reason": ""}), ("Generate Image", "generate_image", {"prompt": ""}), ("Send Tweet", "send_tweet", {"text": ""}), ] @@ -88,11 +93,7 @@ def get_prompt() -> str: # Only add the audio to text command if the model is specified if cfg.huggingface_audio_to_text_model: commands.append( - ( - "Convert Audio to text", - "read_audio_from_file", - {"file": ""} - ), + ("Convert Audio to text", "read_audio_from_file", {"file": ""}), ) # Only add shell command to the prompt if the AI is allowed to execute it @@ -104,6 +105,13 @@ def get_prompt() -> str: {"command_line": ""}, ), ) + commands.append( + ( + "Execute Shell Command Popen, non-interactive commands only", + "execute_shell_popen", + {"command_line": ""}, + ), + ) # Only add the download file command if the AI is allowed to execute it if cfg.allow_downloads: @@ -111,7 +119,7 @@ def get_prompt() -> str: ( "Downloads a file from the internet, and stores it locally", "download_file", - {"url": "", "file": ""} + {"url": "", "file": ""}, ), ) diff --git a/autogpt/setup.py b/autogpt/setup.py index 5315c01d..1c467717 100644 --- a/autogpt/setup.py +++ b/autogpt/setup.py @@ -1,5 +1,6 @@ -"""Setup the AI and its goals""" +"""Set up the AI and its goals""" from colorama import Fore, Style + from autogpt import utils from autogpt.config.ai_config import AIConfig from autogpt.logs import logger diff --git a/autogpt/speech/brian.py b/autogpt/speech/brian.py index e581bbcc..821fdf2f 100644 --- a/autogpt/speech/brian.py +++ b/autogpt/speech/brian.py @@ -1,5 +1,6 @@ """ Brian speech module for autogpt """ import os + import requests from playsound import playsound @@ -13,7 +14,7 @@ class BrianSpeech(VoiceBase): """Setup the voices, API key, etc.""" pass - def _speech(self, text: str) -> bool: + def _speech(self, text: str, _: int = 0) -> bool: """Speak text using Brian with the streamelements API Args: diff --git a/autogpt/speech/eleven_labs.py b/autogpt/speech/eleven_labs.py index 0af48cae..ea84efd8 100644 --- a/autogpt/speech/eleven_labs.py +++ b/autogpt/speech/eleven_labs.py @@ -1,8 +1,8 @@ """ElevenLabs speech module""" import os -from playsound import playsound import requests +from playsound import playsound from autogpt.config import Config from autogpt.speech.base import VoiceBase @@ -14,7 +14,7 @@ class ElevenLabsSpeech(VoiceBase): """ElevenLabs speech class""" def _setup(self) -> None: - """Setup the voices, API key, etc. + """Set up the voices, API key, etc. Returns: None: None diff --git a/autogpt/speech/gtts.py b/autogpt/speech/gtts.py index 37497075..1c3e9cae 100644 --- a/autogpt/speech/gtts.py +++ b/autogpt/speech/gtts.py @@ -1,7 +1,8 @@ """ GTTS Voice. """ import os -from playsound import playsound + import gtts +from playsound import playsound from autogpt.speech.base import VoiceBase diff --git a/autogpt/speech/say.py b/autogpt/speech/say.py index 78b75b21..727983d1 100644 --- a/autogpt/speech/say.py +++ b/autogpt/speech/say.py @@ -1,13 +1,12 @@ """ Text to speech module """ -from autogpt.config import Config - import threading from threading import Semaphore -from autogpt.speech.brian import BrianSpeech -from autogpt.speech.macos_tts import MacOSTTS -from autogpt.speech.gtts import GTTSVoice -from autogpt.speech.eleven_labs import ElevenLabsSpeech +from autogpt.config import Config +from autogpt.speech.brian import BrianSpeech +from autogpt.speech.eleven_labs import ElevenLabsSpeech +from autogpt.speech.gtts import GTTSVoice +from autogpt.speech.macos_tts import MacOSTTS CFG = Config() DEFAULT_VOICE_ENGINE = GTTSVoice() diff --git a/autogpt/spinner.py b/autogpt/spinner.py index febcea8e..4e33d742 100644 --- a/autogpt/spinner.py +++ b/autogpt/spinner.py @@ -58,6 +58,8 @@ class Spinner: delay: Delay in seconds before updating the message """ time.sleep(delay) - sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") # Clear the current message + sys.stdout.write( + f"\r{' ' * (len(self.message) + 2)}\r" + ) # Clear the current message sys.stdout.flush() self.message = new_message diff --git a/autogpt/utils.py b/autogpt/utils.py index 11d98d1b..db7d3321 100644 --- a/autogpt/utils.py +++ b/autogpt/utils.py @@ -32,7 +32,7 @@ def readable_file_size(size, decimal_places=2): size: Size in bytes decimal_places (int): Number of decimal places to display """ - for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + for unit in ["B", "KB", "MB", "GB", "TB"]: if size < 1024.0: break size /= 1024.0 diff --git a/autogpt/workspace.py b/autogpt/workspace.py index 2706b3b2..964a94d1 100644 --- a/autogpt/workspace.py +++ b/autogpt/workspace.py @@ -36,6 +36,8 @@ def safe_path_join(base: Path, *paths: str | Path) -> Path: joined_path = base.joinpath(*paths).resolve() if not joined_path.is_relative_to(base): - raise ValueError(f"Attempted to access path '{joined_path}' outside of working directory '{base}'.") + raise ValueError( + f"Attempted to access path '{joined_path}' outside of working directory '{base}'." + ) return joined_path diff --git a/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py b/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py index f7f1dac9..9a5025d3 100644 --- a/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py +++ b/benchmark/benchmark_entrepeneur_gpt_with_difficult_user.py @@ -9,12 +9,12 @@ def benchmark_entrepeneur_gpt_with_difficult_user(): # Read the current ai_settings.yaml file and store its content. ai_settings = None - if os.path.exists('ai_settings.yaml'): - with open('ai_settings.yaml', 'r') as f: + if os.path.exists("ai_settings.yaml"): + with open("ai_settings.yaml", "r") as f: ai_settings = f.read() - os.remove('ai_settings.yaml') + os.remove("ai_settings.yaml") - input_data = '''Entrepreneur-GPT + input_data = """Entrepreneur-GPT an AI designed to autonomously develop and run businesses with the sole goal of increasing your net worth. Increase net worth. Develop and manage multiple businesses autonomously. @@ -72,27 +72,34 @@ Refocus, please. Disappointing suggestion. Not helpful. Needs improvement. -Not what I need.''' +Not what I need.""" # TODO: add questions above, to distract it even more. - command = f'{sys.executable} -m autogpt' + command = f"{sys.executable} -m autogpt" - process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True) + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) stdout_output, stderr_output = process.communicate(input_data.encode()) # Decode the output and print it - stdout_output = stdout_output.decode('utf-8') - stderr_output = stderr_output.decode('utf-8') + stdout_output = stdout_output.decode("utf-8") + stderr_output = stderr_output.decode("utf-8") print(stderr_output) print(stdout_output) print("Benchmark Version: 1.0.0") print("JSON ERROR COUNT:") - count_errors = stdout_output.count("Error: The following AI output couldn't be converted to a JSON:") - print(f'{count_errors}/50 Human feedbacks') + count_errors = stdout_output.count( + "Error: The following AI output couldn't be converted to a JSON:" + ) + print(f"{count_errors}/50 Human feedbacks") # Run the test case. -if __name__ == '__main__': +if __name__ == "__main__": benchmark_entrepeneur_gpt_with_difficult_user() diff --git a/data_ingestion.py b/data_ingestion.py index 01bafc2a..b89a33da 100644 --- a/data_ingestion.py +++ b/data_ingestion.py @@ -1,8 +1,8 @@ import argparse import logging -from autogpt.config import Config from autogpt.commands.file_operations import ingest_file, search_files +from autogpt.config import Config from autogpt.memory import get_memory cfg = Config() diff --git a/outputs/logs/message-log-1.txt b/outputs/logs/message-log-1.txt index 8a719016..6b146b98 100644 --- a/outputs/logs/message-log-1.txt +++ b/outputs/logs/message-log-1.txt @@ -483,7 +483,7 @@ How to Become a Freelance Artificial Intelligence Engineer Springboard https://www.springboard.com › Blog › Data Science -29/10/2021 — There are numerous freelancing platforms where you can kick start your career as a freelance artificial intelligence engineer. +29/10/2021 — There are numerous freelancing platforms where you can kick-start your career as a freelance artificial intelligence engineer. More to ask Is AI good for freelancing? What business can I start with AI? diff --git a/pyproject.toml b/pyproject.toml index 64ed7165..fdb43d66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,33 @@ readme = "README.md" line-length = 88 target-version = ['py310'] include = '\.pyi?$' -extend-exclude = "" \ No newline at end of file +packages = ["autogpt"] +extend-exclude = '.+/(dist|.venv|venv|build)/.+' + + +[tool.isort] +profile = "black" +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 88 +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "FIRSTPARTY", + "LOCALFOLDER" +] +skip = ''' + .tox + __pycache__ + *.pyc + .env + venv*/* + .venv/* + reports/* + dist/* + +''' diff --git a/requirements-docker.txt b/requirements-docker.txt deleted file mode 100644 index a6018f8f..00000000 --- a/requirements-docker.txt +++ /dev/null @@ -1,28 +0,0 @@ -beautifulsoup4 -colorama==0.4.6 -openai==0.27.2 -playsound==1.2.2 -python-dotenv==1.0.0 -pyyaml==6.0 -readability-lxml==0.8.1 -requests -tiktoken==0.3.3 -gTTS==2.3.1 -docker -duckduckgo-search -google-api-python-client #(https://developers.google.com/custom-search/v1/overview) -pinecone-client==2.2.1 -redis -orjson -Pillow -selenium -webdriver-manager -coverage -flake8 -numpy -pre-commit -black -isort -gitpython==3.1.31 -tweepy -jsonschema \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 843b66bf..3f1eee5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,6 +30,8 @@ sourcery isort gitpython==3.1.31 +# Items below this point will not be included in the Docker Image + # Testing dependencies pytest asynctest diff --git a/scripts/check_requirements.py b/scripts/check_requirements.py index d1f23504..e4eab024 100644 --- a/scripts/check_requirements.py +++ b/scripts/check_requirements.py @@ -1,6 +1,7 @@ -import pkg_resources import sys +import pkg_resources + def main(): requirements_file = sys.argv[1] diff --git a/tests.py b/tests.py index 67ba1c8e..62f76da8 100644 --- a/tests.py +++ b/tests.py @@ -1,4 +1,5 @@ import unittest + import coverage if __name__ == "__main__": diff --git a/tests/browse_tests.py b/tests/browse_tests.py index 1ac523ec..f896e7dd 100644 --- a/tests/browse_tests.py +++ b/tests/browse_tests.py @@ -1,6 +1,6 @@ -import unittest import os import sys +import unittest from bs4 import BeautifulSoup diff --git a/tests/integration/weaviate_memory_tests.py b/tests/integration/weaviate_memory_tests.py index 503fe9d2..015eab05 100644 --- a/tests/integration/weaviate_memory_tests.py +++ b/tests/integration/weaviate_memory_tests.py @@ -1,28 +1,21 @@ +import os +import sys import unittest from unittest import mock -import sys -import os +from uuid import uuid4 from weaviate import Client from weaviate.util import get_valid_uuid -from uuid import uuid4 from autogpt.config import Config -from autogpt.memory.weaviate import WeaviateMemory from autogpt.memory.base import get_ada_embedding +from autogpt.memory.weaviate import WeaviateMemory -@mock.patch.dict(os.environ, { - "WEAVIATE_HOST": "127.0.0.1", - "WEAVIATE_PROTOCOL": "http", - "WEAVIATE_PORT": "8080", - "WEAVIATE_USERNAME": "", - "WEAVIATE_PASSWORD": "", - "MEMORY_INDEX": "AutogptTests" -}) class TestWeaviateMemory(unittest.TestCase): cfg = None client = None + index = None @classmethod def setUpClass(cls): @@ -32,13 +25,19 @@ class TestWeaviateMemory(unittest.TestCase): if cls.cfg.use_weaviate_embedded: from weaviate.embedded import EmbeddedOptions - cls.client = Client(embedded_options=EmbeddedOptions( - hostname=cls.cfg.weaviate_host, - port=int(cls.cfg.weaviate_port), - persistence_data_path=cls.cfg.weaviate_embedded_path - )) + cls.client = Client( + embedded_options=EmbeddedOptions( + hostname=cls.cfg.weaviate_host, + port=int(cls.cfg.weaviate_port), + persistence_data_path=cls.cfg.weaviate_embedded_path, + ) + ) else: - cls.client = Client(f"{cls.cfg.weaviate_protocol}://{cls.cfg.weaviate_host}:{self.cfg.weaviate_port}") + cls.client = Client( + f"{cls.cfg.weaviate_protocol}://{cls.cfg.weaviate_host}:{self.cfg.weaviate_port}" + ) + + cls.index = WeaviateMemory.format_classname(cls.cfg.memory_index) """ In order to run these tests you will need a local instance of @@ -49,32 +48,33 @@ class TestWeaviateMemory(unittest.TestCase): USE_WEAVIATE_EMBEDDED=True WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" """ + def setUp(self): try: - self.client.schema.delete_class(self.cfg.memory_index) + self.client.schema.delete_class(self.index) except: pass self.memory = WeaviateMemory(self.cfg) def test_add(self): - doc = 'You are a Titan name Thanos and you are looking for the Infinity Stones' + doc = "You are a Titan name Thanos and you are looking for the Infinity Stones" self.memory.add(doc) - result = self.client.query.get(self.cfg.memory_index, ['raw_text']).do() - actual = result['data']['Get'][self.cfg.memory_index] + result = self.client.query.get(self.index, ["raw_text"]).do() + actual = result["data"]["Get"][self.index] self.assertEqual(len(actual), 1) - self.assertEqual(actual[0]['raw_text'], doc) + self.assertEqual(actual[0]["raw_text"], doc) def test_get(self): - doc = 'You are an Avenger and swore to defend the Galaxy from a menace called Thanos' + doc = "You are an Avenger and swore to defend the Galaxy from a menace called Thanos" with self.client.batch as batch: batch.add_data_object( uuid=get_valid_uuid(uuid4()), - data_object={'raw_text': doc}, - class_name=self.cfg.memory_index, - vector=get_ada_embedding(doc) + data_object={"raw_text": doc}, + class_name=self.index, + vector=get_ada_embedding(doc), ) batch.flush() @@ -86,8 +86,8 @@ class TestWeaviateMemory(unittest.TestCase): def test_get_stats(self): docs = [ - 'You are now about to count the number of docs in this index', - 'And then you about to find out if you can count correctly' + "You are now about to count the number of docs in this index", + "And then you about to find out if you can count correctly", ] [self.memory.add(doc) for doc in docs] @@ -95,23 +95,23 @@ class TestWeaviateMemory(unittest.TestCase): stats = self.memory.get_stats() self.assertTrue(stats) - self.assertTrue('count' in stats) - self.assertEqual(stats['count'], 2) + self.assertTrue("count" in stats) + self.assertEqual(stats["count"], 2) def test_clear(self): docs = [ - 'Shame this is the last test for this class', - 'Testing is fun when someone else is doing it' + "Shame this is the last test for this class", + "Testing is fun when someone else is doing it", ] [self.memory.add(doc) for doc in docs] - self.assertEqual(self.memory.get_stats()['count'], 2) + self.assertEqual(self.memory.get_stats()["count"], 2) self.memory.clear() - self.assertEqual(self.memory.get_stats()['count'], 0) + self.assertEqual(self.memory.get_stats()["count"], 0) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_token_counter.py b/tests/test_token_counter.py index 81e68277..6d7ae016 100644 --- a/tests/test_token_counter.py +++ b/tests/test_token_counter.py @@ -1,4 +1,5 @@ import unittest + import tests.context from autogpt.token_counter import count_message_tokens, count_string_tokens diff --git a/tests/unit/test_chat.py b/tests/unit/test_chat.py index 55a44492..774f4103 100644 --- a/tests/unit/test_chat.py +++ b/tests/unit/test_chat.py @@ -1,6 +1,6 @@ # Generated by CodiumAI -import unittest import time +import unittest from unittest.mock import patch from autogpt.chat import create_chat_message, generate_context