From 8cbe438ad5c24e4c8fde34fc4c7d4f52abf0f5ab Mon Sep 17 00:00:00 2001 From: "roby.parapat" Date: Sun, 16 Apr 2023 06:33:43 +0700 Subject: [PATCH 01/14] move comment to correct position --- autogpt/commands/execute_code.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autogpt/commands/execute_code.py b/autogpt/commands/execute_code.py index 86d6c177..f174b7bc 100644 --- a/autogpt/commands/execute_code.py +++ b/autogpt/commands/execute_code.py @@ -41,6 +41,9 @@ def execute_python_file(file: str): try: client = docker.from_env() + # You can replace 'python:3.8' with the desired Python image/version + # You can find available Python images on Docker Hub: + # https://hub.docker.com/_/python image_name = "python:3.10" try: client.images.get(image_name) @@ -58,9 +61,6 @@ def execute_python_file(file: str): elif status: print(status) - # You can replace 'python:3.8' with the desired Python image/version - # You can find available Python images on Docker Hub: - # https://hub.docker.com/_/python container = client.containers.run( image_name, f"python {file}", From 7b7d7c1d74b299966e607cf7dc6cf2cea64993ba Mon Sep 17 00:00:00 2001 From: Bates Jernigan Date: Sun, 16 Apr 2023 16:33:52 -0400 Subject: [PATCH 02/14] add space on warning message --- autogpt/memory/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/memory/local.py b/autogpt/memory/local.py index 6c7ee1b3..9b911eef 100644 --- a/autogpt/memory/local.py +++ b/autogpt/memory/local.py @@ -54,7 +54,7 @@ class LocalCache(MemoryProviderSingleton): self.data = CacheContent() else: print( - f"Warning: The file '{self.filename}' does not exist." + f"Warning: The file '{self.filename}' does not exist. " "Local memory would not be saved to a file." ) self.data = CacheContent() From 4269326ddfd81227e78b0745093f52e4ac1ba078 Mon Sep 17 00:00:00 2001 From: 0xf333 <0x333@tuta.io> Date: Sun, 16 Apr 2023 17:03:18 -0400 Subject: [PATCH 03/14] Fix: Update run_continuous.sh to pass all command-line arguments Description: - Modified `run_continuous.sh` to include the `--continuous` flag directly in the command: - Removed the unused `argument` variable. - Added the `--continuous` flag to the `./run.sh` command. - Ensured all command-line arguments are passed through to `run.sh` and the `autogpt` module. This change improves the usability of the `run_continuous.sh` script by allowing users to provide additional command-line arguments along with the `--continuous` flag. It ensures that all arguments are properly passed to the `run.sh` script and eventually to the `autogpt` module, preventing confusion and providing more flexible usage. Suggestion from: https://github.com/Significant-Gravitas/Auto-GPT/pull/1941#discussion_r1167977442 --- run_continuous.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_continuous.sh b/run_continuous.sh index 14c9cfd2..43034f8e 100755 --- a/run_continuous.sh +++ b/run_continuous.sh @@ -1,3 +1,3 @@ #!/bin/bash -argument="--continuous" -./run.sh "$argument" + +./run.sh --continuous "$@" From 5ff7fc340b908281c6eb976358947e87f289c0f7 Mon Sep 17 00:00:00 2001 From: endolith Date: Sun, 16 Apr 2023 08:47:11 -0400 Subject: [PATCH 04/14] Remove extraneous noqa E722 comment E722 is "Do not use bare except, specify exception instead" but except json.JSONDecodeError is not a bare except --- autogpt/json_fixes/auto_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/json_fixes/auto_fix.py b/autogpt/json_fixes/auto_fix.py index 9fcf909a..0d3bd73c 100644 --- a/autogpt/json_fixes/auto_fix.py +++ b/autogpt/json_fixes/auto_fix.py @@ -45,7 +45,7 @@ def fix_json(json_string: str, schema: str) -> str: try: json.loads(result_string) # just check the validity return result_string - except json.JSONDecodeError: # noqa: E722 + except json.JSONDecodeError: # Get the call stack: # import traceback # call_stack = traceback.format_exc() From 4eb8e7823d63ff4f8d67b8927da842ea7ab3ab21 Mon Sep 17 00:00:00 2001 From: 0xf333 <0x333@tuta.io> Date: Sun, 16 Apr 2023 18:07:41 -0400 Subject: [PATCH 05/14] Fix: Remove quotes around $@ in run_continuous.sh Description: Per maintainer's request, removed quotes around `$@` in `run_continuous.sh`. This change allows the script to forward arguments as is. Please note that this modification might cause issues if any of the command-line arguments contain spaces or special characters. However, this update aligns with the preferred format for the repository. Suggestion from: https://github.com/Significant-Gravitas/Auto-GPT/pull/1941#discussion_r1168035557 --- run_continuous.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_continuous.sh b/run_continuous.sh index 43034f8e..1f4436c8 100755 --- a/run_continuous.sh +++ b/run_continuous.sh @@ -1,3 +1,3 @@ #!/bin/bash -./run.sh --continuous "$@" +./run.sh --continuous $@ From c71c61dc584a41d72e2b27b02fe75a9f64e3e029 Mon Sep 17 00:00:00 2001 From: Adrian Scott Date: Sun, 16 Apr 2023 18:14:16 -0500 Subject: [PATCH 06/14] Added one space after period for better formatting --- autogpt/memory/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/memory/local.py b/autogpt/memory/local.py index 6c7ee1b3..9b911eef 100644 --- a/autogpt/memory/local.py +++ b/autogpt/memory/local.py @@ -54,7 +54,7 @@ class LocalCache(MemoryProviderSingleton): self.data = CacheContent() else: print( - f"Warning: The file '{self.filename}' does not exist." + f"Warning: The file '{self.filename}' does not exist. " "Local memory would not be saved to a file." ) self.data = CacheContent() From 15059c2090be47d2a674113f509618b3f58a3510 Mon Sep 17 00:00:00 2001 From: Chris Cheney Date: Sun, 16 Apr 2023 17:28:25 -0500 Subject: [PATCH 07/14] ensure git operations occur in the working directory --- autogpt/commands/git_operations.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autogpt/commands/git_operations.py b/autogpt/commands/git_operations.py index 3ff35cf3..675eb228 100644 --- a/autogpt/commands/git_operations.py +++ b/autogpt/commands/git_operations.py @@ -1,6 +1,7 @@ """Git operations for autogpt""" import git from autogpt.config import Config +from autogpt.workspace import path_in_workspace CFG = Config() @@ -16,8 +17,9 @@ def clone_repository(repo_url: str, clone_path: str) -> str: str: The result of the clone operation""" split_url = repo_url.split("//") auth_repo_url = f"//{CFG.github_username}:{CFG.github_api_key}@".join(split_url) + safe_clone_path = path_in_workspace(clone_path) try: - git.Repo.clone_from(auth_repo_url, clone_path) - return f"""Cloned {repo_url} to {clone_path}""" + git.Repo.clone_from(auth_repo_url, safe_clone_path) + return f"""Cloned {repo_url} to {safe_clone_path}""" except Exception as e: return f"Error: {str(e)}" From 56ecbeeef734019d1d3112fc2a657a14bff69ccb Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Mon, 17 Apr 2023 02:22:18 +0200 Subject: [PATCH 08/14] Clean up README --- README.md | 227 ++++++++++++++++++++++++------------------------------ 1 file changed, 102 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index f60aa9ff..07a088e3 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,19 @@ # Auto-GPT: An Autonomous GPT-4 Experiment +[![Unit Tests](https://img.shields.io/github/actions/workflow/status/Significant-Gravitas/Auto-GPT/ci.yml?label=unit%20tests)](https://github.com/Significant-Gravitas/Auto-GPT/actions/workflows/ci.yml) +[![Discord Follow](https://dcbadge.vercel.app/api/server/autogpt?style=flat)](https://discord.gg/autogpt) +[![GitHub Repo stars](https://img.shields.io/github/stars/Significant-Gravitas/auto-gpt?style=social)](https://github.com/Significant-Gravitas/Auto-GPT/stargazers) +[![Twitter Follow](https://img.shields.io/twitter/follow/siggravitas?style=social)](https://twitter.com/SigGravitas) + + +
+ ### πŸ”΄ πŸ”΄ πŸ”΄ Urgent: USE `stable` not `master` πŸ”΄ πŸ”΄ πŸ”΄ -> This information takes precedence over any other information provided and should be followed accordingly. +**Download the latest `stable` release from here: https://github.com/Significant-Gravitas/Auto-GPT/releases/latest.** +The `master` branch may often be in a **broken** state. -Our workflow has been improved, but please note that `master` branch may often be in a **broken** state. -Please download the latest `stable` release from here: https://github.com/Torantulino/Auto-GPT/releases/latest. +
-![GitHub Repo stars](https://img.shields.io/github/stars/Significant-Gravitas/auto-gpt?style=social) -[![Twitter Follow](https://img.shields.io/twitter/follow/siggravitas?style=social)](https://twitter.com/SigGravitas) -[![Discord Follow](https://dcbadge.vercel.app/api/server/autogpt?style=flat)](https://discord.gg/autogpt) -[![Unit Tests](https://github.com/Significant-Gravitaso/Auto-GPT/actions/workflows/ci.yml/badge.svg)](https://github.com/Significant-Gravitas/Auto-GPT/actions/workflows/ci.yml) Auto-GPT is an experimental open-source application showcasing the capabilities of the GPT-4 language model. This program, driven by GPT-4, chains together LLM "thoughts", to autonomously achieve whatever goal you set. As one of the first examples of GPT-4 running fully autonomously, Auto-GPT pushes the boundaries of what is possible with AI. @@ -37,42 +41,6 @@ Development of this free, open-source project is made possible by all the Dradstone  CrypteorCapital  avy-ai  shawnharmsen  sunchongren  DailyBotHQ  mathewhawkins  MediConCenHK  kMag410  nicoguyon  Mobivs  jazgarewal  marv-technology  rapidstartup  Brodie0  lucas-chu  rejunity  comet-ml  ColinConwell  cfarquhar  ikarosai  ChrisDMT  Odin519Tomas  vkozacek  belharethsami  sultanmeghji  scryptedinc  johnculkin  RealChrisSean  fruition  jd3655  Web3Capital  allenstecat  tob-le-rone  SwftCoins  MetaPath01  joaomdmoura  ternary5  refinery1  josephcmiller2  webbcolton  tommygeee  lmaugustin  garythebat  Cameron-Fulton  angiaou  caitlynmeeks  MBassi91  Daniel1357  omphos  abhinav-pandey29  DataMetis  concreit  st617  RThaweewat  KiaArmani  Pythagora-io  AryaXAI  fabrietech  jun784  Mr-Bishop42  rickscode  projectonegames  rocks6  GalaxyVideoAgency  thisisjeffchen  TheStoneMX  txtr99  ZERO-A-ONE  

- - -## Table of Contents - -- [Auto-GPT: An Autonomous GPT-4 Experiment](#auto-gpt-an-autonomous-gpt-4-experiment) - - [πŸ”΄ πŸ”΄ πŸ”΄ Urgent: USE `stable` not `master` πŸ”΄ πŸ”΄ πŸ”΄](#----urgent-use-stable-not-master----) - - [Demo (30/03/2023):](#demo-30032023) - - [Table of Contents](#table-of-contents) - - [πŸš€ Features](#-features) - - [πŸ“‹ Requirements](#-requirements) - - [πŸ’Ύ Installation](#-installation) - - [πŸ”§ Usage](#-usage) - - [Logs](#logs) - - [Docker](#docker) - - [Command Line Arguments](#command-line-arguments) - - [πŸ—£οΈ Speech Mode](#️-speech-mode) - - [πŸ” Google API Keys Configuration](#-google-api-keys-configuration) - - [Setting up environment variables](#setting-up-environment-variables) - - [Memory Backend Setup](#memory-backend-setup) - - [Redis Setup](#redis-setup) - - [🌲 Pinecone API Key Setup](#-pinecone-api-key-setup) - - [Milvus Setup](#milvus-setup) - - [Weaviate Setup](#weaviate-setup) - - [Setting up environment variables](#setting-up-environment-variables-1) - - [Setting Your Cache Type](#setting-your-cache-type) - - [View Memory Usage](#view-memory-usage) - - [🧠 Memory pre-seeding](#-memory-pre-seeding) - - [πŸ’€ Continuous Mode ⚠️](#-continuous-mode-️) - - [GPT3.5 ONLY Mode](#gpt35-only-mode) - - [πŸ–Ό Image Generation](#-image-generation) - - [⚠️ Limitations](#️-limitations) - - [πŸ›‘ Disclaimer](#-disclaimer) - - [🐦 Connect with Us on Twitter](#-connect-with-us-on-twitter) - - [Run tests](#run-tests) - - [Run linter](#run-linter) - ## πŸš€ Features - 🌐 Internet access for searches and information gathering @@ -83,16 +51,17 @@ Development of this free, open-source project is made possible by all the

Blake Werlinger +

πŸ’– Help Fund Auto-GPT's Development πŸ’–

If you can spare a coffee, you can help to cover the costs of developing Auto-GPT and help push the boundaries of fully autonomous AI! From 9589334a305198c837bfb8720ed6f06176b2f216 Mon Sep 17 00:00:00 2001 From: EH Date: Mon, 17 Apr 2023 03:34:02 +0100 Subject: [PATCH 13/14] Add File Downloading Capabilities (#1680) * Added 'download_file' command * Added util and fixed spinner * Fixed comma and added autogpt/auto_gpt_workspace to .gitignore * Fix linter issues * Fix more linter issues * Fix Lint Issues * Added 'download_file' command * Added util and fixed spinner * Fixed comma and added autogpt/auto_gpt_workspace to .gitignore * Fix linter issues * Fix more linter issues * Conditionally add the 'download_file' prompt * Update args.py * Removed Duplicate Prompt * Switched to using path_in_workspace function --- .gitignore | 1 + autogpt/app.py | 5 +++ autogpt/args.py | 16 +++++++++- autogpt/commands/file_operations.py | 49 ++++++++++++++++++++++++++++- autogpt/config/config.py | 1 + autogpt/prompt.py | 10 ++++++ autogpt/spinner.py | 15 ++++++++- autogpt/utils.py | 13 ++++++++ 8 files changed, 107 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index eda7f327..2220ef6e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ autogpt/keys.py autogpt/*json autogpt/node_modules/ autogpt/__pycache__/keys.cpython-310.pyc +autogpt/auto_gpt_workspace package-lock.json *.pyc auto_gpt_workspace/* diff --git a/autogpt/app.py b/autogpt/app.py index 78b5bd2f..19c075f0 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -17,6 +17,7 @@ from autogpt.commands.file_operations import ( read_file, search_files, write_to_file, + download_file ) from autogpt.json_fixes.parsing import fix_and_parse_json from autogpt.memory import get_memory @@ -164,6 +165,10 @@ def execute_command(command_name: str, arguments): return delete_file(arguments["file"]) elif command_name == "search_files": return search_files(arguments["directory"]) + elif command_name == "download_file": + if not CFG.allow_downloads: + return "Error: You do not have user authorization to download files locally." + return download_file(arguments["url"], arguments["file"]) elif command_name == "browse_website": return browse_website(arguments["url"], arguments["question"]) # TODO: Change these to take in a file rather than pasted code, if diff --git a/autogpt/args.py b/autogpt/args.py index eca32334..f0e9c07a 100644 --- a/autogpt/args.py +++ b/autogpt/args.py @@ -1,7 +1,7 @@ """This module contains the argument parsing logic for the script.""" import argparse -from colorama import Fore +from colorama import Fore, Back, Style from autogpt import utils from autogpt.config import Config from autogpt.logs import logger @@ -63,6 +63,12 @@ def parse_arguments() -> None: help="Specifies which ai_settings.yaml file to use, will also automatically" " skip the re-prompt.", ) + parser.add_argument( + '--allow-downloads', + action='store_true', + dest='allow_downloads', + help='Dangerous: Allows Auto-GPT to download files natively.' + ) args = parser.parse_args() if args.debug: @@ -133,5 +139,13 @@ def parse_arguments() -> None: CFG.ai_settings_file = file CFG.skip_reprompt = True + if args.allow_downloads: + logger.typewriter_log("Native Downloading:", Fore.GREEN, "ENABLED") + logger.typewriter_log("WARNING: ", Fore.YELLOW, + f"{Back.LIGHTYELLOW_EX}Auto-GPT will now be able to download and save files to your machine.{Back.RESET} " + + "It is recommended that you monitor any files it downloads carefully.") + logger.typewriter_log("WARNING: ", Fore.YELLOW, f"{Back.RED + Style.BRIGHT}ALWAYS REMEMBER TO NEVER OPEN FILES YOU AREN'T SURE OF!{Style.RESET_ALL}") + CFG.allow_downloads = True + if args.browser_name: CFG.selenium_web_browser = args.browser_name diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index 8abc2e23..d273c1a3 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -4,9 +4,16 @@ from __future__ import annotations import os import os.path from pathlib import Path -from typing import Generator +from typing import Generator, List +import requests +from requests.adapters import HTTPAdapter +from requests.adapters import Retry +from colorama import Fore, Back +from autogpt.spinner import Spinner +from autogpt.utils import readable_file_size from autogpt.workspace import path_in_workspace, WORKSPACE_PATH + LOG_FILE = "file_logger.txt" LOG_FILE_PATH = WORKSPACE_PATH / LOG_FILE @@ -214,3 +221,43 @@ def search_files(directory: str) -> list[str]: found_files.append(relative_path) return found_files + + +def download_file(url, filename): + """Downloads a file + Args: + url (str): URL of the file to download + filename (str): Filename to save the file as + """ + safe_filename = path_in_workspace(filename) + try: + message = f"{Fore.YELLOW}Downloading file from {Back.LIGHTBLUE_EX}{url}{Back.RESET}{Fore.RESET}" + with Spinner(message) as spinner: + session = requests.Session() + retry = Retry(total=3, backoff_factor=1, status_forcelist=[502, 503, 504]) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + + total_size = 0 + downloaded_size = 0 + + with session.get(url, allow_redirects=True, stream=True) as r: + r.raise_for_status() + total_size = int(r.headers.get('Content-Length', 0)) + downloaded_size = 0 + + with open(safe_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + downloaded_size += len(chunk) + + # Update the progress message + progress = f"{readable_file_size(downloaded_size)} / {readable_file_size(total_size)}" + spinner.update_message(f"{message} {progress}") + + return f'Successfully downloaded and locally stored file: "{filename}"! (Size: {readable_file_size(total_size)})' + except requests.HTTPError as e: + return f"Got an HTTP Error whilst trying to download file: {e}" + except Exception as e: + return "Error: " + str(e) diff --git a/autogpt/config/config.py b/autogpt/config/config.py index 22da52b0..fe6f4f32 100644 --- a/autogpt/config/config.py +++ b/autogpt/config/config.py @@ -24,6 +24,7 @@ class Config(metaclass=Singleton): self.continuous_limit = 0 self.speak_mode = False self.skip_reprompt = False + self.allow_downloads = False self.selenium_web_browser = os.getenv("USE_WEB_BROWSER", "chrome") self.ai_settings_file = os.getenv("AI_SETTINGS_FILE", "ai_settings.yaml") diff --git a/autogpt/prompt.py b/autogpt/prompt.py index 18a5736c..a2b20b1f 100644 --- a/autogpt/prompt.py +++ b/autogpt/prompt.py @@ -105,6 +105,16 @@ def get_prompt() -> str: ), ) + # Only add the download file command if the AI is allowed to execute it + if cfg.allow_downloads: + commands.append( + ( + "Downloads a file from the internet, and stores it locally", + "download_file", + {"url": "", "file": ""} + ), + ) + # Add these command last. commands.append( ("Do Nothing", "do_nothing", {}), diff --git a/autogpt/spinner.py b/autogpt/spinner.py index 56b4f20a..febcea8e 100644 --- a/autogpt/spinner.py +++ b/autogpt/spinner.py @@ -29,12 +29,14 @@ class Spinner: time.sleep(self.delay) sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") - def __enter__(self) -> None: + def __enter__(self): """Start the spinner""" self.running = True self.spinner_thread = threading.Thread(target=self.spin) self.spinner_thread.start() + return self + def __exit__(self, exc_type, exc_value, exc_traceback) -> None: """Stop the spinner @@ -48,3 +50,14 @@ class Spinner: self.spinner_thread.join() sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") sys.stdout.flush() + + def update_message(self, new_message, delay=0.1): + """Update the spinner message + Args: + new_message (str): New message to display + delay: Delay in seconds before updating the message + """ + time.sleep(delay) + sys.stdout.write(f"\r{' ' * (len(self.message) + 2)}\r") # Clear the current message + sys.stdout.flush() + self.message = new_message diff --git a/autogpt/utils.py b/autogpt/utils.py index 59709d02..11d98d1b 100644 --- a/autogpt/utils.py +++ b/autogpt/utils.py @@ -24,3 +24,16 @@ def validate_yaml_file(file: str): ) return (True, f"Successfully validated {Fore.CYAN}`{file}`{Fore.RESET}!") + + +def readable_file_size(size, decimal_places=2): + """Converts the given size in bytes to a readable format. + Args: + size: Size in bytes + decimal_places (int): Number of decimal places to display + """ + for unit in ['B', 'KB', 'MB', 'GB', 'TB']: + if size < 1024.0: + break + size /= 1024.0 + return f"{size:.{decimal_places}f} {unit}" From 0fa807394711010a17fe37a3afbce81978e233e2 Mon Sep 17 00:00:00 2001 From: Ben Song Date: Mon, 17 Apr 2023 11:53:05 +0800 Subject: [PATCH 14/14] add docker requirements - jsonschema --- requirements-docker.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-docker.txt b/requirements-docker.txt index 3a8a344c..a6018f8f 100644 --- a/requirements-docker.txt +++ b/requirements-docker.txt @@ -24,4 +24,5 @@ pre-commit black isort gitpython==3.1.31 -tweepy \ No newline at end of file +tweepy +jsonschema \ No newline at end of file