From d53bd020ea7244a0246d900640f1ab377224384e Mon Sep 17 00:00:00 2001 From: Duncan Roosma Date: Sat, 15 Apr 2023 11:42:12 +0200 Subject: [PATCH 1/7] Update docker-compose.yml Needed to add this to get `docker-compose run auto-gpt` to run successfully --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 79f20bb5..be225d7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,8 @@ services: depends_on: - redis build: ./ + env_file: + - .env volumes: - "./autogpt:/app" - ".env:/app/.env" From a40f33546449b3944f7dc924a6a4dfe31ae2a358 Mon Sep 17 00:00:00 2001 From: Shai Perednik Date: Sat, 15 Apr 2023 05:58:04 -0400 Subject: [PATCH 2/7] typo in TOC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the new memory seeiding url was wrong - [🧠 Memory pre-seeding](#-memory-pre-seeding) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 55c3d833..51b62c12 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Your support is greatly appreciated - [Setting up environment variables](#setting-up-environment-variables-1) - [Setting Your Cache Type](#setting-your-cache-type) - [View Memory Usage](#view-memory-usage) - - [🧠 Memory pre-seeding](#memory-pre-seeding) + - [🧠 Memory pre-seeding](#-memory-pre-seeding) - [💀 Continuous Mode ⚠️](#-continuous-mode-️) - [GPT3.5 ONLY Mode](#gpt35-only-mode) - [🖼 Image Generation](#-image-generation) From c69050fc84e1dc0484da28204d350ab0bdf7f3c5 Mon Sep 17 00:00:00 2001 From: Pietro Monticone <38562595+pitmonticone@users.noreply.github.com> Date: Sat, 15 Apr 2023 14:05:23 +0200 Subject: [PATCH 3/7] Fix typo --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 55c3d833..6b5dc7d4 100644 --- a/README.md +++ b/README.md @@ -35,13 +35,16 @@ Your support is greatly appreciated ## Table of Contents - [Auto-GPT: An Autonomous GPT-4 Experiment](#auto-gpt-an-autonomous-gpt-4-experiment) - - [Demo (30/03/2023):](#demo-30032023) + - [🔴 🔴 🔴 Urgent: USE `stable` not `master` 🔴 🔴 🔴](#----urgent-use-stable-not-master----) + - [Demo (30/03/2023):](#demo-30032023) - [Table of Contents](#table-of-contents) - [🚀 Features](#-features) - [📋 Requirements](#-requirements) - [💾 Installation](#-installation) - [🔧 Usage](#-usage) - [Logs](#logs) + - [Docker](#docker) + - [Command Line Arguments](#command-line-arguments) - [🗣️ Speech Mode](#️-speech-mode) - [🔍 Google API Keys Configuration](#-google-api-keys-configuration) - [Setting up environment variables](#setting-up-environment-variables) @@ -50,7 +53,7 @@ Your support is greatly appreciated - [Setting up environment variables](#setting-up-environment-variables-1) - [Setting Your Cache Type](#setting-your-cache-type) - [View Memory Usage](#view-memory-usage) - - [🧠 Memory pre-seeding](#memory-pre-seeding) + - [🧠 Memory pre-seeding](#-memory-pre-seeding) - [💀 Continuous Mode ⚠️](#-continuous-mode-️) - [GPT3.5 ONLY Mode](#gpt35-only-mode) - [🖼 Image Generation](#-image-generation) @@ -333,7 +336,7 @@ Memories will be available to the AI immediately as they are ingested, even if i In the example above, the script initializes the memory, ingests all files within the seed_data directory into memory with an overlap between chunks of 200 and a maximum length of each chunk of 4000. Note that you can also use the --file argument to ingest a single file into memory and that the script will only ingest files within the auto_gpt_workspace directory. -You can adjust the max_length and overlap parameters to fine-tune the way the docuents are presented to the AI when it "recall" that memory: +You can adjust the max_length and overlap parameters to fine-tune the way the documents are presented to the AI when it "recall" that memory: - Adjusting the overlap value allows the AI to access more contextual information from each chunk when recalling information, but will result in more chunks being created and therefore increase memory backend usage and OpenAI API requests. - Reducing the max_length value will create more chunks, which can save prompt tokens by allowing for more message history in the context, but will also increase the number of chunks. From cc7d421c7743542e53b73e01208d346f26c11222 Mon Sep 17 00:00:00 2001 From: Pietro Monticone <38562595+pitmonticone@users.noreply.github.com> Date: Sat, 15 Apr 2023 14:07:39 +0200 Subject: [PATCH 4/7] Update README.md --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6b5dc7d4..1381e957 100644 --- a/README.md +++ b/README.md @@ -35,16 +35,13 @@ Your support is greatly appreciated ## Table of Contents - [Auto-GPT: An Autonomous GPT-4 Experiment](#auto-gpt-an-autonomous-gpt-4-experiment) - - [🔴 🔴 🔴 Urgent: USE `stable` not `master` 🔴 🔴 🔴](#----urgent-use-stable-not-master----) - - [Demo (30/03/2023):](#demo-30032023) + - [Demo (30/03/2023):](#demo-30032023) - [Table of Contents](#table-of-contents) - [🚀 Features](#-features) - [📋 Requirements](#-requirements) - [💾 Installation](#-installation) - [🔧 Usage](#-usage) - [Logs](#logs) - - [Docker](#docker) - - [Command Line Arguments](#command-line-arguments) - [🗣️ Speech Mode](#️-speech-mode) - [🔍 Google API Keys Configuration](#-google-api-keys-configuration) - [Setting up environment variables](#setting-up-environment-variables) @@ -53,7 +50,7 @@ Your support is greatly appreciated - [Setting up environment variables](#setting-up-environment-variables-1) - [Setting Your Cache Type](#setting-your-cache-type) - [View Memory Usage](#view-memory-usage) - - [🧠 Memory pre-seeding](#-memory-pre-seeding) + - [🧠 Memory pre-seeding](#memory-pre-seeding) - [💀 Continuous Mode ⚠️](#-continuous-mode-️) - [GPT3.5 ONLY Mode](#gpt35-only-mode) - [🖼 Image Generation](#-image-generation) From bdefa24ac6adcc810ff144bd62b08a0d0e6b9c40 Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Sat, 15 Apr 2023 14:50:54 +0200 Subject: [PATCH 5/7] test: Write unit tests for token_counter --- autogpt/token_counter.py | 2 +- tests/test_token_counter.py | 61 +++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 tests/test_token_counter.py diff --git a/autogpt/token_counter.py b/autogpt/token_counter.py index c1239722..a85a54be 100644 --- a/autogpt/token_counter.py +++ b/autogpt/token_counter.py @@ -27,7 +27,7 @@ def count_message_tokens( logger.warn("Warning: model not found. Using cl100k_base encoding.") encoding = tiktoken.get_encoding("cl100k_base") if model == "gpt-3.5-turbo": - # !Node: gpt-3.5-turbo may change over time. + # !Note: gpt-3.5-turbo may change over time. # Returning num tokens assuming gpt-3.5-turbo-0301.") return count_message_tokens(messages, model="gpt-3.5-turbo-0301") elif model == "gpt-4": diff --git a/tests/test_token_counter.py b/tests/test_token_counter.py new file mode 100644 index 00000000..d13f2ae0 --- /dev/null +++ b/tests/test_token_counter.py @@ -0,0 +1,61 @@ +import unittest +import tests.context + +from scripts.token_counter import count_message_tokens, count_string_tokens + +class TestTokenCounter(unittest.TestCase): + + def test_count_message_tokens(self): + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"} + ] + self.assertEqual(count_message_tokens(messages), 17) + + def test_count_message_tokens_with_name(self): + messages = [ + {"role": "user", "content": "Hello", "name": "John"}, + {"role": "assistant", "content": "Hi there!"} + ] + self.assertEqual(count_message_tokens(messages), 17) + + def test_count_message_tokens_empty_input(self): + self.assertEqual(count_message_tokens([]), 3) + + def test_count_message_tokens_invalid_model(self): + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"} + ] + with self.assertRaises(KeyError): + count_message_tokens(messages, model="invalid_model") + + def test_count_message_tokens_gpt_4(self): + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"} + ] + self.assertEqual(count_message_tokens(messages, model="gpt-4-0314"), 15) + + def test_count_string_tokens(self): + string = "Hello, world!" + self.assertEqual(count_string_tokens(string, model_name="gpt-3.5-turbo-0301"), 4) + + def test_count_string_tokens_empty_input(self): + self.assertEqual(count_string_tokens("", model_name="gpt-3.5-turbo-0301"), 0) + + def test_count_message_tokens_invalid_model(self): + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"} + ] + with self.assertRaises(NotImplementedError): + count_message_tokens(messages, model="invalid_model") + + def test_count_string_tokens_gpt_4(self): + string = "Hello, world!" + self.assertEqual(count_string_tokens(string, model_name="gpt-4-0314"), 4) + + +if __name__ == '__main__': + unittest.main() From d52381eead2f4ae8d5db3ca2e747a6c5e3dc1cea Mon Sep 17 00:00:00 2001 From: Drikus Roor Date: Sat, 15 Apr 2023 15:20:19 +0200 Subject: [PATCH 6/7] fix: Fix imports --- tests/test_token_counter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_token_counter.py b/tests/test_token_counter.py index d13f2ae0..42670e60 100644 --- a/tests/test_token_counter.py +++ b/tests/test_token_counter.py @@ -1,7 +1,7 @@ import unittest import tests.context +from autogpt.token_counter import count_message_tokens, count_string_tokens -from scripts.token_counter import count_message_tokens, count_string_tokens class TestTokenCounter(unittest.TestCase): From e90e618c5ee9b37ebbc38bd4c75805d3cbceef17 Mon Sep 17 00:00:00 2001 From: Eesa Hamza Date: Sat, 15 Apr 2023 16:28:34 +0300 Subject: [PATCH 7/7] Added agnostic browser support --- .env.template | 3 +++ autogpt/__main__.py | 4 +++- autogpt/args.py | 9 +++++++++ autogpt/commands/web_selenium.py | 28 +++++++++++++++++++++------- autogpt/config/config.py | 1 + 5 files changed, 37 insertions(+), 8 deletions(-) diff --git a/.env.template b/.env.template index 22bf8d74..4a334bea 100644 --- a/.env.template +++ b/.env.template @@ -11,6 +11,9 @@ BROWSE_SUMMARY_MAX_TOKEN=300 # USER_AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36" # AI_SETTINGS_FILE - Specifies which AI Settings file to use (defaults to ai_settings.yaml) AI_SETTINGS_FILE=ai_settings.yaml +# USE_WEB_BROWSER - Sets the web-browser drivers to use with selenium (defaults to chrome). +# Note: set this to either 'chrome', 'firefox', or 'safari' depending on your current browser +# USE_WEB_BROWSER=chrome ################################################################################ ### LLM PROVIDER diff --git a/autogpt/__main__.py b/autogpt/__main__.py index f90f1532..5105df55 100644 --- a/autogpt/__main__.py +++ b/autogpt/__main__.py @@ -1,5 +1,6 @@ """Main script for the autogpt package.""" import logging +from colorama import Fore from autogpt.agent.agent import Agent from autogpt.args import parse_arguments @@ -33,7 +34,8 @@ def main() -> None: # Initialize memory and make sure it is empty. # this is particularly important for indexing and referencing pinecone memory memory = get_memory(cfg, init=True) - print(f"Using memory of type: {memory.__class__.__name__}") + logger.typewriter_log(f"Using memory of type:", Fore.GREEN, f"{memory.__class__.__name__}") + logger.typewriter_log(f"Using Browser:", Fore.GREEN, cfg.selenium_web_browser) agent = Agent( ai_name=ai_name, memory=memory, diff --git a/autogpt/args.py b/autogpt/args.py index 6c907a1f..0bc3e680 100644 --- a/autogpt/args.py +++ b/autogpt/args.py @@ -50,6 +50,12 @@ def parse_arguments() -> None: action="store_true", help="Skips the re-prompting messages at the beginning of the script", ) + parser.add_argument( + "--use-browser", + "-b", + dest="browser_name", + help="Specifies which web-browser to use when using selenium to scrape the web." + ) parser.add_argument( "--ai-settings", "-C", @@ -126,3 +132,6 @@ def parse_arguments() -> None: logger.typewriter_log("Using AI Settings File:", Fore.GREEN, file) CFG.ai_settings_file = file CFG.skip_reprompt = True + + if args.browser_name: + CFG.selenium_web_browser = args.browser_name diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 4d5703aa..379d8cbe 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -7,7 +7,10 @@ from selenium.webdriver.common.by import By from selenium.webdriver.support.wait import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from webdriver_manager.chrome import ChromeDriverManager -from selenium.webdriver.chrome.options import Options +from webdriver_manager.firefox import GeckoDriverManager +from selenium.webdriver.chrome.options import Options as ChromeOptions +from selenium.webdriver.firefox.options import Options as FirefoxOptions +from selenium.webdriver.safari.options import Options as SafariOptions import logging from pathlib import Path from autogpt.config import Config @@ -49,14 +52,25 @@ def scrape_text_with_selenium(url: str) -> tuple[WebDriver, str]: """ logging.getLogger("selenium").setLevel(logging.CRITICAL) - options = Options() + options_available = {'chrome': ChromeOptions, 'safari': SafariOptions, 'firefox': FirefoxOptions} + + options = options_available[CFG.selenium_web_browser]() options.add_argument( - "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - " (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36" - ) - driver = webdriver.Chrome( - executable_path=ChromeDriverManager().install(), options=options + "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36" ) + + if CFG.selenium_web_browser == "firefox": + driver = webdriver.Firefox( + executable_path=GeckoDriverManager().install(), options=options + ) + elif CFG.selenium_web_browser == "safari": + # Requires a bit more setup on the users end + # See https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari + driver = webdriver.Safari(options=options) + else: + driver = webdriver.Chrome( + executable_path=ChromeDriverManager().install(), options=options + ) driver.get(url) WebDriverWait(driver, 10).until( diff --git a/autogpt/config/config.py b/autogpt/config/config.py index 46764b9c..87e53304 100644 --- a/autogpt/config/config.py +++ b/autogpt/config/config.py @@ -25,6 +25,7 @@ class Config(metaclass=Singleton): self.speak_mode = False self.skip_reprompt = False + self.selenium_web_browser = os.getenv("USE_WEB_BROWSER", "chrome") self.ai_settings_file = os.getenv("AI_SETTINGS_FILE", "ai_settings.yaml") self.fast_llm_model = os.getenv("FAST_LLM_MODEL", "gpt-3.5-turbo") self.smart_llm_model = os.getenv("SMART_LLM_MODEL", "gpt-4")