From 986d32ca42671b1e092fa2842196463ac0992915 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Fri, 7 Apr 2023 20:41:07 +0100 Subject: [PATCH 01/48] added support for multiple memory provider and added weaviate integration --- .env.template | 8 +- README.md | 26 ++++- requirements.txt | 1 + scripts/commands.py | 4 +- scripts/config.py | 8 ++ scripts/factory.py | 11 ++ scripts/main.py | 4 +- scripts/providers/__init__.py | 0 scripts/providers/memory.py | 26 +++++ scripts/{memory.py => providers/pinecone.py} | 18 +--- scripts/providers/weaviate.py | 100 +++++++++++++++++++ 11 files changed, 185 insertions(+), 21 deletions(-) create mode 100644 scripts/factory.py create mode 100644 scripts/providers/__init__.py create mode 100644 scripts/providers/memory.py rename scripts/{memory.py => providers/pinecone.py} (80%) create mode 100644 scripts/providers/weaviate.py diff --git a/.env.template b/.env.template index e9ccda5e..c9a45b2b 100644 --- a/.env.template +++ b/.env.template @@ -9,4 +9,10 @@ CUSTOM_SEARCH_ENGINE_ID= USE_AZURE=False OPENAI_API_BASE=your-base-url-for-azure OPENAI_API_VERSION=api-version-for-azure -OPENAI_DEPLOYMENT_ID=deployment-id-for-azure \ No newline at end of file +OPENAI_DEPLOYMENT_ID=deployment-id-for-azure +WEAVIATE_HOST="http://127.0.0.1" +WEAVIATE_PORT="8080" +WEAVIATE_USERNAME= +WEAVIATE_PASSWORD= +WEAVIATE_INDEX="Autogpt" +MEMORY_PROVIDER="weaviate" \ No newline at end of file diff --git a/README.md b/README.md index a89c5d03..9e8c24f2 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,13 @@ export CUSTOM_SEARCH_ENGINE_ID="YOUR_CUSTOM_SEARCH_ENGINE_ID" ``` -## 🌲 Pinecone API Key Setup +## Vector based memory provider +Auto-GPT supports two providers for vector-based memory, [Pinecone](https://www.pinecone.io/) and [Weaviate](https://weaviate.io/). To select the provider to use, specify the following in your `.env`: + +``` +MEMORY_PROVIDER="pinecone" # change to "weaviate" to use weaviate as the memory provider +``` +### 🌲 Pinecone API Key Setup Pinecone enable a vector based memory so a vast memory can be stored and only relevant memories are loaded for the agent at any given time. @@ -149,7 +155,7 @@ are loaded for the agent at any given time. 2. Choose the `Starter` plan to avoid being charged. 3. Find your API key and region under the default project in the left sidebar. -### Setting up environment variables +#### Setting up environment variables For Windows Users: ``` setx PINECONE_API_KEY "YOUR_PINECONE_API_KEY" @@ -165,6 +171,22 @@ export PINECONE_ENV="Your pinecone region" # something like: us-east4-gcp Or you can set them in the `.env` file. +### Weaviate Setup + +[Weaviate](https://weaviate.io/) is an open-source vector database. It allows to store data objects and vector embeddings from ML-models and scales seamlessly to billion of data objects. [An instance of Weaviate can be created locally (using Docker), on Kubernetes or using Weaviate Cloud Services](https://weaviate.io/developers/weaviate/quickstart). + +#### Setting up enviornment variables + +In your `.env` file set the following: + +``` +WEAVIATE_HOST="http://127.0.0.1" # the URL of the running Weaviate instance +WEAVIATE_PORT="8080" +WEAVIATE_USERNAME="your username" +WEAVIATE_PASSWORD="your password" +WEAVIATE_INDEX="Autogpt" # name of the index to create for the application +``` + ## View Memory Usage 1. View memory usage by using the `--debug` flag :) diff --git a/requirements.txt b/requirements.txt index ce247098..aed03226 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ docker duckduckgo-search google-api-python-client #(https://developers.google.com/custom-search/v1/overview) pinecone-client==2.2.1 +weaviate-client==3.15.4 diff --git a/scripts/commands.py b/scripts/commands.py index fc10d1d0..13037c34 100644 --- a/scripts/commands.py +++ b/scripts/commands.py @@ -1,6 +1,6 @@ import browse import json -from memory import PineconeMemory +from factory import MemoryFactory import datetime import agent_manager as agents import speak @@ -52,7 +52,7 @@ def get_command(response): def execute_command(command_name, arguments): - memory = PineconeMemory() + memory = MemoryFactory.get_memory(cfg.memory_provider) try: if command_name == "google": diff --git a/scripts/config.py b/scripts/config.py index fe48d298..bc88bbb9 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -50,9 +50,17 @@ class Config(metaclass=Singleton): self.google_api_key = os.getenv("GOOGLE_API_KEY") self.custom_search_engine_id = os.getenv("CUSTOM_SEARCH_ENGINE_ID") + self.memory_provider = os.getenv("MEMORY_PROVIDER", 'pinecone') self.pinecone_api_key = os.getenv("PINECONE_API_KEY") self.pinecone_region = os.getenv("PINECONE_ENV") + self.weaviate_host = os.getenv("WEAVIATE_HOST") + self.weaviate_port = os.getenv("WEAVIATE_PORT") + self.weaviate_username = os.getenv("WEAVIATE_USERNAME", None) + self.weaviate_password = os.getenv("WEAVIATE_PASSWORD", None) + self.weaviate_scopes = os.getenv("WEAVIATE_SCOPES", None) + self.weaviate_index = os.getenv("WEAVIATE_INDEX", 'auto-gpt') + # User agent headers to use when browsing web # Some websites might just completely deny request with an error code if no user agent was found. self.user_agent_header = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"} diff --git a/scripts/factory.py b/scripts/factory.py new file mode 100644 index 00000000..44901631 --- /dev/null +++ b/scripts/factory.py @@ -0,0 +1,11 @@ +from providers.pinecone import PineconeMemory +from providers.weaviate import WeaviateMemory + +class MemoryFactory: + @staticmethod + def get_memory(mem_type): + if mem_type == 'pinecone': + return PineconeMemory() + + if mem_type == 'weaviate': + return WeaviateMemory() \ No newline at end of file diff --git a/scripts/main.py b/scripts/main.py index a79fd553..795c2ac4 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -1,7 +1,7 @@ import json import random import commands as cmd -from memory import PineconeMemory +from factory import MemoryFactory import data import chat from colorama import Fore, Style @@ -283,7 +283,7 @@ user_input = "Determine which next command to use, and respond using the format # Initialize memory and make sure it is empty. # this is particularly important for indexing and referencing pinecone memory -memory = PineconeMemory() +memory = MemoryFactory.get_memory(cfg.memory_provider) memory.clear() print('Using memory of type: ' + memory.__class__.__name__) diff --git a/scripts/providers/__init__.py b/scripts/providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/providers/memory.py b/scripts/providers/memory.py new file mode 100644 index 00000000..0440536e --- /dev/null +++ b/scripts/providers/memory.py @@ -0,0 +1,26 @@ +from config import Singleton +import openai + +def get_ada_embedding(text): + text = text.replace("\n", " ") + return openai.Embedding.create(input=[text], model="text-embedding-ada-002")["data"][0]["embedding"] + + +def get_text_from_embedding(embedding): + return openai.Embedding.retrieve(embedding, model="text-embedding-ada-002")["data"][0]["text"] + +class Memory(metaclass=Singleton): + def add(self, data): + raise NotImplementedError() + + def get(self, data): + raise NotImplementedError() + + def clear(self): + raise NotImplementedError() + + def get_relevant(self, data, num_relevant=5): + raise NotImplementedError() + + def get_stats(self): + raise NotImplementedError() \ No newline at end of file diff --git a/scripts/memory.py b/scripts/providers/pinecone.py similarity index 80% rename from scripts/memory.py rename to scripts/providers/pinecone.py index 0d265a31..971ef186 100644 --- a/scripts/memory.py +++ b/scripts/providers/pinecone.py @@ -1,20 +1,10 @@ -from config import Config, Singleton +from config import Config +from providers.memory import Memory, get_ada_embedding import pinecone -import openai cfg = Config() - -def get_ada_embedding(text): - text = text.replace("\n", " ") - return openai.Embedding.create(input=[text], model="text-embedding-ada-002")["data"][0]["embedding"] - - -def get_text_from_embedding(embedding): - return openai.Embedding.retrieve(embedding, model="text-embedding-ada-002")["data"][0]["text"] - - -class PineconeMemory(metaclass=Singleton): +class PineconeMemory(Memory): def __init__(self): pinecone_api_key = cfg.pinecone_api_key pinecone_region = cfg.pinecone_region @@ -58,4 +48,4 @@ class PineconeMemory(metaclass=Singleton): return [str(item['metadata']["raw_text"]) for item in sorted_results] def get_stats(self): - return self.index.describe_index_stats() + return self.index.describe_index_stats() \ No newline at end of file diff --git a/scripts/providers/weaviate.py b/scripts/providers/weaviate.py new file mode 100644 index 00000000..21718a03 --- /dev/null +++ b/scripts/providers/weaviate.py @@ -0,0 +1,100 @@ +from config import Config +from providers.memory import Memory, get_ada_embedding +from weaviate import Client +import weaviate +import uuid + +from weaviate.util import generate_uuid5 + +cfg = Config() + +SCHEMA = { + "class": cfg.weaviate_index, + "properties": [ + { + "name": "raw_text", + "dataType": ["text"], + "description": "original text for the embedding" + } + ], +} + +class WeaviateMemory(Memory): + + def __init__(self): + auth_credentials = self._build_auth_credentials() + + url = f'{cfg.weaviate_host}:{cfg.weaviate_port}' + + self.client = Client(url, auth_client_secret=auth_credentials) + + self._create_schema() + + def _create_schema(self): + if not self.client.schema.contains(SCHEMA): + self.client.schema.create_class(SCHEMA) + + @staticmethod + def _build_auth_credentials(): + if cfg.weaviate_username and cfg.weaviate_password: + return weaviate_auth.AuthClientPassword(cfg.weaviate_username, cfg.weaviate_password) + else: + return None + + def add(self, data): + vector = get_ada_embedding(data) + + doc_uuid = generate_uuid5(data, cfg.weaviate_index) + data_object = { + 'class': cfg.weaviate_index, + 'raw_text': data + } + + with self.client.batch as batch: + batch.add_data_object( + uuid=doc_uuid, + data_object=data_object, + class_name=cfg.weaviate_index, + vector=vector + ) + + batch.flush() + + return f"Inserting data into memory at uuid: {doc_uuid}:\n data: {data}" + + + def get(self, data): + return self.get_relevant(data, 1) + + + def clear(self): + self.client.schema.delete_all() + + # weaviate does not yet have a neat way to just remove the items in an index + # without removing the entire schema, therefore we need to re-create it + # after a call to delete_all + self._create_schema() + + return 'Obliterated' + + def get_relevant(self, data, num_relevant=5): + query_embedding = get_ada_embedding(data) + try: + results = self.client.query.get(cfg.weaviate_index, ['raw_text']) \ + .with_near_vector({'vector': query_embedding, 'certainty': 0.7}) \ + .with_limit(num_relevant) \ + .do() + + print(results) + + if len(results['data']['Get'][cfg.weaviate_index]) > 0: + return [str(item['raw_text']) for item in results['data']['Get'][cfg.weaviate_index]] + else: + return [] + + except Exception as err: + print(f'Unexpected error {err=}, {type(err)=}') + return [] + + def get_stats(self): + return self.client.index_stats.get(cfg.weaviate_index) \ No newline at end of file From da4ba3c10f66c43e0d211ada2a1277bf1b4ce713 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Fri, 7 Apr 2023 22:07:08 +0100 Subject: [PATCH 02/48] added factory tests --- scripts/factory.py | 4 +++- tests/memory_tests.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 tests/memory_tests.py diff --git a/scripts/factory.py b/scripts/factory.py index 44901631..19f67ba0 100644 --- a/scripts/factory.py +++ b/scripts/factory.py @@ -8,4 +8,6 @@ class MemoryFactory: return PineconeMemory() if mem_type == 'weaviate': - return WeaviateMemory() \ No newline at end of file + return WeaviateMemory() + + raise ValueError('Unknown memory provider') \ No newline at end of file diff --git a/tests/memory_tests.py b/tests/memory_tests.py new file mode 100644 index 00000000..4985c022 --- /dev/null +++ b/tests/memory_tests.py @@ -0,0 +1,55 @@ +import unittest +from unittest import mock +import sys +import os + +sys.path.append(os.path.abspath('./scripts')) + +from factory import MemoryFactory +from providers.weaviate import WeaviateMemory +from providers.pinecone import PineconeMemory + +class TestMemoryFactory(unittest.TestCase): + + def test_invalid_memory_provider(self): + + with self.assertRaises(ValueError): + memory = MemoryFactory.get_memory('Thanos') + + def test_create_pinecone_provider(self): + + # mock the init function of the provider to bypass + # connection to the external pinecone service + def __init__(self): + pass + + with mock.patch.object(PineconeMemory, '__init__', __init__): + memory = MemoryFactory.get_memory('pinecone') + self.assertIsInstance(memory, PineconeMemory) + + def test_create_weaviate_provider(self): + + # mock the init function of the provider to bypass + # connection to the external weaviate service + def __init__(self): + pass + + with mock.patch.object(WeaviateMemory, '__init__', __init__): + memory = MemoryFactory.get_memory('weaviate') + self.assertIsInstance(memory, WeaviateMemory) + + def test_provider_is_singleton(self): + + def __init__(self): + pass + + with mock.patch.object(WeaviateMemory, '__init__', __init__): + instance = MemoryFactory.get_memory('weaviate') + other_instance = MemoryFactory.get_memory('weaviate') + + self.assertIs(instance, other_instance) + + +if __name__ == '__main__': + unittest.main() + From 0ce0c553a60472cbffd42dc1650dc6a9bc0ff36e Mon Sep 17 00:00:00 2001 From: cs0lar Date: Sat, 8 Apr 2023 08:13:17 +0100 Subject: [PATCH 03/48] the three memory related commands memory_add, memory_del, memory_ovr are absent in the latest version of execute_command therefore the corresponding handlers commit_memory, delete_memory and overwrite_memory have been removed also because they assume a memory with a different interface than the proposed one. --- scripts/commands.py | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/scripts/commands.py b/scripts/commands.py index 13037c34..117f0e9d 100644 --- a/scripts/commands.py +++ b/scripts/commands.py @@ -182,49 +182,6 @@ def get_hyperlinks(url): return link_list -def commit_memory(string): - _text = f"""Committing memory with string "{string}" """ - mem.permanent_memory.append(string) - return _text - - -def delete_memory(key): - if key >= 0 and key < len(mem.permanent_memory): - _text = "Deleting memory with key " + str(key) - del mem.permanent_memory[key] - print(_text) - return _text - else: - print("Invalid key, cannot delete memory.") - return None - - -def overwrite_memory(key, string): - # Check if the key is a valid integer - if is_valid_int(key): - key_int = int(key) - # Check if the integer key is within the range of the permanent_memory list - if 0 <= key_int < len(mem.permanent_memory): - _text = "Overwriting memory with key " + str(key) + " and string " + string - # Overwrite the memory slot with the given integer key and string - mem.permanent_memory[key_int] = string - print(_text) - return _text - else: - print(f"Invalid key '{key}', out of range.") - return None - # Check if the key is a valid string - elif isinstance(key, str): - _text = "Overwriting memory with key " + key + " and string " + string - # Overwrite the memory slot with the given string key and string - mem.permanent_memory[key] = string - print(_text) - return _text - else: - print(f"Invalid key '{key}', must be an integer or a string.") - return None - - def shutdown(): print("Shutting down...") quit() From 76a1462e370297d3dab0d2b2c4ef4fa8739c9232 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Sat, 8 Apr 2023 16:11:31 +0100 Subject: [PATCH 04/48] moved pinecone api config settings into provider class --- scripts/main.py | 2 -- scripts/providers/pinecone.py | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/main.py b/scripts/main.py index 51495e20..4740c626 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -281,8 +281,6 @@ next_action_count = 0 # Make a constant: user_input = "Determine which next command to use, and respond using the format specified above:" -# raise an exception if pinecone_api_key or region is not provided -if not cfg.pinecone_api_key or not cfg.pinecone_region: raise Exception("Please provide pinecone_api_key and pinecone_region") # Initialize memory and make sure it is empty. # this is particularly important for indexing and referencing pinecone memory memory = MemoryFactory.get_memory(cfg.memory_provider) diff --git a/scripts/providers/pinecone.py b/scripts/providers/pinecone.py index 971ef186..e8cf019a 100644 --- a/scripts/providers/pinecone.py +++ b/scripts/providers/pinecone.py @@ -6,6 +6,9 @@ cfg = Config() class PineconeMemory(Memory): def __init__(self): + # raise an exception if pinecone_api_key or region is not provided + if not cfg.pinecone_api_key or not cfg.pinecone_region: raise Exception("Please provide pinecone_api_key and pinecone_region") + pinecone_api_key = cfg.pinecone_api_key pinecone_region = cfg.pinecone_region pinecone.init(api_key=pinecone_api_key, environment=pinecone_region) From 786ee6003c7716e747af32a67ffbaf10618f9784 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Tue, 11 Apr 2023 13:50:02 +0100 Subject: [PATCH 05/48] fixed formatting --- README.md | 2 -- scripts/commands.py | 2 +- scripts/memory/pinecone.py | 2 +- scripts/memory/weaviate.py | 2 +- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3e415361..18809f58 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,6 @@ Pinecone enables the storage of vast amounts of vector-based memory, allowing fo 2. Choose the `Starter` plan to avoid being charged. 3. Find your API key and region under the default project in the left sidebar. - ### Setting up environment variables Simply set them in the `.env` file. @@ -193,7 +192,6 @@ Simply set them in the `.env` file. Alternatively, you can set them from the command line (advanced): For Windows Users: - ``` setx PINECONE_API_KEY "YOUR_PINECONE_API_KEY" setx PINECONE_ENV "Your pinecone region" # something like: us-east4-gcp diff --git a/scripts/commands.py b/scripts/commands.py index 5415c1e0..ce5d04ff 100644 --- a/scripts/commands.py +++ b/scripts/commands.py @@ -298,4 +298,4 @@ def delete_agent(key): result = agents.delete_agent(key) if not result: return f"Agent {key} does not exist." - return f"Agent {key} deleted." \ No newline at end of file + return f"Agent {key} deleted." diff --git a/scripts/memory/pinecone.py b/scripts/memory/pinecone.py index 287bc1dc..d89daf25 100644 --- a/scripts/memory/pinecone.py +++ b/scripts/memory/pinecone.py @@ -47,4 +47,4 @@ class PineconeMemory(MemoryProviderSingleton): return [str(item['metadata']["raw_text"]) for item in sorted_results] def get_stats(self): - return self.index.describe_index_stats() \ No newline at end of file + return self.index.describe_index_stats() diff --git a/scripts/memory/weaviate.py b/scripts/memory/weaviate.py index b6032648..d29192ea 100644 --- a/scripts/memory/weaviate.py +++ b/scripts/memory/weaviate.py @@ -97,4 +97,4 @@ class WeaviateMemory(MemoryProviderSingleton): .do() class_data = result['data']['Aggregate'][self.index] - return class_data[0]['meta'] if class_data else {} \ No newline at end of file + return class_data[0]['meta'] if class_data else {} From 3c7767fab07d8a1479fc04a7b3dd09d516fe01a9 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Tue, 11 Apr 2023 13:51:31 +0100 Subject: [PATCH 06/48] fixed formatting --- scripts/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/main.py b/scripts/main.py index 21746118..8661bfad 100644 --- a/scripts/main.py +++ b/scripts/main.py @@ -415,3 +415,4 @@ while True: chat.create_chat_message( "system", "Unable to execute command")) print_to_console("SYSTEM: ", Fore.YELLOW, "Unable to execute command") + From 96c5e929be81e05954a1f3dcb25a03fc8dede91c Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 05:40:24 +0100 Subject: [PATCH 07/48] added support for weaviate embedded --- .env.template | 4 +++- scripts/config.py | 2 ++ scripts/memory/weaviate.py | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.env.template b/.env.template index bc3bea20..9e4a1a9f 100644 --- a/.env.template +++ b/.env.template @@ -11,7 +11,9 @@ OPENAI_API_BASE=your-base-url-for-azure OPENAI_API_VERSION=api-version-for-azure OPENAI_DEPLOYMENT_ID=deployment-id-for-azure WEAVIATE_HOST="http://127.0.0.1" -WEAVIATE_PORT="8080" +WEAVIATE_PORT=8080 +USE_WEAVIATE_EMBEDDED=False +WEAVIATE_EMBEDDED_PATH="~/.local/share/weaviate" WEAVIATE_USERNAME= WEAVIATE_PASSWORD= IMAGE_PROVIDER=dalle diff --git a/scripts/config.py b/scripts/config.py index 2a0c0a94..7608a504 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -69,6 +69,8 @@ class Config(metaclass=Singleton): self.weaviate_username = os.getenv("WEAVIATE_USERNAME", None) self.weaviate_password = os.getenv("WEAVIATE_PASSWORD", None) self.weaviate_scopes = os.getenv("WEAVIATE_SCOPES", None) + self.weaviate_embedded_path = os.getenv('WEAVIATE_EMBEDDED_PATH', '~/.local/share/weaviate') + self.use_weaviate_embedded = os.getenv("USE_WEAVIATE_EMBEDDED", False) self.image_provider = os.getenv("IMAGE_PROVIDER") self.huggingface_api_token = os.getenv("HUGGINGFACE_API_TOKEN") diff --git a/scripts/memory/weaviate.py b/scripts/memory/weaviate.py index d29192ea..10a76640 100644 --- a/scripts/memory/weaviate.py +++ b/scripts/memory/weaviate.py @@ -3,6 +3,7 @@ from memory.base import MemoryProviderSingleton, get_ada_embedding import uuid import weaviate from weaviate import Client +from weaviate.embedded import EmbeddedOptions from weaviate.util import generate_uuid5 def default_schema(weaviate_index): @@ -23,7 +24,17 @@ class WeaviateMemory(MemoryProviderSingleton): url = f'{cfg.weaviate_host}:{cfg.weaviate_port}' - self.client = Client(url, auth_client_secret=auth_credentials) + if cfg.use_weaviate_embedded: + self.client = Client(embedded_options=EmbeddedOptions( + hostname=cfg.weaviate_host, + port=int(cfg.weaviate_port), + persistence_data_path=cfg.weaviate_embedded_path + )) + + print(f"Weaviate Embedded running on: {url} with persistence path: {cfg.weaviate_embedded_path}") + else: + self.client = Client(url, auth_client_secret=auth_credentials) + self.index = cfg.memory_index self._create_schema() @@ -59,7 +70,6 @@ class WeaviateMemory(MemoryProviderSingleton): return f"Inserting data into memory at uuid: {doc_uuid}:\n data: {data}" - def get(self, data): return self.get_relevant(data, 1) From 453b428d33aa573608d76d8a64ef448e5070d77a Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 08:21:41 +0100 Subject: [PATCH 08/48] added support for weaviate embedded --- .env.template | 5 +++-- requirements.txt | 2 +- scripts/config.py | 3 ++- scripts/memory/weaviate.py | 2 +- tests/test_weaviate_memory.py | 22 ++++++++++++++++++---- 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.env.template b/.env.template index 9e4a1a9f..b9869fb9 100644 --- a/.env.template +++ b/.env.template @@ -10,10 +10,11 @@ USE_AZURE=False OPENAI_API_BASE=your-base-url-for-azure OPENAI_API_VERSION=api-version-for-azure OPENAI_DEPLOYMENT_ID=deployment-id-for-azure -WEAVIATE_HOST="http://127.0.0.1" +WEAVIATE_HOST="127.0.0.1" WEAVIATE_PORT=8080 +WEAVIATE_PROTOCOL="http" USE_WEAVIATE_EMBEDDED=False -WEAVIATE_EMBEDDED_PATH="~/.local/share/weaviate" +WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" WEAVIATE_USERNAME= WEAVIATE_PASSWORD= IMAGE_PROVIDER=dalle diff --git a/requirements.txt b/requirements.txt index 8004319d..d86ebe97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,4 @@ pinecone-client==2.2.1 redis orjson Pillow -weaviate-client==3.15.4 +weaviate-client==3.15.5 diff --git a/scripts/config.py b/scripts/config.py index 7608a504..ba43dca4 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -66,11 +66,12 @@ class Config(metaclass=Singleton): self.weaviate_host = os.getenv("WEAVIATE_HOST") self.weaviate_port = os.getenv("WEAVIATE_PORT") + self.weaviate_protocol = os.getenv("WEAVIATE_PROTOCOL", "http") self.weaviate_username = os.getenv("WEAVIATE_USERNAME", None) self.weaviate_password = os.getenv("WEAVIATE_PASSWORD", None) self.weaviate_scopes = os.getenv("WEAVIATE_SCOPES", None) self.weaviate_embedded_path = os.getenv('WEAVIATE_EMBEDDED_PATH', '~/.local/share/weaviate') - self.use_weaviate_embedded = os.getenv("USE_WEAVIATE_EMBEDDED", False) + self.use_weaviate_embedded = os.getenv("USE_WEAVIATE_EMBEDDED", "False") == "True" self.image_provider = os.getenv("IMAGE_PROVIDER") self.huggingface_api_token = os.getenv("HUGGINGFACE_API_TOKEN") diff --git a/scripts/memory/weaviate.py b/scripts/memory/weaviate.py index 10a76640..2eac5839 100644 --- a/scripts/memory/weaviate.py +++ b/scripts/memory/weaviate.py @@ -22,7 +22,7 @@ class WeaviateMemory(MemoryProviderSingleton): def __init__(self, cfg): auth_credentials = self._build_auth_credentials(cfg) - url = f'{cfg.weaviate_host}:{cfg.weaviate_port}' + url = f'{cfg.weaviate_protocol}://{cfg.weaviate_host}:{cfg.weaviate_port}' if cfg.use_weaviate_embedded: self.client = Client(embedded_options=EmbeddedOptions( diff --git a/tests/test_weaviate_memory.py b/tests/test_weaviate_memory.py index 41709bc0..6f39a203 100644 --- a/tests/test_weaviate_memory.py +++ b/tests/test_weaviate_memory.py @@ -13,10 +13,11 @@ from memory.weaviate import WeaviateMemory from memory.base import get_ada_embedding @mock.patch.dict(os.environ, { - "WEAVIATE_HOST": "http://127.0.0.1", + "WEAVIATE_HOST": "127.0.0.1", + "WEAVIATE_PROTOCOL": "http", "WEAVIATE_PORT": "8080", - "WEAVIATE_USERNAME": '', - "WEAVIATE_PASSWORD": '', + "WEAVIATE_USERNAME": "", + "WEAVIATE_PASSWORD": "", "MEMORY_INDEX": "AutogptTests" }) class TestWeaviateMemory(unittest.TestCase): @@ -24,11 +25,24 @@ class TestWeaviateMemory(unittest.TestCase): In order to run these tests you will need a local instance of Weaviate running. Refer to https://weaviate.io/developers/weaviate/installation/docker-compose for creating local instances using docker. + Alternatively in your .env file set the following environmental variables to run Weaviate embedded (see: https://weaviate.io/developers/weaviate/installation/embedded): + + USE_WEAVIATE_EMBEDDED=True + WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" """ def setUp(self): self.cfg = Config() - self.client = Client('http://127.0.0.1:8080') + if self.cfg.use_weaviate_embedded: + from weaviate.embedded import EmbeddedOptions + + self.client = Client(embedded_options=EmbeddedOptions( + hostname=self.cfg.weaviate_host, + port=int(self.cfg.weaviate_port), + persistence_data_path=self.cfg.weaviate_embedded_path + )) + else: + self.client = Client(f"{self.cfg.weaviate_protocol}://{self.cfg.weaviate_host}:{self.cfg.weaviate_port}") try: self.client.schema.delete_class(self.cfg.memory_index) From f2a6ac5dc203129a403e242d2516b71a385c90bc Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 09:20:29 +0100 Subject: [PATCH 09/48] fixed order and removed dupes --- .env.template | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.env.template b/.env.template index b9869fb9..602dc929 100644 --- a/.env.template +++ b/.env.template @@ -10,6 +10,9 @@ USE_AZURE=False OPENAI_API_BASE=your-base-url-for-azure OPENAI_API_VERSION=api-version-for-azure OPENAI_DEPLOYMENT_ID=deployment-id-for-azure +IMAGE_PROVIDER=dalle +HUGGINGFACE_API_TOKEN= +USE_MAC_OS_TTS=False WEAVIATE_HOST="127.0.0.1" WEAVIATE_PORT=8080 WEAVIATE_PROTOCOL="http" @@ -17,12 +20,5 @@ USE_WEAVIATE_EMBEDDED=False WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" WEAVIATE_USERNAME= WEAVIATE_PASSWORD= -IMAGE_PROVIDER=dalle -HUGGINGFACE_API_TOKEN= -OPENAI_AZURE_API_BASE=your-base-url-for-azure -OPENAI_AZURE_API_VERSION=api-version-for-azure -OPENAI_AZURE_DEPLOYMENT_ID=deployment-id-for-azure -IMAGE_PROVIDER=dalle -USE_MAC_OS_TTS=False MEMORY_INDEX="auto-gpt" MEMORY_BACKEND="local" From e3aea6d6c40119bdf7094a6e85691a2448a1abc3 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 17:21:37 +0100 Subject: [PATCH 10/48] added weaviate embedded section in README --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 18809f58..9959f55d 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,7 @@ export PINECONE_ENV="Your pinecone region" # something like: us-east4-gcp ## Weaviate Setup [Weaviate](https://weaviate.io/) is an open-source vector database. It allows to store data objects and vector embeddings from ML-models and scales seamlessly to billion of data objects. [An instance of Weaviate can be created locally (using Docker), on Kubernetes or using Weaviate Cloud Services](https://weaviate.io/developers/weaviate/quickstart). +Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) is supported which allows the Auto-GPT process itself to start a Weaviate instance. To enable it, set `USE_WEAVIATE_EMBEDDED` to `True`. #### Setting up enviornment variables @@ -214,10 +215,13 @@ In your `.env` file set the following: ``` MEMORY_BACKEND=weaviate -WEAVIATE_HOST="http://127.0.0.1" # the URL of the running Weaviate instance +WEAVIATE_HOST="127.0.0.1" # the IP or domain of the running Weaviate instance WEAVIATE_PORT="8080" +WEAVIATE_PROTOCOL="http" WEAVIATE_USERNAME="your username" WEAVIATE_PASSWORD="your password" +WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" # this is optional and indicates where the data should be persisted when running an embedded instance +USE_WEAVIATE_EMBEDDED=False MEMORY_INDEX="Autogpt" # name of the index to create for the application ``` From 67b84b58115df6e2dfa52eb6c6a400c60b50eeda Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 17:54:59 +0100 Subject: [PATCH 11/48] added client install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9959f55d..9f71ceaa 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ export PINECONE_ENV="Your pinecone region" # something like: us-east4-gcp ## Weaviate Setup [Weaviate](https://weaviate.io/) is an open-source vector database. It allows to store data objects and vector embeddings from ML-models and scales seamlessly to billion of data objects. [An instance of Weaviate can be created locally (using Docker), on Kubernetes or using Weaviate Cloud Services](https://weaviate.io/developers/weaviate/quickstart). -Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) is supported which allows the Auto-GPT process itself to start a Weaviate instance. To enable it, set `USE_WEAVIATE_EMBEDDED` to `True`. +Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) is supported which allows the Auto-GPT process itself to start a Weaviate instance. To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `pip install "weaviate-client>=3.15.4`. #### Setting up enviornment variables From 415c1cb4b5212617777c0aca4dab07847d68747d Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 18:55:34 +0100 Subject: [PATCH 12/48] fixed quotes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dd7547a2..3f5163f7 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ export PINECONE_ENV="Your pinecone region" # something like: us-east4-gcp ## Weaviate Setup [Weaviate](https://weaviate.io/) is an open-source vector database. It allows to store data objects and vector embeddings from ML-models and scales seamlessly to billion of data objects. [An instance of Weaviate can be created locally (using Docker), on Kubernetes or using Weaviate Cloud Services](https://weaviate.io/developers/weaviate/quickstart). -Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) is supported which allows the Auto-GPT process itself to start a Weaviate instance. To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `pip install "weaviate-client>=3.15.4`. +Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) is supported which allows the Auto-GPT process itself to start a Weaviate instance. To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `pip install "weaviate-client>=3.15.4"`. #### Setting up environment variables From 35ecd95c498708a00bf579053eb2f131c7721b83 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 18:56:42 +0100 Subject: [PATCH 13/48] removed unnecessary flush() --- scripts/memory/weaviate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/memory/weaviate.py b/scripts/memory/weaviate.py index 2eac5839..e494a931 100644 --- a/scripts/memory/weaviate.py +++ b/scripts/memory/weaviate.py @@ -66,8 +66,6 @@ class WeaviateMemory(MemoryProviderSingleton): vector=vector ) - batch.flush() - return f"Inserting data into memory at uuid: {doc_uuid}:\n data: {data}" def get(self, data): From b7d0cc3b247cf80de031bc5e80232834d5f84111 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 19:00:30 +0100 Subject: [PATCH 14/48] removed the extra class property --- scripts/memory/weaviate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/memory/weaviate.py b/scripts/memory/weaviate.py index e494a931..6de8be70 100644 --- a/scripts/memory/weaviate.py +++ b/scripts/memory/weaviate.py @@ -54,7 +54,6 @@ class WeaviateMemory(MemoryProviderSingleton): doc_uuid = generate_uuid5(data, self.index) data_object = { - 'class': self.index, 'raw_text': data } From 530894608b29cab2e0a4ac425894d4ca3f90398e Mon Sep 17 00:00:00 2001 From: cs0lar Date: Wed, 12 Apr 2023 19:09:52 +0100 Subject: [PATCH 15/48] added support of API key based auth --- .env.template | 1 + README.md | 3 ++- scripts/config.py | 3 ++- scripts/memory/weaviate.py | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.env.template b/.env.template index c8c33dd8..20736005 100644 --- a/.env.template +++ b/.env.template @@ -24,5 +24,6 @@ USE_WEAVIATE_EMBEDDED=False WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" WEAVIATE_USERNAME= WEAVIATE_PASSWORD= +WEAVIATE_API_KEY= MEMORY_INDEX="auto-gpt" MEMORY_BACKEND="local" diff --git a/README.md b/README.md index 3f5163f7..9b7753b4 100644 --- a/README.md +++ b/README.md @@ -226,7 +226,7 @@ export PINECONE_ENV="Your pinecone region" # something like: us-east4-gcp ## Weaviate Setup [Weaviate](https://weaviate.io/) is an open-source vector database. It allows to store data objects and vector embeddings from ML-models and scales seamlessly to billion of data objects. [An instance of Weaviate can be created locally (using Docker), on Kubernetes or using Weaviate Cloud Services](https://weaviate.io/developers/weaviate/quickstart). -Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) is supported which allows the Auto-GPT process itself to start a Weaviate instance. To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `pip install "weaviate-client>=3.15.4"`. +Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded) is supported which allows the Auto-GPT process itself to start a Weaviate instance. To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `pip install "weaviate-client>=3.15.4"`. #### Setting up environment variables @@ -239,6 +239,7 @@ WEAVIATE_PORT="8080" WEAVIATE_PROTOCOL="http" WEAVIATE_USERNAME="your username" WEAVIATE_PASSWORD="your password" +WEAVIATE_API_KEY="your weaviate API key if you have one" WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" # this is optional and indicates where the data should be persisted when running an embedded instance USE_WEAVIATE_EMBEDDED=False # set to True to run Embedded Weaviate MEMORY_INDEX="Autogpt" # name of the index to create for the application diff --git a/scripts/config.py b/scripts/config.py index 2c08f762..065e37dd 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -74,7 +74,8 @@ class Config(metaclass=Singleton): self.weaviate_username = os.getenv("WEAVIATE_USERNAME", None) self.weaviate_password = os.getenv("WEAVIATE_PASSWORD", None) self.weaviate_scopes = os.getenv("WEAVIATE_SCOPES", None) - self.weaviate_embedded_path = os.getenv('WEAVIATE_EMBEDDED_PATH', '~/.local/share/weaviate') + self.weaviate_embedded_path = os.getenv("WEAVIATE_EMBEDDED_PATH", "~/.local/share/weaviate") + self.weaviate_api_key = os.getenv("WEAVIATE_API_KEY", None) self.use_weaviate_embedded = os.getenv("USE_WEAVIATE_EMBEDDED", "False") == "True" self.image_provider = os.getenv("IMAGE_PROVIDER") diff --git a/scripts/memory/weaviate.py b/scripts/memory/weaviate.py index 6de8be70..b5710cb5 100644 --- a/scripts/memory/weaviate.py +++ b/scripts/memory/weaviate.py @@ -46,6 +46,8 @@ class WeaviateMemory(MemoryProviderSingleton): def _build_auth_credentials(self, cfg): if cfg.weaviate_username and cfg.weaviate_password: return weaviate_auth.AuthClientPassword(cfg.weaviate_username, cfg.weaviate_password) + if cfg.weaviate_api_key: + return weaviate.auth.AuthApiKey(api_key=cfg.weaviate_api_key) else: return None From 0c3562fcdd901bb399cf477564dfe814a80ff4bd Mon Sep 17 00:00:00 2001 From: cs0lar Date: Thu, 13 Apr 2023 18:50:56 +0100 Subject: [PATCH 16/48] fixed config bug --- scripts/memory/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/memory/base.py b/scripts/memory/base.py index 1d908531..1bb4e89f 100644 --- a/scripts/memory/base.py +++ b/scripts/memory/base.py @@ -3,6 +3,7 @@ import abc from config import AbstractSingleton, Config import openai +cfg = Config() def get_ada_embedding(text): text = text.replace("\n", " ") From 005be024f19dc0e95da8a339d5e1f5fec356dc64 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Sat, 15 Apr 2023 14:45:16 +0100 Subject: [PATCH 17/48] fixed typo --- .env.template | 1 - 1 file changed, 1 deletion(-) diff --git a/.env.template b/.env.template index e6ef227f..f9d1ec10 100644 --- a/.env.template +++ b/.env.template @@ -57,7 +57,6 @@ SMART_TOKEN_LIMIT=8000 ################################################################################ # MEMORY_BACKEND - Memory backend type (Default: local) ->>>>>>> master MEMORY_BACKEND=local ### PINECONE From b2bfd395ed3f2c458eb88d81fbb6fbd353e55eed Mon Sep 17 00:00:00 2001 From: cs0lar Date: Sat, 15 Apr 2023 15:49:24 +0100 Subject: [PATCH 18/48] fixed formatting --- autogpt/memory/__init__.py | 3 ++- autogpt/memory/base.py | 1 + autogpt/memory/weaviate.py | 5 +++-- tests/integration/weaviate_memory_tests.py | 11 +++++------ 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/autogpt/memory/__init__.py b/autogpt/memory/__init__.py index d56b2de2..7b545ea3 100644 --- a/autogpt/memory/__init__.py +++ b/autogpt/memory/__init__.py @@ -27,6 +27,7 @@ except ImportError: print("Weaviate not installed. Skipping import.") WeaviateMemory = None + def get_memory(cfg, init=False): memory = None if cfg.memory_backend == "pinecone": @@ -53,7 +54,7 @@ def get_memory(cfg, init=False): " use Weaviate as a memory backend.") else: memory = WeaviateMemory(cfg) - + elif cfg.memory_backend == "no_memory": memory = NoMemory(cfg) diff --git a/autogpt/memory/base.py b/autogpt/memory/base.py index 784483fa..691e2299 100644 --- a/autogpt/memory/base.py +++ b/autogpt/memory/base.py @@ -7,6 +7,7 @@ from autogpt.config import AbstractSingleton, Config cfg = Config() + def get_ada_embedding(text): text = text.replace("\n", " ") if cfg.use_azure: diff --git a/autogpt/memory/weaviate.py b/autogpt/memory/weaviate.py index fdac8e85..48816973 100644 --- a/autogpt/memory/weaviate.py +++ b/autogpt/memory/weaviate.py @@ -6,6 +6,7 @@ from weaviate import Client from weaviate.embedded import EmbeddedOptions from weaviate.util import generate_uuid5 + def default_schema(weaviate_index): return { "class": weaviate_index, @@ -18,6 +19,7 @@ def default_schema(weaviate_index): ], } + class WeaviateMemory(MemoryProviderSingleton): def __init__(self, cfg): auth_credentials = self._build_auth_credentials(cfg) @@ -72,12 +74,11 @@ class WeaviateMemory(MemoryProviderSingleton): def get(self, data): return self.get_relevant(data, 1) - def clear(self): self.client.schema.delete_all() # weaviate does not yet have a neat way to just remove the items in an index - # without removing the entire schema, therefore we need to re-create it + # without removing the entire schema, therefore we need to re-create it # after a call to delete_all self._create_schema() diff --git a/tests/integration/weaviate_memory_tests.py b/tests/integration/weaviate_memory_tests.py index fa456c8a..503fe9d2 100644 --- a/tests/integration/weaviate_memory_tests.py +++ b/tests/integration/weaviate_memory_tests.py @@ -11,6 +11,7 @@ from autogpt.config import Config from autogpt.memory.weaviate import WeaviateMemory from autogpt.memory.base import get_ada_embedding + @mock.patch.dict(os.environ, { "WEAVIATE_HOST": "127.0.0.1", "WEAVIATE_PROTOCOL": "http", @@ -38,13 +39,13 @@ class TestWeaviateMemory(unittest.TestCase): )) else: cls.client = Client(f"{cls.cfg.weaviate_protocol}://{cls.cfg.weaviate_host}:{self.cfg.weaviate_port}") - + """ In order to run these tests you will need a local instance of Weaviate running. Refer to https://weaviate.io/developers/weaviate/installation/docker-compose for creating local instances using docker. Alternatively in your .env file set the following environmental variables to run Weaviate embedded (see: https://weaviate.io/developers/weaviate/installation/embedded): - + USE_WEAVIATE_EMBEDDED=True WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" """ @@ -53,7 +54,7 @@ class TestWeaviateMemory(unittest.TestCase): self.client.schema.delete_class(self.cfg.memory_index) except: pass - + self.memory = WeaviateMemory(self.cfg) def test_add(self): @@ -67,7 +68,7 @@ class TestWeaviateMemory(unittest.TestCase): def test_get(self): doc = 'You are an Avenger and swore to defend the Galaxy from a menace called Thanos' - + with self.client.batch as batch: batch.add_data_object( uuid=get_valid_uuid(uuid4()), @@ -83,7 +84,6 @@ class TestWeaviateMemory(unittest.TestCase): self.assertEqual(len(actual), 1) self.assertEqual(actual[0], doc) - def test_get_stats(self): docs = [ 'You are now about to count the number of docs in this index', @@ -98,7 +98,6 @@ class TestWeaviateMemory(unittest.TestCase): self.assertTrue('count' in stats) self.assertEqual(stats['count'], 2) - def test_clear(self): docs = [ 'Shame this is the last test for this class', From 8916b76f113aef3926fd57f938ae27a783e04192 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Sat, 15 Apr 2023 18:52:59 +0100 Subject: [PATCH 19/48] fixed change request --- autogpt/memory/weaviate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/memory/weaviate.py b/autogpt/memory/weaviate.py index 48816973..3e112ca6 100644 --- a/autogpt/memory/weaviate.py +++ b/autogpt/memory/weaviate.py @@ -47,7 +47,7 @@ class WeaviateMemory(MemoryProviderSingleton): def _build_auth_credentials(self, cfg): if cfg.weaviate_username and cfg.weaviate_password: - return weaviate_auth.AuthClientPassword(cfg.weaviate_username, cfg.weaviate_password) + return weaviate.auth.AuthClientPassword(cfg.weaviate_username, cfg.weaviate_password) if cfg.weaviate_api_key: return weaviate.auth.AuthApiKey(api_key=cfg.weaviate_api_key) else: From 899c815676178eef2c68f2ff3984474c25cc41f4 Mon Sep 17 00:00:00 2001 From: cs0lar Date: Sat, 15 Apr 2023 18:55:45 +0100 Subject: [PATCH 20/48] fixed auth code --- autogpt/memory/weaviate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autogpt/memory/weaviate.py b/autogpt/memory/weaviate.py index 3e112ca6..6fcce0a0 100644 --- a/autogpt/memory/weaviate.py +++ b/autogpt/memory/weaviate.py @@ -47,9 +47,9 @@ class WeaviateMemory(MemoryProviderSingleton): def _build_auth_credentials(self, cfg): if cfg.weaviate_username and cfg.weaviate_password: - return weaviate.auth.AuthClientPassword(cfg.weaviate_username, cfg.weaviate_password) + return weaviate.AuthClientPassword(cfg.weaviate_username, cfg.weaviate_password) if cfg.weaviate_api_key: - return weaviate.auth.AuthApiKey(api_key=cfg.weaviate_api_key) + return weaviate.AuthApiKey(api_key=cfg.weaviate_api_key) else: return None From 2576b299e7d3476a3173b0a2913a960cebc5771b Mon Sep 17 00:00:00 2001 From: chao ma Date: Sun, 16 Apr 2023 12:42:01 +0800 Subject: [PATCH 21/48] Fix google api fetch error --- autogpt/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/autogpt/app.py b/autogpt/app.py index e7b16adc..62a6848e 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -125,10 +125,11 @@ def execute_command(command_name: str, arguments): key = CFG.google_api_key if key and key.strip() and key != "your-google-api-key": google_result = google_official_search(arguments["input"]) + return google_result else: google_result = google_search(arguments["input"]) - safe_message = google_result.encode("utf-8", "ignore") - return str(safe_message) + safe_message = google_result.encode("utf-8", "ignore") + return str(safe_message) elif command_name == "memory_add": return memory.add(arguments["string"]) elif command_name == "start_agent": From 4cd412c39fb26cb6a3b4aa8e1de1646dfc800d39 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 01:55:34 -0500 Subject: [PATCH 22/48] Update requirements.txt --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4abb8b43..1cdedec2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,6 @@ pinecone-client==2.2.1 redis orjson Pillow -weaviate-client==3.15.5 selenium webdriver-manager coverage @@ -28,4 +27,4 @@ isort gitpython==3.1.31 pytest pytest-mock -tweepy \ No newline at end of file +tweepy From b865e2c2f85893724210c34e1090f94bbae5e8e3 Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 02:08:38 -0500 Subject: [PATCH 23/48] Fix README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 501429e0..00ada54f 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Development of this free, open-source project is made possible by all the Date: Sun, 16 Apr 2023 19:18:08 +1200 Subject: [PATCH 24/48] Updates Sponsors --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 00ada54f..fcff589f 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,12 @@ Development of this free, open-source project is made possible by all the InfluxData    Roost.AI    NucleiAI    AlgohashFe    

-

Monthly Sponsors

+

Individual Sponsors

robinicus  prompthero  crizzler  tob-le-rone  FSTatSBS  toverly1  ddtarazona  Nalhos  Kazamario  pingbotan  indoor47  AuroraHolding  kreativai  hunteraraujo  Explorergt92  judegomila   thepok   SpacingLily  merwanehamadi  m  zkonduit  maxxflyer  tekelsey  digisomni  nocodeclarity  tjarmain -avy-ai  garythebat  Web3Capital  throb  shawnharmsen  MediConCenHK  Mobivs  GalaxyVideoAgency  quintendf  RThaweewat  SwftCoins  MBassi91  Odin519Tomas  Dradstone  lucas-chu  joaomdmoura  comet-ml  sultanmeghji  Brodie0  fabrietech  omphos  ZERO-A-ONE  jazgarewal  vkozacek  ternary5  josephcmiller2  ikarosai  DailyBotHQ  belharethsami  DataMetis  st617  cfarquhar  ColinConwell  Pythagora-io  dwcar49us  KiaArmani  lmaugustin  MetaPath01  scryptedinc  nicoguyon  refinery1  johnculkin  Cameron-Fulton  mathewhawkins  Mr-Bishop42  rejunity  caitlynmeeks  allenstecat  Daniel1357  rapidstartup  sunchongren  marv-technology  TheStoneMX  concreit  AryaXAI  abhinav-pandey29  tob-le-rone  angiaou  rickscode  RealChrisSean  thisisjeffchen  tommygeee  CrypteorCapital  kMag410  ChrisDMT  jd3655  rocks6  webbcolton  projectonegames  jun784  fruition  txtr99  

+Dradstone  CrypteorCapital  avy-ai  shawnharmsen  sunchongren  DailyBotHQ  mathewhawkins  MediConCenHK  kMag410  nicoguyon  Mobivs  jazgarewal  marv-technology  rapidstartup  Brodie0  lucas-chu  rejunity  comet-ml  ColinConwell  cfarquhar  ikarosai  ChrisDMT  Odin519Tomas  vkozacek  belharethsami  sultanmeghji  scryptedinc  johnculkin  RealChrisSean  fruition  jd3655  Web3Capital  allenstecat  tob-le-rone  SwftCoins  MetaPath01  joaomdmoura  ternary5  refinery1  josephcmiller2  webbcolton  tommygeee  lmaugustin  garythebat  Cameron-Fulton  angiaou  caitlynmeeks  MBassi91  Daniel1357  omphos  abhinav-pandey29  DataMetis  concreit  st617  RThaweewat  KiaArmani  Pythagora-io  AryaXAI  fabrietech  jun784  Mr-Bishop42  rickscode  projectonegames  rocks6  GalaxyVideoAgency  thisisjeffchen  TheStoneMX  txtr99  ZERO-A-ONE  

From bf98791330b0d59815cc614a64874b6b54558e28 Mon Sep 17 00:00:00 2001 From: GyDi Date: Sun, 16 Apr 2023 15:50:26 +0800 Subject: [PATCH 25/48] fix: add necessary space to the prompt --- autogpt/promptgenerator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/promptgenerator.py b/autogpt/promptgenerator.py index 82dba8e0..4f518615 100644 --- a/autogpt/promptgenerator.py +++ b/autogpt/promptgenerator.py @@ -132,5 +132,5 @@ class PromptGenerator: f"{self._generate_numbered_list(self.performance_evaluation)}\n\n" "You should only respond in JSON format as described below \nResponse" f" Format: \n{formatted_response_format} \nEnsure the response can be" - "parsed by Python json.loads" + " parsed by Python json.loads" ) From bc09ce93eb0708ad917eba92d44d2b4e48790349 Mon Sep 17 00:00:00 2001 From: Richard Beales Date: Sun, 16 Apr 2023 09:20:16 +0100 Subject: [PATCH 26/48] Create a Docker image on DockerHub on release to stable --- .github/workflows/dockerhub-imagepush.yml | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/dockerhub-imagepush.yml diff --git a/.github/workflows/dockerhub-imagepush.yml b/.github/workflows/dockerhub-imagepush.yml new file mode 100644 index 00000000..6805eebf --- /dev/null +++ b/.github/workflows/dockerhub-imagepush.yml @@ -0,0 +1,24 @@ +name: Push Docker Image on Release + +on: + push: + branches: [ "stable" ] + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Log in to Docker hub + env: + DOCKER_USER: ${{secrets.DOCKER_USER}} + DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}} + run: | + docker login -u $DOCKER_USER -p $DOCKER_PASSWORD + - name: Build the Docker image + run: docker build . --file Dockerfile --tag ${{secrets.DOCKER_USER}}/auto-gpt:$(git describe --tags `git rev-list --tags --max-count=1`) + - name: Docker Push + run: docker push ${{secrets.DOCKER_USER}}/auto-gpt From 4f3bb609df261cd9ff2e1a6b528088254f64dc40 Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Sun, 16 Apr 2023 14:23:02 +0530 Subject: [PATCH 27/48] Remove least relevant items from memory first --- autogpt/chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/chat.py b/autogpt/chat.py index b0886967..1f6bca96 100644 --- a/autogpt/chat.py +++ b/autogpt/chat.py @@ -96,7 +96,7 @@ def chat_with_ai( while current_tokens_used > 2500: # remove memories until we are under 2500 tokens - relevant_memory = relevant_memory[1:] + relevant_memory = relevant_memory[:-1] ( next_message_to_add_index, current_tokens_used, From 405632f187f066c5873cfc74d660cf36a4181dd0 Mon Sep 17 00:00:00 2001 From: HDKiller Date: Sun, 16 Apr 2023 08:57:23 +0000 Subject: [PATCH 28/48] Only add audio to text command to the prompt if model is set --- autogpt/prompt.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/autogpt/prompt.py b/autogpt/prompt.py index 97bacb71..5cc98cd3 100644 --- a/autogpt/prompt.py +++ b/autogpt/prompt.py @@ -82,10 +82,19 @@ def get_prompt() -> str: ), ("Execute Python File", "execute_python_file", {"file": ""}), ("Generate Image", "generate_image", {"prompt": ""}), - ("Convert Audio to text", "read_audio_from_file", {"file": ""}), ("Send Tweet", "send_tweet", {"text": ""}), ] + # Only add the audio to text command if the model is specified + if cfg.huggingface_audio_to_text_model: + commands.append( + ( + "Convert Audio to text", + "read_audio_from_file", + {"file": ""} + ), + ) + # Only add shell command to the prompt if the AI is allowed to execute it if cfg.execute_local_commands: commands.append( From ad7cefa10c0647feee85114d58559fcf83ba6743 Mon Sep 17 00:00:00 2001 From: 0xArty Date: Sun, 16 Apr 2023 10:30:59 +0100 Subject: [PATCH 29/48] updated contributing docs --- CODE_OF_CONDUCT.md | 40 ++++++++++++++ CONTRIBUTING.md | 135 ++++++++++++++++++++++++++++----------------- 2 files changed, 125 insertions(+), 50 deletions(-) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..d2331b4c --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,40 @@ +# Code of Conduct for auto-gpt + +## 1. Purpose + +The purpose of this Code of Conduct is to provide guidelines for contributors to the auto-gpt project on GitHub. We aim to create a positive and inclusive environment where all participants can contribute and collaborate effectively. By participating in this project, you agree to abide by this Code of Conduct. + +## 2. Scope + +This Code of Conduct applies to all contributors, maintainers, and users of the auto-gpt project. It extends to all project spaces, including but not limited to issues, pull requests, code reviews, comments, and other forms of communication within the project. + +## 3. Our Standards + +We encourage the following behavior: + +* Being respectful and considerate to others +* Actively seeking diverse perspectives +* Providing constructive feedback and assistance +* Demonstrating empathy and understanding + +We discourage the following behavior: + +* Harassment or discrimination of any kind +* Disrespectful, offensive, or inappropriate language or content +* Personal attacks or insults +* Unwarranted criticism or negativity + +## 4. Reporting and Enforcement + +If you witness or experience any violations of this Code of Conduct, please report them to the project maintainers by email or other appropriate means. The maintainers will investigate and take appropriate action, which may include warnings, temporary or permanent bans, or other measures as necessary. + +Maintainers are responsible for ensuring compliance with this Code of Conduct and may take action to address any violations. + +## 5. Acknowledgements + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/version/2/0/code_of_conduct.html). + +## 6. Contact + +If you have any questions or concerns, please contact the project maintainers. + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 49c95991..b2a2490c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,64 +1,99 @@ +# Contributing to ProjectName -To contribute to this GitHub project, you can follow these steps: +First of all, thank you for considering contributing to our project! We appreciate your time and effort, and we value any contribution, whether it's reporting a bug, suggesting a new feature, or submitting a pull request. -1. Fork the repository you want to contribute to by clicking the "Fork" button on the project page. +This document provides guidelines and best practices to help you contribute effectively. -2. Clone the repository to your local machine using the following command: +## Table of Contents +- [Code of Conduct](#code-of-conduct) +- [Getting Started](#getting-started) +- [How to Contribute](#how-to-contribute) + - [Reporting Bugs](#reporting-bugs) + - [Suggesting Enhancements](#suggesting-enhancements) + - [Submitting Pull Requests](#submitting-pull-requests) +- [Style Guidelines](#style-guidelines) + - [Code Formatting](#code-formatting) + - [Pre-Commit Hooks](#pre-commit-hooks) + +## Code of Conduct + +By participating in this project, you agree to abide by our [Code of Conduct](CODE_OF_CONDUCT.md). Please read it to understand the expectations we have for everyone who contributes to this project. + +## Getting Started + +To start contributing, follow these steps: + +1. Fork the repository and clone your fork. +2. Create a new branch for your changes (use a descriptive name, such as `fix-bug-123` or `add-new-feature`). +3. Make your changes in the new branch. +4. Test your changes thoroughly. +5. Commit and push your changes to your fork. +6. Create a pull request following the guidelines in the [Submitting Pull Requests](#submitting-pull-requests) section. + +## How to Contribute + +### Reporting Bugs + +If you find a bug in the project, please create an issue on GitHub with the following information: + +- A clear, descriptive title for the issue. +- A description of the problem, including steps to reproduce the issue. +- Any relevant logs, screenshots, or other supporting information. + +### Suggesting Enhancements + +If you have an idea for a new feature or improvement, please create an issue on GitHub with the following information: + +- A clear, descriptive title for the issue. +- A detailed description of the proposed enhancement, including any benefits and potential drawbacks. +- Any relevant examples, mockups, or supporting information. + +### Submitting Pull Requests + +When submitting a pull request, please ensure that your changes meet the following criteria: + +- Your pull request should be atomic and focus on a single change. +- Your pull request should include tests for your change. +- You should have thoroughly tested your changes with multiple different prompts. +- You should have considered potential risks and mitigations for your changes. +- You should have documented your changes clearly and comprehensively. +- You should not include any unrelated or "extra" small tweaks or changes. + +## Style Guidelines + +### Code Formatting + +We use the `black` code formatter to maintain a consistent coding style across the project. Please ensure that your code is formatted using `black` before submitting a pull request. You can install `black` using `pip`: + +```bash +pip install black ``` -git clone https://github.com//Auto-GPT + +To format your code, run the following command in the project's root directory: + +```bash +black . ``` -3. Install the project requirements -``` -pip install -r requirements.txt -``` -4. Install pre-commit hooks +### Pre-Commit Hooks +We use pre-commit hooks to ensure that code formatting and other checks are performed automatically before each commit. To set up pre-commit hooks for this project, follow these steps: + +Install the pre-commit package using pip: +```bash +pip install pre-commit ``` + +Run the following command in the project's root directory to install the pre-commit hooks: +```bash pre-commit install ``` -5. Create a new branch for your changes using the following command: -``` -git checkout -b "branch-name" -``` -6. Make your changes to the code or documentation. -- Example: Improve User Interface or Add Documentation. +Now, the pre-commit hooks will run automatically before each commit, checking your code formatting and other requirements. +If you encounter any issues or have questions, feel free to reach out to the maintainers or open a new issue on GitHub. We're here to help and appreciate your efforts to contribute to the project. -7. Add the changes to the staging area using the following command: -``` -git add . -``` +Happy coding, and once again, thank you for your contributions! -8. Commit the changes with a meaningful commit message using the following command: -``` -git commit -m "your commit message" -``` -9. Push the changes to your forked repository using the following command: -``` -git push origin branch-name -``` -10. Go to the GitHub website and navigate to your forked repository. +Maintainers will look at PR that have no merge conflicts when deciding what to add to the project. Make sure your PR shows up here: -11. Click the "New pull request" button. - -12. Select the branch you just pushed to and the branch you want to merge into on the original repository. - -13. Add a description of your changes and click the "Create pull request" button. - -14. Wait for the project maintainer to review your changes and provide feedback. - -15. Make any necessary changes based on feedback and repeat steps 5-12 until your changes are accepted and merged into the main project. - -16. Once your changes are merged, you can update your forked repository and local copy of the repository with the following commands: - -``` -git fetch upstream -git checkout master -git merge upstream/master -``` -Finally, delete the branch you created with the following command: -``` -git branch -d branch-name -``` -That's it you made it 🐣⭐⭐ +https://github.com/Torantulino/Auto-GPT/pulls?q=is%3Apr+is%3Aopen+-is%3Aconflict+ \ No newline at end of file From fb9430da0abd73916c5ca31196fc0aea15664ac4 Mon Sep 17 00:00:00 2001 From: Sabin Mendiguren Date: Sun, 16 Apr 2023 09:12:50 -0700 Subject: [PATCH 30/48] Update .env.template Small fix for the TEMPERATURE to show the real default value --- .env.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.template b/.env.template index 685ed19f..eeff2907 100644 --- a/.env.template +++ b/.env.template @@ -21,7 +21,7 @@ AI_SETTINGS_FILE=ai_settings.yaml ### OPENAI # OPENAI_API_KEY - OpenAI API Key (Example: my-openai-api-key) -# TEMPERATURE - Sets temperature in OpenAI (Default: 1) +# TEMPERATURE - Sets temperature in OpenAI (Default: 0) # USE_AZURE - Use Azure OpenAI or not (Default: False) OPENAI_API_KEY=your-openai-api-key TEMPERATURE=0 From 4a67c687c3778d33f2860e8ec489ac86b4f99066 Mon Sep 17 00:00:00 2001 From: Bently Date: Sun, 16 Apr 2023 17:20:30 +0100 Subject: [PATCH 31/48] simply removing a duplicate "Milvus Setup" in the README.md --- README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/README.md b/README.md index fcff589f..d9a5463d 100644 --- a/README.md +++ b/README.md @@ -375,18 +375,6 @@ WEAVIATE_EMBEDDED_PATH="/home/me/.local/share/weaviate" # this is optional and i USE_WEAVIATE_EMBEDDED=False # set to True to run Embedded Weaviate MEMORY_INDEX="Autogpt" # name of the index to create for the application ``` - -### Milvus Setup - -[Milvus](https://milvus.io/) is a open-source, high scalable vector database to storage huge amount of vector-based memory and provide fast relevant search. - -- setup milvus database, keep your pymilvus version and milvus version same to avoid compatible issues. - - setup by open source [Install Milvus](https://milvus.io/docs/install_standalone-operator.md) - - or setup by [Zilliz Cloud](https://zilliz.com/cloud) -- set `MILVUS_ADDR` in `.env` to your milvus address `host:ip`. -- set `MEMORY_BACKEND` in `.env` to `milvus` to enable milvus as backend. -- optional - - set `MILVUS_COLLECTION` in `.env` to change milvus collection name as you want, `autogpt` is the default name. ## View Memory Usage From 5b428f509bb69b76c426aab3c19ba4c52fdb16ed Mon Sep 17 00:00:00 2001 From: Steve Date: Sun, 16 Apr 2023 14:33:20 +0000 Subject: [PATCH 32/48] fix file logging issue --- autogpt/commands/file_operations.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index d02b125a..5350d2f4 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -45,7 +45,7 @@ def log_operation(operation: str, filename: str) -> None: with open(LOG_FILE_PATH, "w", encoding="utf-8") as f: f.write("File Operation Logger ") - append_to_file(LOG_FILE, log_entry) + append_to_file(LOG_FILE, log_entry, shouldLog = False) def safe_join(base: str, *paths) -> str: @@ -171,7 +171,7 @@ def write_to_file(filename: str, text: str) -> str: return f"Error: {str(e)}" -def append_to_file(filename: str, text: str) -> str: +def append_to_file(filename: str, text: str, shouldLog: bool = True) -> str: """Append text to a file Args: @@ -185,7 +185,10 @@ def append_to_file(filename: str, text: str) -> str: filepath = safe_join(WORKING_DIRECTORY, filename) with open(filepath, "a") as f: f.write(text) - log_operation("append", filename) + + if shouldLog: + log_operation("append", filename) + return "Text appended successfully." except Exception as e: return f"Error: {str(e)}" From ccf3c7b89e38965d3e55b310e3e3606722c60b54 Mon Sep 17 00:00:00 2001 From: Pi Date: Sun, 16 Apr 2023 17:24:18 +0100 Subject: [PATCH 33/48] Update file_operations.py --- autogpt/commands/file_operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index 5350d2f4..31500e8e 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -185,7 +185,7 @@ def append_to_file(filename: str, text: str, shouldLog: bool = True) -> str: filepath = safe_join(WORKING_DIRECTORY, filename) with open(filepath, "a") as f: f.write(text) - + if shouldLog: log_operation("append", filename) From 83930335f0ab984d79e69debce9fdd5a3d55c7d2 Mon Sep 17 00:00:00 2001 From: liuyachen <364579759@qq.com> Date: Sun, 16 Apr 2023 19:47:29 +0800 Subject: [PATCH 34/48] Fix README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index fcff589f..7e012959 100644 --- a/README.md +++ b/README.md @@ -189,18 +189,18 @@ Here are some common arguments you can use when running Auto-GPT: > Replace anything in angled brackets (<>) to a value you want to specify * View all available command line arguments ```bash -python scripts/main.py --help +python -m autogpt --help ``` * Run Auto-GPT with a different AI Settings file ```bash -python scripts/main.py --ai-settings +python -m autogpt --ai-settings ``` * Specify one of 3 memory backends: `local`, `redis`, `pinecone` or `no_memory` ```bash -python scripts/main.py --use-memory +python -m autogpt --use-memory ``` -> **NOTE**: There are shorthands for some of these flags, for example `-m` for `--use-memory`. Use `python scripts/main.py --help` for more information +> **NOTE**: There are shorthands for some of these flags, for example `-m` for `--use-memory`. Use `python -m autogpt --help` for more information ## 🗣️ Speech Mode From c3f01d9b2fe36cfdf70c81db69673a8c5da47c76 Mon Sep 17 00:00:00 2001 From: GyDi Date: Sun, 16 Apr 2023 18:25:28 +0800 Subject: [PATCH 35/48] fix: config save and load path inconsistent --- autogpt/prompt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/prompt.py b/autogpt/prompt.py index 5cc98cd3..18a5736c 100644 --- a/autogpt/prompt.py +++ b/autogpt/prompt.py @@ -177,7 +177,7 @@ Continue (y/n): """ if not config.ai_name: config = prompt_user() - config.save() + config.save(CFG.ai_settings_file) # Get rid of this global: global ai_name From 11620cc57188b57b413e38f4c6873c3304bd4906 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Sun, 16 Apr 2023 18:52:22 +0200 Subject: [PATCH 36/48] Fix and consolidate command workspace resolution --- autogpt/commands/audio_text.py | 6 ++-- autogpt/commands/execute_code.py | 14 ++++----- autogpt/commands/file_operations.py | 45 ++++++----------------------- autogpt/commands/image_gen.py | 8 ++--- autogpt/workspace.py | 39 +++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 53 deletions(-) create mode 100644 autogpt/workspace.py diff --git a/autogpt/commands/audio_text.py b/autogpt/commands/audio_text.py index b9ca988c..84819d5e 100644 --- a/autogpt/commands/audio_text.py +++ b/autogpt/commands/audio_text.py @@ -2,15 +2,13 @@ import requests import json from autogpt.config import Config -from autogpt.commands.file_operations import safe_join +from autogpt.workspace import path_in_workspace cfg = Config() -working_directory = "auto_gpt_workspace" - def read_audio_from_file(audio_path): - audio_path = safe_join(working_directory, audio_path) + audio_path = path_in_workspace(audio_path) with open(audio_path, "rb") as audio_file: audio = audio_file.read() return read_audio(audio) diff --git a/autogpt/commands/execute_code.py b/autogpt/commands/execute_code.py index 86d6c177..eaafa00a 100644 --- a/autogpt/commands/execute_code.py +++ b/autogpt/commands/execute_code.py @@ -1,12 +1,11 @@ """Execute code in a Docker container""" import os -from pathlib import Path import subprocess import docker from docker.errors import ImageNotFound -WORKING_DIRECTORY = Path(__file__).parent.parent / "auto_gpt_workspace" +from autogpt.workspace import path_in_workspace, WORKSPACE_PATH def execute_python_file(file: str): @@ -19,12 +18,12 @@ def execute_python_file(file: str): str: The output of the file """ - print(f"Executing file '{file}' in workspace '{WORKING_DIRECTORY}'") + print(f"Executing file '{file}' in workspace '{WORKSPACE_PATH}'") if not file.endswith(".py"): return "Error: Invalid file type. Only .py files are allowed." - file_path = os.path.join(WORKING_DIRECTORY, file) + file_path = path_in_workspace(file) if not os.path.isfile(file_path): return f"Error: File '{file}' does not exist." @@ -65,7 +64,7 @@ def execute_python_file(file: str): image_name, f"python {file}", volumes={ - os.path.abspath(WORKING_DIRECTORY): { + os.path.abspath(WORKSPACE_PATH): { "bind": "/workspace", "mode": "ro", } @@ -100,9 +99,8 @@ def execute_shell(command_line: str) -> str: """ current_dir = os.getcwd() # Change dir into workspace if necessary - if str(WORKING_DIRECTORY) not in current_dir: - work_dir = os.path.join(os.getcwd(), WORKING_DIRECTORY) - os.chdir(work_dir) + if str(WORKSPACE_PATH) not in current_dir: + os.chdir(WORKSPACE_PATH) print(f"Executing command '{command_line}' in working directory '{os.getcwd()}'") diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index 31500e8e..7ce90a38 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -1,19 +1,11 @@ """File operations for AutoGPT""" import os import os.path -from pathlib import Path +from autogpt.workspace import path_in_workspace, WORKSPACE_PATH from typing import Generator, List -# Set a dedicated folder for file I/O -WORKING_DIRECTORY = Path(os.getcwd()) / "auto_gpt_workspace" - -# Create the directory if it doesn't exist -if not os.path.exists(WORKING_DIRECTORY): - os.makedirs(WORKING_DIRECTORY) - LOG_FILE = "file_logger.txt" -LOG_FILE_PATH = WORKING_DIRECTORY / LOG_FILE -WORKING_DIRECTORY = str(WORKING_DIRECTORY) +LOG_FILE_PATH = WORKSPACE_PATH / LOG_FILE def check_duplicate_operation(operation: str, filename: str) -> bool: @@ -48,25 +40,6 @@ def log_operation(operation: str, filename: str) -> None: append_to_file(LOG_FILE, log_entry, shouldLog = False) -def safe_join(base: str, *paths) -> str: - """Join one or more path components intelligently. - - Args: - base (str): The base path - *paths (str): The paths to join to the base path - - Returns: - str: The joined path - """ - new_path = os.path.join(base, *paths) - norm_new_path = os.path.normpath(new_path) - - if os.path.commonprefix([base, norm_new_path]) != base: - raise ValueError("Attempted to access outside of working directory.") - - return norm_new_path - - def split_file( content: str, max_length: int = 4000, overlap: int = 0 ) -> Generator[str, None, None]: @@ -104,7 +77,7 @@ def read_file(filename: str) -> str: str: The contents of the file """ try: - filepath = safe_join(WORKING_DIRECTORY, filename) + filepath = path_in_workspace(filename) with open(filepath, "r", encoding="utf-8") as f: content = f.read() return content @@ -159,7 +132,7 @@ def write_to_file(filename: str, text: str) -> str: if check_duplicate_operation("write", filename): return "Error: File has already been updated." try: - filepath = safe_join(WORKING_DIRECTORY, filename) + filepath = path_in_workspace(filename) directory = os.path.dirname(filepath) if not os.path.exists(directory): os.makedirs(directory) @@ -182,7 +155,7 @@ def append_to_file(filename: str, text: str, shouldLog: bool = True) -> str: str: A message indicating success or failure """ try: - filepath = safe_join(WORKING_DIRECTORY, filename) + filepath = path_in_workspace(filename) with open(filepath, "a") as f: f.write(text) @@ -206,7 +179,7 @@ def delete_file(filename: str) -> str: if check_duplicate_operation("delete", filename): return "Error: File has already been deleted." try: - filepath = safe_join(WORKING_DIRECTORY, filename) + filepath = path_in_workspace(filename) os.remove(filepath) log_operation("delete", filename) return "File deleted successfully." @@ -226,15 +199,15 @@ def search_files(directory: str) -> List[str]: found_files = [] if directory in {"", "/"}: - search_directory = WORKING_DIRECTORY + search_directory = WORKSPACE_PATH else: - search_directory = safe_join(WORKING_DIRECTORY, directory) + search_directory = path_in_workspace(directory) for root, _, files in os.walk(search_directory): for file in files: if file.startswith("."): continue - relative_path = os.path.relpath(os.path.join(root, file), WORKING_DIRECTORY) + relative_path = os.path.relpath(os.path.join(root, file), WORKSPACE_PATH) found_files.append(relative_path) return found_files diff --git a/autogpt/commands/image_gen.py b/autogpt/commands/image_gen.py index 39e08845..6243616e 100644 --- a/autogpt/commands/image_gen.py +++ b/autogpt/commands/image_gen.py @@ -7,13 +7,11 @@ from base64 import b64decode import openai import requests from PIL import Image -from pathlib import Path from autogpt.config import Config +from autogpt.workspace import path_in_workspace CFG = Config() -WORKING_DIRECTORY = Path(__file__).parent.parent / "auto_gpt_workspace" - def generate_image(prompt: str) -> str: """Generate an image from a prompt. @@ -65,7 +63,7 @@ def generate_image_with_hf(prompt: str, filename: str) -> str: image = Image.open(io.BytesIO(response.content)) print(f"Image Generated for prompt:{prompt}") - image.save(os.path.join(WORKING_DIRECTORY, filename)) + image.save(path_in_workspace(filename)) return f"Saved to disk:{filename}" @@ -93,7 +91,7 @@ def generate_image_with_dalle(prompt: str, filename: str) -> str: image_data = b64decode(response["data"][0]["b64_json"]) - with open(f"{WORKING_DIRECTORY}/{filename}", mode="wb") as png: + with open(path_in_workspace(filename), mode="wb") as png: png.write(image_data) return f"Saved to disk:{filename}" diff --git a/autogpt/workspace.py b/autogpt/workspace.py new file mode 100644 index 00000000..79134919 --- /dev/null +++ b/autogpt/workspace.py @@ -0,0 +1,39 @@ +import os +from pathlib import Path + +# Set a dedicated folder for file I/O +WORKSPACE_PATH = Path(os.getcwd()) / "auto_gpt_workspace" + +# Create the directory if it doesn't exist +if not os.path.exists(WORKSPACE_PATH): + os.makedirs(WORKSPACE_PATH) + + +def path_in_workspace(relative_path: str | Path) -> Path: + """Get full path for item in workspace + + Parameters: + relative_path (str | Path): Path to translate into the workspace + + Returns: + Path: Absolute path for the given path in the workspace + """ + return safe_path_join(WORKSPACE_PATH, relative_path) + + +def safe_path_join(base: Path, *paths: str | Path) -> Path: + """Join one or more path components, asserting the resulting path is within the workspace. + + Args: + base (Path): The base path + *paths (str): The paths to join to the base path + + Returns: + Path: The joined path + """ + joined_path = base.joinpath(*paths).resolve() + + if not joined_path.is_relative_to(base): + raise ValueError(f"Attempted to access path '{joined_path}' outside of working directory '{base}'.") + + return joined_path From 5698689361449854a865e8e1b28340f19e2a386e Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Sun, 16 Apr 2023 16:38:03 +0200 Subject: [PATCH 37/48] Update bug report template Add GPT-3 checkbox & emphasize to search for existing issues first --- .github/ISSUE_TEMPLATE/1.bug.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/1.bug.yml b/.github/ISSUE_TEMPLATE/1.bug.yml index e2404c76..6e6d00ae 100644 --- a/.github/ISSUE_TEMPLATE/1.bug.yml +++ b/.github/ISSUE_TEMPLATE/1.bug.yml @@ -2,6 +2,15 @@ name: Bug report 🐛 description: Create a bug report for Auto-GPT. labels: ['status: needs triage'] body: + - type: checkboxes + attributes: + label: ⚠️ Search for existing issues first ⚠️ + description: > + Please [search the history](https://github.com/Torantulino/Auto-GPT/issues) + to see if an issue already exists for the same problem. + options: + - label: I have searched the existing issues, and there is no existing issue for my problem + required: true - type: markdown attributes: value: | @@ -19,13 +28,14 @@ body: - Provide commit-hash (`git rev-parse HEAD` gets it) - If it's a pip/packages issue, provide pip version, python version - If it's a crash, provide traceback. - - type: checkboxes attributes: - label: Duplicates - description: Please [search the history](https://github.com/Torantulino/Auto-GPT/issues) to see if an issue already exists for the same problem. + label: GPT-3 or GPT-4 + description: > + If you are using Auto-GPT with `--gpt3only`, your problems may be caused by + the limitations of GPT-3.5 options: - - label: I have searched the existing issues + - label: I am using Auto-GPT with GPT-3 (GPT-3.5) required: true - type: textarea attributes: From 41a0a687827ad16d51af79497a9d2b903af774d8 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Sun, 16 Apr 2023 16:44:08 +0200 Subject: [PATCH 38/48] fix(issue template): GPT-3 checkbox not required --- .github/ISSUE_TEMPLATE/1.bug.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/1.bug.yml b/.github/ISSUE_TEMPLATE/1.bug.yml index 6e6d00ae..7f1d2771 100644 --- a/.github/ISSUE_TEMPLATE/1.bug.yml +++ b/.github/ISSUE_TEMPLATE/1.bug.yml @@ -36,7 +36,6 @@ body: the limitations of GPT-3.5 options: - label: I am using Auto-GPT with GPT-3 (GPT-3.5) - required: true - type: textarea attributes: label: Steps to reproduce 🕹 From 92ab3e0e8b2e430c2a82c765220a35098aa75157 Mon Sep 17 00:00:00 2001 From: Peter Svensson Date: Sun, 16 Apr 2023 13:25:04 +0200 Subject: [PATCH 39/48] fixes #1821 by installing required drivers and adding options to chromedriver --- Dockerfile | 3 ++- autogpt/commands/web_selenium.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 309b857c..82672c94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ FROM python:3.11-slim # Install git RUN apt-get -y update RUN apt-get -y install git +RUN apt-get install -y libglib2.0 libnss3 libgconf-2-4 libfontconfig1 chromium-driver # Set environment variables ENV PIP_NO_CACHE_DIR=yes \ @@ -24,4 +25,4 @@ RUN pip install --no-cache-dir --user -r requirements-docker.txt COPY --chown=appuser:appuser autogpt/ ./autogpt # Set the entrypoint -ENTRYPOINT ["python", "-m", "autogpt"] +ENTRYPOINT ["python", "-m", "autogpt", "--debug"] diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 359803ee..3dbd1b98 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -74,6 +74,10 @@ def scrape_text_with_selenium(url: str) -> Tuple[WebDriver, str]: # See https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari driver = webdriver.Safari(options=options) else: + options.add_argument('--no-sandbox') + options.add_argument('--window-size=1420,1080') + options.add_argument('--headless') + options.add_argument('--disable-gpu') driver = webdriver.Chrome( executable_path=ChromeDriverManager().install(), options=options ) From e6d2de78932c156809550898b51d919aa12748c0 Mon Sep 17 00:00:00 2001 From: Peter Svensson Date: Sun, 16 Apr 2023 13:34:48 +0200 Subject: [PATCH 40/48] removed debug flag --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 82672c94..8c3c0d8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,4 +25,4 @@ RUN pip install --no-cache-dir --user -r requirements-docker.txt COPY --chown=appuser:appuser autogpt/ ./autogpt # Set the entrypoint -ENTRYPOINT ["python", "-m", "autogpt", "--debug"] +ENTRYPOINT ["python", "-m", "autogpt"] From cd78f21b51ce3ddd786338650a099ae4ea5100f2 Mon Sep 17 00:00:00 2001 From: bvoo <60059541+bvoo@users.noreply.github.com> Date: Sun, 16 Apr 2023 05:27:14 -0700 Subject: [PATCH 41/48] cleanup --- Dockerfile | 3 +-- autogpt/commands/web_selenium.py | 10 ++++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8c3c0d8c..9886d742 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,7 @@ FROM python:3.11-slim # Install git RUN apt-get -y update -RUN apt-get -y install git -RUN apt-get install -y libglib2.0 libnss3 libgconf-2-4 libfontconfig1 chromium-driver +RUN apt-get -y install git chromium-driver # Set environment variables ENV PIP_NO_CACHE_DIR=yes \ diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 3dbd1b98..021d149c 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -64,6 +64,12 @@ def scrape_text_with_selenium(url: str) -> Tuple[WebDriver, str]: options.add_argument( "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36" ) + options.add_argument( + '--no-sandbox' + ) + options.add_argument( + '--headless' + ) if CFG.selenium_web_browser == "firefox": driver = webdriver.Firefox( @@ -74,10 +80,6 @@ def scrape_text_with_selenium(url: str) -> Tuple[WebDriver, str]: # See https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari driver = webdriver.Safari(options=options) else: - options.add_argument('--no-sandbox') - options.add_argument('--window-size=1420,1080') - options.add_argument('--headless') - options.add_argument('--disable-gpu') driver = webdriver.Chrome( executable_path=ChromeDriverManager().install(), options=options ) From 4fa97e92189b85e5ee15ec80bdcbedb35d07e0be Mon Sep 17 00:00:00 2001 From: Peter Svensson Date: Sun, 16 Apr 2023 17:59:47 +0200 Subject: [PATCH 42/48] remvoed options so that @pi can merge this and another commit easily --- autogpt/commands/web_selenium.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 021d149c..9b64ac2e 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -64,13 +64,7 @@ def scrape_text_with_selenium(url: str) -> Tuple[WebDriver, str]: options.add_argument( "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36" ) - options.add_argument( - '--no-sandbox' - ) - options.add_argument( - '--headless' - ) - + if CFG.selenium_web_browser == "firefox": driver = webdriver.Firefox( executable_path=GeckoDriverManager().install(), options=options From 5634eee2cfb1fc8b139d79a6995134c5d9d6fe95 Mon Sep 17 00:00:00 2001 From: Peter Svensson Date: Sun, 16 Apr 2023 19:33:27 +0200 Subject: [PATCH 43/48] removed erroneous whitespace to appease lint --- autogpt/commands/web_selenium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 9b64ac2e..359803ee 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -64,7 +64,7 @@ def scrape_text_with_selenium(url: str) -> Tuple[WebDriver, str]: options.add_argument( "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36" ) - + if CFG.selenium_web_browser == "firefox": driver = webdriver.Firefox( executable_path=GeckoDriverManager().install(), options=options From f02b6832e234f7288c57665ba8e5f958b51783af Mon Sep 17 00:00:00 2001 From: SBNovaScript Date: Sat, 15 Apr 2023 16:03:22 -0400 Subject: [PATCH 44/48] Fix google result encoding. --- autogpt/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autogpt/app.py b/autogpt/app.py index eb7cdbea..11ab9724 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -128,8 +128,8 @@ def execute_command(command_name: str, arguments): return google_result else: google_result = google_search(arguments["input"]) - safe_message = google_result.encode("utf-8", "ignore") - return str(safe_message) + safe_message = [google_result_single.encode('utf-8', 'ignore') for google_result_single in google_result] + return str(safe_message) elif command_name == "memory_add": return memory.add(arguments["string"]) elif command_name == "start_agent": From 13602b4a63b1b4632ae58dfa4e83217e90cb21ce Mon Sep 17 00:00:00 2001 From: SBNovaScript Date: Sat, 15 Apr 2023 16:39:26 -0400 Subject: [PATCH 45/48] Add list type check --- autogpt/app.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/autogpt/app.py b/autogpt/app.py index 11ab9724..6ead0d52 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -128,7 +128,13 @@ def execute_command(command_name: str, arguments): return google_result else: google_result = google_search(arguments["input"]) - safe_message = [google_result_single.encode('utf-8', 'ignore') for google_result_single in google_result] + + # google_result can be a list or a string depending on the search results + if isinstance(google_result, list): + safe_message = [google_result_single.encode('utf-8', 'ignore') for google_result_single in google_result] + else: + safe_message = google_result.encode('utf-8', 'ignore') + return str(safe_message) elif command_name == "memory_add": return memory.add(arguments["string"]) From 89909115226dfa2c26799dbbd684428dc12198f6 Mon Sep 17 00:00:00 2001 From: jayceslesar Date: Sun, 16 Apr 2023 14:02:48 -0400 Subject: [PATCH 46/48] unify annotations to future syntax --- autogpt/agent/agent_manager.py | 9 +++++---- autogpt/commands/evaluate_code.py | 4 ++-- autogpt/commands/file_operations.py | 8 +++++--- autogpt/commands/google_search.py | 5 +++-- autogpt/commands/improve_code.py | 5 +++-- autogpt/commands/web_playwright.py | 5 +++-- autogpt/commands/web_requests.py | 11 ++++++----- autogpt/commands/web_selenium.py | 9 +++++---- autogpt/commands/write_tests.py | 7 ++++--- autogpt/config/ai_config.py | 6 ++++-- autogpt/json_fixes/bracket_termination.py | 5 +++-- autogpt/json_fixes/parsing.py | 11 ++++++----- autogpt/llm_utils.py | 13 +++++++------ autogpt/memory/local.py | 10 ++++++---- autogpt/memory/no_memory.py | 8 +++++--- autogpt/memory/redismem.py | 8 +++++--- autogpt/processing/html.py | 7 ++++--- autogpt/promptgenerator.py | 8 +++++--- autogpt/token_counter.py | 4 ++-- 19 files changed, 83 insertions(+), 60 deletions(-) diff --git a/autogpt/agent/agent_manager.py b/autogpt/agent/agent_manager.py index 3467f8bf..e4bfb126 100644 --- a/autogpt/agent/agent_manager.py +++ b/autogpt/agent/agent_manager.py @@ -1,5 +1,6 @@ """Agent manager for managing GPT agents""" -from typing import List, Tuple, Union +from __future__ import annotations + from autogpt.llm_utils import create_chat_completion from autogpt.config.config import Singleton @@ -14,7 +15,7 @@ class AgentManager(metaclass=Singleton): # Create new GPT agent # TODO: Centralise use of create_chat_completion() to globally enforce token limit - def create_agent(self, task: str, prompt: str, model: str) -> Tuple[int, str]: + def create_agent(self, task: str, prompt: str, model: str) -> tuple[int, str]: """Create a new agent and return its key Args: @@ -47,7 +48,7 @@ class AgentManager(metaclass=Singleton): return key, agent_reply - def message_agent(self, key: Union[str, int], message: str) -> str: + def message_agent(self, key: str | int, message: str) -> str: """Send a message to an agent and return its response Args: @@ -73,7 +74,7 @@ class AgentManager(metaclass=Singleton): return agent_reply - def list_agents(self) -> List[Tuple[Union[str, int], str]]: + def list_agents(self) -> list[tuple[str | int, str]]: """Return a list of all agents Returns: diff --git a/autogpt/commands/evaluate_code.py b/autogpt/commands/evaluate_code.py index a36952e5..8f7cbca9 100644 --- a/autogpt/commands/evaluate_code.py +++ b/autogpt/commands/evaluate_code.py @@ -1,10 +1,10 @@ """Code evaluation module.""" -from typing import List +from __future__ import annotations from autogpt.llm_utils import call_ai_function -def evaluate_code(code: str) -> List[str]: +def evaluate_code(code: str) -> list[str]: """ A function that takes in a string and returns a response from create chat completion api call. diff --git a/autogpt/commands/file_operations.py b/autogpt/commands/file_operations.py index 31500e8e..2911d601 100644 --- a/autogpt/commands/file_operations.py +++ b/autogpt/commands/file_operations.py @@ -1,8 +1,10 @@ """File operations for AutoGPT""" +from __future__ import annotations + import os import os.path from pathlib import Path -from typing import Generator, List +from typing import Generator # Set a dedicated folder for file I/O WORKING_DIRECTORY = Path(os.getcwd()) / "auto_gpt_workspace" @@ -214,14 +216,14 @@ def delete_file(filename: str) -> str: return f"Error: {str(e)}" -def search_files(directory: str) -> List[str]: +def search_files(directory: str) -> list[str]: """Search for files in a directory Args: directory (str): The directory to search in Returns: - List[str]: A list of files found in the directory + list[str]: A list of files found in the directory """ found_files = [] diff --git a/autogpt/commands/google_search.py b/autogpt/commands/google_search.py index 6deb9b50..148ba1d0 100644 --- a/autogpt/commands/google_search.py +++ b/autogpt/commands/google_search.py @@ -1,6 +1,7 @@ """Google search command for Autogpt.""" +from __future__ import annotations + import json -from typing import List, Union from duckduckgo_search import ddg @@ -33,7 +34,7 @@ def google_search(query: str, num_results: int = 8) -> str: return json.dumps(search_results, ensure_ascii=False, indent=4) -def google_official_search(query: str, num_results: int = 8) -> Union[str, List[str]]: +def google_official_search(query: str, num_results: int = 8) -> str | list[str]: """Return the results of a google search using the official Google API Args: diff --git a/autogpt/commands/improve_code.py b/autogpt/commands/improve_code.py index 05fe89e9..e3440d8b 100644 --- a/autogpt/commands/improve_code.py +++ b/autogpt/commands/improve_code.py @@ -1,10 +1,11 @@ +from __future__ import annotations + import json -from typing import List from autogpt.llm_utils import call_ai_function -def improve_code(suggestions: List[str], code: str) -> str: +def improve_code(suggestions: list[str], code: str) -> str: """ A function that takes in code and suggestions and returns a response from create chat completion api call. diff --git a/autogpt/commands/web_playwright.py b/autogpt/commands/web_playwright.py index 93a46ac9..a1abb6cb 100644 --- a/autogpt/commands/web_playwright.py +++ b/autogpt/commands/web_playwright.py @@ -1,4 +1,6 @@ """Web scraping commands using Playwright""" +from __future__ import annotations + try: from playwright.sync_api import sync_playwright except ImportError: @@ -7,7 +9,6 @@ except ImportError: ) from bs4 import BeautifulSoup from autogpt.processing.html import extract_hyperlinks, format_hyperlinks -from typing import List, Union def scrape_text(url: str) -> str: @@ -45,7 +46,7 @@ def scrape_text(url: str) -> str: return text -def scrape_links(url: str) -> Union[str, List[str]]: +def scrape_links(url: str) -> str | list[str]: """Scrape links from a webpage Args: diff --git a/autogpt/commands/web_requests.py b/autogpt/commands/web_requests.py index a6161ec5..50d8d383 100644 --- a/autogpt/commands/web_requests.py +++ b/autogpt/commands/web_requests.py @@ -1,5 +1,6 @@ """Browse a webpage and summarize it using the LLM model""" -from typing import List, Tuple, Union +from __future__ import annotations + from urllib.parse import urljoin, urlparse import requests @@ -66,7 +67,7 @@ def check_local_file_access(url: str) -> bool: def get_response( url: str, timeout: int = 10 -) -> Union[Tuple[None, str], Tuple[Response, None]]: +) -> tuple[None, str] | tuple[Response, None]: """Get the response from a URL Args: @@ -74,7 +75,7 @@ def get_response( timeout (int): The timeout for the HTTP request Returns: - Tuple[None, str] | Tuple[Response, None]: The response and error message + tuple[None, str] | tuple[Response, None]: The response and error message Raises: ValueError: If the URL is invalid @@ -136,14 +137,14 @@ def scrape_text(url: str) -> str: return text -def scrape_links(url: str) -> Union[str, List[str]]: +def scrape_links(url: str) -> str | list[str]: """Scrape links from a webpage Args: url (str): The URL to scrape links from Returns: - Union[str, List[str]]: The scraped links + str | list[str]: The scraped links """ response, error_message = get_response(url) if error_message: diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 359803ee..1d078d76 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -1,4 +1,6 @@ """Selenium web scraping module.""" +from __future__ import annotations + from selenium import webdriver from autogpt.processing.html import extract_hyperlinks, format_hyperlinks import autogpt.processing.text as summary @@ -15,13 +17,12 @@ from selenium.webdriver.safari.options import Options as SafariOptions import logging from pathlib import Path from autogpt.config import Config -from typing import List, Tuple, Union FILE_DIR = Path(__file__).parent.parent CFG = Config() -def browse_website(url: str, question: str) -> Tuple[str, WebDriver]: +def browse_website(url: str, question: str) -> tuple[str, WebDriver]: """Browse a website and return the answer and links to the user Args: @@ -43,7 +44,7 @@ def browse_website(url: str, question: str) -> Tuple[str, WebDriver]: return f"Answer gathered from website: {summary_text} \n \n Links: {links}", driver -def scrape_text_with_selenium(url: str) -> Tuple[WebDriver, str]: +def scrape_text_with_selenium(url: str) -> tuple[WebDriver, str]: """Scrape text from a website using selenium Args: @@ -97,7 +98,7 @@ def scrape_text_with_selenium(url: str) -> Tuple[WebDriver, str]: return driver, text -def scrape_links_with_selenium(driver: WebDriver, url: str) -> List[str]: +def scrape_links_with_selenium(driver: WebDriver, url: str) -> list[str]: """Scrape links from a website using selenium Args: diff --git a/autogpt/commands/write_tests.py b/autogpt/commands/write_tests.py index f1d6c9b2..138a1adb 100644 --- a/autogpt/commands/write_tests.py +++ b/autogpt/commands/write_tests.py @@ -1,16 +1,17 @@ """A module that contains a function to generate test cases for the submitted code.""" +from __future__ import annotations + import json -from typing import List from autogpt.llm_utils import call_ai_function -def write_tests(code: str, focus: List[str]) -> str: +def write_tests(code: str, focus: list[str]) -> str: """ A function that takes in code and focus topics and returns a response from create chat completion api call. Parameters: - focus (List): A list of suggestions around what needs to be improved. + focus (list): A list of suggestions around what needs to be improved. code (str): Code for test cases to be generated against. Returns: A result string from create chat completion. Test cases for the submitted code diff --git a/autogpt/config/ai_config.py b/autogpt/config/ai_config.py index 014e360f..86171357 100644 --- a/autogpt/config/ai_config.py +++ b/autogpt/config/ai_config.py @@ -2,8 +2,10 @@ """ A module that contains the AIConfig class object that contains the configuration """ +from __future__ import annotations + import os -from typing import List, Optional, Type +from typing import Type import yaml @@ -18,7 +20,7 @@ class AIConfig: """ def __init__( - self, ai_name: str = "", ai_role: str = "", ai_goals: Optional[List] = None + self, ai_name: str = "", ai_role: str = "", ai_goals: list | None = None ) -> None: """ Initialize a class instance diff --git a/autogpt/json_fixes/bracket_termination.py b/autogpt/json_fixes/bracket_termination.py index 692461aa..822eed4a 100644 --- a/autogpt/json_fixes/bracket_termination.py +++ b/autogpt/json_fixes/bracket_termination.py @@ -1,7 +1,8 @@ """Fix JSON brackets.""" +from __future__ import annotations + import contextlib import json -from typing import Optional import regex from colorama import Fore @@ -46,7 +47,7 @@ def attempt_to_fix_json_by_finding_outermost_brackets(json_string: str): return json_string -def balance_braces(json_string: str) -> Optional[str]: +def balance_braces(json_string: str) -> str | None: """ Balance the braces in a JSON string. diff --git a/autogpt/json_fixes/parsing.py b/autogpt/json_fixes/parsing.py index 26d06793..0f154411 100644 --- a/autogpt/json_fixes/parsing.py +++ b/autogpt/json_fixes/parsing.py @@ -1,8 +1,9 @@ """Fix and parse JSON strings.""" +from __future__ import annotations import contextlib import json -from typing import Any, Dict, Union +from typing import Any from autogpt.config import Config from autogpt.json_fixes.auto_fix import fix_json @@ -71,7 +72,7 @@ def correct_json(json_to_load: str) -> str: def fix_and_parse_json( json_to_load: str, try_to_fix_with_gpt: bool = True -) -> Union[str, Dict[Any, Any]]: +) -> str | dict[Any, Any]: """Fix and parse JSON string Args: @@ -80,7 +81,7 @@ def fix_and_parse_json( Defaults to True. Returns: - Union[str, Dict[Any, Any]]: The parsed JSON. + str or dict[Any, Any]: The parsed JSON. """ with contextlib.suppress(json.JSONDecodeError): @@ -109,7 +110,7 @@ def fix_and_parse_json( def try_ai_fix( try_to_fix_with_gpt: bool, exception: Exception, json_to_load: str -) -> Union[str, Dict[Any, Any]]: +) -> str | dict[Any, Any]: """Try to fix the JSON with the AI Args: @@ -121,7 +122,7 @@ def try_ai_fix( exception: If try_to_fix_with_gpt is False. Returns: - Union[str, Dict[Any, Any]]: The JSON string or dictionary. + str or dict[Any, Any]: The JSON string or dictionary. """ if not try_to_fix_with_gpt: raise exception diff --git a/autogpt/llm_utils.py b/autogpt/llm_utils.py index 43739009..2075f934 100644 --- a/autogpt/llm_utils.py +++ b/autogpt/llm_utils.py @@ -1,6 +1,7 @@ +from __future__ import annotations + from ast import List import time -from typing import Dict, Optional import openai from openai.error import APIError, RateLimitError @@ -14,7 +15,7 @@ openai.api_key = CFG.openai_api_key def call_ai_function( - function: str, args: List, description: str, model: Optional[str] = None + function: str, args: list, description: str, model: str | None = None ) -> str: """Call an AI function @@ -51,15 +52,15 @@ def call_ai_function( # Overly simple abstraction until we create something better # simple retry mechanism when getting a rate error or a bad gateway def create_chat_completion( - messages: List, # type: ignore - model: Optional[str] = None, + messages: list, # type: ignore + model: str | None = None, temperature: float = CFG.temperature, - max_tokens: Optional[int] = None, + max_tokens: int | None = None, ) -> str: """Create a chat completion using the OpenAI API Args: - messages (List[Dict[str, str]]): The messages to send to the chat completion + messages (list[dict[str, str]]): The messages to send to the chat completion model (str, optional): The model to use. Defaults to None. temperature (float, optional): The temperature to use. Defaults to 0.9. max_tokens (int, optional): The max tokens to use. Defaults to None. diff --git a/autogpt/memory/local.py b/autogpt/memory/local.py index 004153c1..6c7ee1b3 100644 --- a/autogpt/memory/local.py +++ b/autogpt/memory/local.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import dataclasses import os -from typing import Any, List, Optional, Tuple +from typing import Any import numpy as np import orjson @@ -97,7 +99,7 @@ class LocalCache(MemoryProviderSingleton): self.data = CacheContent() return "Obliviated" - def get(self, data: str) -> Optional[List[Any]]: + def get(self, data: str) -> list[Any] | None: """ Gets the data from the memory that is most relevant to the given data. @@ -108,7 +110,7 @@ class LocalCache(MemoryProviderSingleton): """ return self.get_relevant(data, 1) - def get_relevant(self, text: str, k: int) -> List[Any]: + def get_relevant(self, text: str, k: int) -> list[Any]: """ " matrix-vector mult to find score-for-each-row-of-matrix get indices for top-k winning scores @@ -127,7 +129,7 @@ class LocalCache(MemoryProviderSingleton): return [self.data.texts[i] for i in top_k_indices] - def get_stats(self) -> Tuple[int, Tuple[int, ...]]: + def get_stats(self) -> tuple[int, tuple[int, ...]]: """ Returns: The stats of the local cache. """ diff --git a/autogpt/memory/no_memory.py b/autogpt/memory/no_memory.py index 0a976690..4035a657 100644 --- a/autogpt/memory/no_memory.py +++ b/autogpt/memory/no_memory.py @@ -1,5 +1,7 @@ """A class that does not store any data. This is the default memory provider.""" -from typing import Optional, List, Any +from __future__ import annotations + +from typing import Any from autogpt.memory.base import MemoryProviderSingleton @@ -31,7 +33,7 @@ class NoMemory(MemoryProviderSingleton): """ return "" - def get(self, data: str) -> Optional[List[Any]]: + def get(self, data: str) -> list[Any] | None: """ Gets the data from the memory that is most relevant to the given data. NoMemory always returns None. @@ -51,7 +53,7 @@ class NoMemory(MemoryProviderSingleton): """ return "" - def get_relevant(self, data: str, num_relevant: int = 5) -> Optional[List[Any]]: + def get_relevant(self, data: str, num_relevant: int = 5) ->list[Any] | None: """ Returns all the data in the memory that is relevant to the given data. NoMemory always returns None. diff --git a/autogpt/memory/redismem.py b/autogpt/memory/redismem.py index 4d73b741..0e8dd71d 100644 --- a/autogpt/memory/redismem.py +++ b/autogpt/memory/redismem.py @@ -1,5 +1,7 @@ """Redis memory provider.""" -from typing import Any, List, Optional +from __future__ import annotations + +from typing import Any import numpy as np import redis @@ -99,7 +101,7 @@ class RedisMemory(MemoryProviderSingleton): pipe.execute() return _text - def get(self, data: str) -> Optional[List[Any]]: + def get(self, data: str) -> list[Any] | None: """ Gets the data from the memory that is most relevant to the given data. @@ -119,7 +121,7 @@ class RedisMemory(MemoryProviderSingleton): self.redis.flushall() return "Obliviated" - def get_relevant(self, data: str, num_relevant: int = 5) -> Optional[List[Any]]: + def get_relevant(self, data: str, num_relevant: int = 5) -> list[Any] | None: """ Returns all the data in the memory that is relevant to the given data. Args: diff --git a/autogpt/processing/html.py b/autogpt/processing/html.py index c43a0b74..e1912b6a 100644 --- a/autogpt/processing/html.py +++ b/autogpt/processing/html.py @@ -1,10 +1,11 @@ """HTML processing functions""" +from __future__ import annotations + from requests.compat import urljoin -from typing import List, Tuple from bs4 import BeautifulSoup -def extract_hyperlinks(soup: BeautifulSoup, base_url: str) -> List[Tuple[str, str]]: +def extract_hyperlinks(soup: BeautifulSoup, base_url: str) -> list[tuple[str, str]]: """Extract hyperlinks from a BeautifulSoup object Args: @@ -20,7 +21,7 @@ def extract_hyperlinks(soup: BeautifulSoup, base_url: str) -> List[Tuple[str, st ] -def format_hyperlinks(hyperlinks: List[Tuple[str, str]]) -> List[str]: +def format_hyperlinks(hyperlinks: list[tuple[str, str]]) -> list[str]: """Format hyperlinks to be displayed to the user Args: diff --git a/autogpt/promptgenerator.py b/autogpt/promptgenerator.py index 4f518615..0ad7046a 100644 --- a/autogpt/promptgenerator.py +++ b/autogpt/promptgenerator.py @@ -1,6 +1,8 @@ """ A module for generating custom prompt strings.""" +from __future__ import annotations + import json -from typing import Any, Dict, List +from typing import Any class PromptGenerator: @@ -61,7 +63,7 @@ class PromptGenerator: self.commands.append(command) - def _generate_command_string(self, command: Dict[str, Any]) -> str: + def _generate_command_string(self, command: dict[str, Any]) -> str: """ Generate a formatted string representation of a command. @@ -94,7 +96,7 @@ class PromptGenerator: """ self.performance_evaluation.append(evaluation) - def _generate_numbered_list(self, items: List[Any], item_type="list") -> str: + def _generate_numbered_list(self, items: list[Any], item_type="list") -> str: """ Generate a numbered list from given items based on the item_type. diff --git a/autogpt/token_counter.py b/autogpt/token_counter.py index a85a54be..338fe6be 100644 --- a/autogpt/token_counter.py +++ b/autogpt/token_counter.py @@ -1,5 +1,5 @@ """Functions for counting the number of tokens in a message or string.""" -from typing import Dict, List +from __future__ import annotations import tiktoken @@ -7,7 +7,7 @@ from autogpt.logs import logger def count_message_tokens( - messages: List[Dict[str, str]], model: str = "gpt-3.5-turbo-0301" + messages: list[dict[str, str]], model: str = "gpt-3.5-turbo-0301" ) -> int: """ Returns the number of tokens used by a list of messages. From 1df47bb0be87bbda9b794226ceb4a2eef47ad45b Mon Sep 17 00:00:00 2001 From: BillSchumacher <34168009+BillSchumacher@users.noreply.github.com> Date: Sun, 16 Apr 2023 13:08:16 -0500 Subject: [PATCH 47/48] Add in one more place. --- autogpt/workspace.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/autogpt/workspace.py b/autogpt/workspace.py index 79134919..2706b3b2 100644 --- a/autogpt/workspace.py +++ b/autogpt/workspace.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os from pathlib import Path From a91ef5695403066d5a9435ba0cee0f6186836c10 Mon Sep 17 00:00:00 2001 From: Richard Beales Date: Sun, 16 Apr 2023 19:08:10 +0100 Subject: [PATCH 48/48] Remove warnings if memory backend is not installed --- autogpt/memory/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autogpt/memory/__init__.py b/autogpt/memory/__init__.py index 817155dc..e2ee44a4 100644 --- a/autogpt/memory/__init__.py +++ b/autogpt/memory/__init__.py @@ -10,7 +10,7 @@ try: supported_memory.append("redis") except ImportError: - print("Redis not installed. Skipping import.") + # print("Redis not installed. Skipping import.") RedisMemory = None try: @@ -18,19 +18,19 @@ try: supported_memory.append("pinecone") except ImportError: - print("Pinecone not installed. Skipping import.") + # print("Pinecone not installed. Skipping import.") PineconeMemory = None try: from autogpt.memory.weaviate import WeaviateMemory except ImportError: - print("Weaviate not installed. Skipping import.") + # print("Weaviate not installed. Skipping import.") WeaviateMemory = None try: from autogpt.memory.milvus import MilvusMemory except ImportError: - print("pymilvus not installed. Skipping import.") + # print("pymilvus not installed. Skipping import.") MilvusMemory = None