From 0dddc94bdac94707062b1863f3c5a72d113432ca Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 19:13:04 +0200 Subject: [PATCH 01/13] Add file ingestion methode in file_operations.py Add the following functions to ingest data into memory before Auto-GPT run. - split_file: given a content, split it in chunks of max_length with (or without) a specified overlap - ingest_file: read a file, use split_file to split it in chunks and load each chunk in memory - ingest_directory: ingest all files in a directory in memory --- scripts/file_operations.py | 75 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index c6066ef9..3493c2bf 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -1,5 +1,10 @@ import os import os.path +from config import Config +from memory import get_memory + +cfg = Config() +memory = get_memory(cfg) # Set a dedicated folder for file I/O working_directory = "auto_gpt_workspace" @@ -20,6 +25,30 @@ def safe_join(base, *paths): return norm_new_path +def split_file(content, max_length=4000, overlap=0): + """ + Split text into chunks of a specified maximum length with a specified overlap + between chunks. + + :param text: The input text to be split into chunks + :param max_length: The maximum length of each chunk, default is 4000 (about 1k token) + :param overlap: The number of overlapping characters between chunks, default is no overlap + :return: A generator yielding chunks of text + """ + start = 0 + content_length = len(content) + + while start < content_length: + end = start + max_length + chunk = content[start:end] + yield chunk + start += max_length - overlap + if start + max_length - overlap >= content_length: + break + if end + overlap > content_length: + start = content_length - max_length + + def read_file(filename): """Read a file and return the contents""" try: @@ -31,6 +60,52 @@ def read_file(filename): return "Error: " + str(e) +def ingest_file(filename, memory, max_length=4000, overlap=200): + """ + Ingest a file by reading its content, splitting it into chunks with a specified + maximum length and overlap, and adding the chunks to the memory storage. + + :param filename: The name of the file to ingest + :param memory: An object with an add() method to store the chunks in memory + :param max_length: The maximum length of each chunk, default is 4000 + :param overlap: The number of overlapping characters between chunks, default is 200 + """ + try: + print(f"Working with file {filename}") + content = read_file(filename) + content_length = len(content) + print(f"File length: {content_length} characters") + + chunks = list(split_file(content, max_length=max_length, overlap=overlap)) + + num_chunks = len(chunks) + for i, chunk in enumerate(chunks): + print(f"Ingesting chunk {i + 1} / {num_chunks} into memory") + memory_to_add = f"Filename: {filename}\n" \ + f"Content part#{i + 1}/{num_chunks}: {chunk}" + + memory.add(memory_to_add) + + print(f"Done ingesting {num_chunks} chunks from {filename}.") + except Exception as e: + print(f"Error while ingesting file '{filename}': {str(e)}") + + +def ingest_directory(directory, memory): + """ + Ingest all files in a directory by calling the ingest_file function for each file. + + :param directory: The directory containing the files to ingest + :param memory: An object with an add() method to store the chunks in memory + """ + try: + files = search_files(directory) + for file in files: + ingest_file(file, memory) + except Exception as e: + print(f"Error while ingesting directory '{directory}': {str(e)}") + + def write_to_file(filename, text): """Write text to a file""" try: From 7975c184a55a477e884e1920ed87dc67ca4b4261 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 19:46:39 +0200 Subject: [PATCH 02/13] Update .gitignore add new log file to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index aa0dceaa..fc496609 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ log.txt # Coverage reports .coverage coverage.xml -htmlcov/ \ No newline at end of file +htmlcov/ +log-ingestion.txt From c91117616f7b5e16743208b8649ce4335077915b Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 19:46:58 +0200 Subject: [PATCH 03/13] Update file_operations.py revert change in import as we don't need them --- scripts/file_operations.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index 3493c2bf..8e807bba 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -1,10 +1,5 @@ import os import os.path -from config import Config -from memory import get_memory - -cfg = Config() -memory = get_memory(cfg) # Set a dedicated folder for file I/O working_directory = "auto_gpt_workspace" From 8faa6ef949bf7fbbb8bd875a66bfd4fd231ecebc Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 19:47:51 +0200 Subject: [PATCH 04/13] Create data_ingestion.py This script is use when we want to seed Auto-GPT memory with one or multiple documents. The document are read, split into chunks and store in the memory. --- scripts/data_ingestion.py | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 scripts/data_ingestion.py diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py new file mode 100644 index 00000000..3f6d1322 --- /dev/null +++ b/scripts/data_ingestion.py @@ -0,0 +1,52 @@ +import argparse +import logging +from config import Config +from memory import get_memory +from file_operations import ingest_file, ingest_directory + +cfg = Config() + +def configure_logging(): + logging.basicConfig(filename='log-ingestion.txt', + filemode='a', + format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', + datefmt='%H:%M:%S', + level=logging.DEBUG) + return logging.getLogger('AutoGPT-Ingestion') + + +def main(): + logger = configure_logging() + + parser = argparse.ArgumentParser(description="Ingest a file or a directory with multiple files into memory. Make sure to set your .env before running this script.") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--file", type=str, help="The file to ingest.") + group.add_argument("--dir", type=str, help="The directory containing the files to ingest.") + parser.add_argument("--init", action='store_true', help="Init the memory and wipe its content", default=False) + args = parser.parse_args() + + + # Initialize memory + memory = get_memory(cfg, init=args.init) + print('Using memory of type: ' + memory.__class__.__name__) + + if args.file: + try: + ingest_file(args.file, memory) + print(f"File '{args.file}' ingested successfully.") + except Exception as e: + logger.error(f"Error while ingesting file '{args.file}': {str(e)}") + print(f"Error while ingesting file '{args.file}': {str(e)}") + elif args.dir: + try: + ingest_directory(args.dir, memory) + print(f"Directory '{args.dir}' ingested successfully.") + except Exception as e: + logger.error(f"Error while ingesting directory '{args.dir}': {str(e)}") + print(f"Error while ingesting directory '{args.dir}': {str(e)}") + else: + print("Please provide either a file path (--file) or a directory name (--dir) inside the auto_gpt_workspace directory as input.") + + +if __name__ == "__main__": + main() From 4465486ea39b0bc65715e48a1c7861a565b5126f Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 20:18:59 +0200 Subject: [PATCH 05/13] Update file_operations.py move the search_file function inside the data_ingestion script --- scripts/file_operations.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index 8e807bba..e664fcc9 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -86,21 +86,6 @@ def ingest_file(filename, memory, max_length=4000, overlap=200): print(f"Error while ingesting file '{filename}': {str(e)}") -def ingest_directory(directory, memory): - """ - Ingest all files in a directory by calling the ingest_file function for each file. - - :param directory: The directory containing the files to ingest - :param memory: An object with an add() method to store the chunks in memory - """ - try: - files = search_files(directory) - for file in files: - ingest_file(file, memory) - except Exception as e: - print(f"Error while ingesting directory '{directory}': {str(e)}") - - def write_to_file(filename, text): """Write text to a file""" try: From 280647ff387bc29127b8403c7dd46f2c94d6a0b8 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 20:19:36 +0200 Subject: [PATCH 06/13] Update data_ingestion.py move the search_file function inside the data_ingestion script add memory initialisation argument add overlap argument add chunk max_length argument --- scripts/data_ingestion.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py index 3f6d1322..32811166 100644 --- a/scripts/data_ingestion.py +++ b/scripts/data_ingestion.py @@ -2,7 +2,7 @@ import argparse import logging from config import Config from memory import get_memory -from file_operations import ingest_file, ingest_directory +from file_operations import ingest_file, search_files cfg = Config() @@ -15,6 +15,21 @@ def configure_logging(): return logging.getLogger('AutoGPT-Ingestion') +def ingest_directory(directory, memory, args): + """ + Ingest all files in a directory by calling the ingest_file function for each file. + + :param directory: The directory containing the files to ingest + :param memory: An object with an add() method to store the chunks in memory + """ + try: + files = search_files(directory) + for file in files: + ingest_file(file, memory, args.max_length, args.overlap) + except Exception as e: + print(f"Error while ingesting directory '{directory}': {str(e)}") + + def main(): logger = configure_logging() @@ -22,7 +37,10 @@ def main(): group = parser.add_mutually_exclusive_group(required=True) group.add_argument("--file", type=str, help="The file to ingest.") group.add_argument("--dir", type=str, help="The directory containing the files to ingest.") - parser.add_argument("--init", action='store_true', help="Init the memory and wipe its content", default=False) + parser.add_argument("--init", action='store_true', help="Init the memory and wipe its content (default: False)", default=False) + parser.add_argument("--overlap", type=int, help="The overlap size between chunks when ingesting files (default: 200)", default=200) + parser.add_argument("--max_length", type=int, help="The max_length of each chunk when ingesting files (default: 4000)", default=4000) + args = parser.parse_args() @@ -32,14 +50,14 @@ def main(): if args.file: try: - ingest_file(args.file, memory) + ingest_file(args.file, memory, args.max_length, args.overlap) print(f"File '{args.file}' ingested successfully.") except Exception as e: logger.error(f"Error while ingesting file '{args.file}': {str(e)}") print(f"Error while ingesting file '{args.file}': {str(e)}") elif args.dir: try: - ingest_directory(args.dir, memory) + ingest_directory(args.dir, memory, args) print(f"Directory '{args.dir}' ingested successfully.") except Exception as e: logger.error(f"Error while ingesting directory '{args.dir}': {str(e)}") From 65cc4f833f56000ae3928cccc3c9821fece53958 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 20:47:46 +0200 Subject: [PATCH 07/13] Add Memory Pre-Seeding information to readme.md Add the documentation for memory pre-seeding --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/README.md b/README.md index 2900daa9..6262467d 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Your support is greatly appreciated - [๐Ÿ—ฃ๏ธ Speech Mode](#๏ธ-speech-mode) - [๐Ÿ” Google API Keys Configuration](#-google-api-keys-configuration) - [Setting up environment variables](#setting-up-environment-variables) + - [๐Ÿง  Memory pre-seeding](#memory-pre-seeding) - [๐Ÿ’€ Continuous Mode โš ๏ธ](#-continuous-mode-๏ธ) - [GPT3.5 ONLY Mode](#gpt35-only-mode) - [๐Ÿ–ผ Image Generation](#image-generation) @@ -245,6 +246,52 @@ To switch to either, change the `MEMORY_BACKEND` env variable to the value that 1. View memory usage by using the `--debug` flag :) + +## ๐Ÿง  Memory pre-seeding + +``` +# python scripts/data_ingestion.py -h +usage: data_ingestion.py [-h] (--file FILE | --dir DIR) [--init] [--overlap OVERLAP] [--max_length MAX_LENGTH] + +Ingest a file or a directory with multiple files into memory. Make sure to set your .env before running this script. + +options: + -h, --help show this help message and exit + --file FILE The file to ingest. + --dir DIR The directory containing the files to ingest. + --init Init the memory and wipe its content (default: False) + --overlap OVERLAP The overlap size between chunks when ingesting files (default: 200) + --max_length MAX_LENGTH The max_length of each chunk when ingesting files (default: 4000 + +# python scripts/data_ingestion.py --dir seed_data --init --overlap 200 --max_length 1000 +``` + +This script located at scripts/data_ingestion.py, allows you to ingest files into memory and pre-seed it before running Auto-GPT. + +Memory pre-seeding is a technique that involves ingesting relevant documents or data into the AI's memory so that it can use this information to generate more informed and accurate responses. + +To pre-seed the memory, the content of each document is split into chunks of a specified maximum length with a specified overlap between chunks, and then each chunk is added to the memory backend set in the .env file. When the AI is prompted to recall information, it can then access those pre-seeded memories to generate more informed and accurate responses. + +This technique is particularly useful when working with large amounts of data or when there is specific information that the AI needs to be able to access quickly. +By pre-seeding the memory, the AI can retrieve and use this information more efficiently, saving time, API call and improving the accuracy of its responses. + +You could for example download the documentation of an API, a Github repository, etc. and ingest it into memory before running Auto-GPT. + +โš ๏ธ If you use Redis as your memory, make sure to run Auto-GPT with the WIPE_REDIS_ON_START set to False in your .env file. + +โš ๏ธFor other memory backend, we currently forcefully wipe the memory when starting Auto-GPT. To ingest data with those memory backend, you can call the data_ingestion.py script anytime during an Auto-GPT run. + +Memories will be available to the AI immediately as they are ingested, even if ingested while Auto-GPT is running. + +In the example above, the script initializes the memory, ingests all files within the seed_data directory into memory with an overlap between chunks of 200 and a maximum length of each chunk of 4000. +Note that you can also use the --file argument to ingest a single file into memory and that the script will only ingest files within the auto_gpt_workspace directory. + +You can adjust the max_length and overlap parameters to fine-tune the way the docuents are presented to the AI when it "recall" that memory: + +- Adjusting the overlap value allows the AI to access more contextual information from each chunk when recalling information, but will result in more chunks being created and therefore increase memory backend usage and OpenAI API requests. +- Reducing the max_length value will create more chunks, which can save prompt tokens by allowing for more message history in the context, but will also increase the number of chunks. +- Increasing the max_length value will provide the AI with more contextual information from each chunk, reducing the number of chunks created and saving on OpenAI API requests. However, this may also use more prompt tokens and decrease the overall context available to the AI. + ## ๐Ÿ’€ Continuous Mode โš ๏ธ Run the AI **without** user authorisation, 100% automated. From 2e0b44ae05fce7795f662a81c765eeeeae32a768 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 22:46:49 +0200 Subject: [PATCH 08/13] fix chunk creation the last chunk wasn't correctly created, this commit fix that issue. --- scripts/file_operations.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index e664fcc9..f2a2b072 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -38,11 +38,12 @@ def split_file(content, max_length=4000, overlap=0): chunk = content[start:end] yield chunk start += max_length - overlap - if start + max_length - overlap >= content_length: - break - if end + overlap > content_length: + if start + max_length > content_length: start = content_length - max_length - + end = content_length + chunk = content[start:end] + yield chunk + break def read_file(filename): """Read a file and return the contents""" From 4e914e5ec1a4f7d39cba04cc2ebc0ba7f0055423 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 22:51:52 +0200 Subject: [PATCH 09/13] Revert "Update .gitignore" This reverts commit 7975c184a55a477e884e1920ed87dc67ca4b4261. --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index fc496609..aa0dceaa 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,4 @@ log.txt # Coverage reports .coverage coverage.xml -htmlcov/ -log-ingestion.txt +htmlcov/ \ No newline at end of file From 2f1181f9a12bbbbf55b8f2224ecc645d22c5d90d Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 22:52:37 +0200 Subject: [PATCH 10/13] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cfa3b08b..403417eb 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ ai_settings.yaml .idea/* auto-gpt.json log.txt +log-ingestion.txt # Coverage reports .coverage From 36d455c20e52aa1e09766a01c880f7914c5c24b7 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Wed, 12 Apr 2023 23:31:26 +0200 Subject: [PATCH 11/13] split_file() rework rework the split_file function to make it simple and only have one yield while providing an overlap at the start and end of each chunk --- scripts/file_operations.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index db4702c5..c12774b9 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -35,15 +35,12 @@ def split_file(content, max_length=4000, overlap=0): while start < content_length: end = start + max_length - chunk = content[start:end] + if end + overlap < content_length: + chunk = content[start:end+overlap] + else: + chunk = content[start:content_length] yield chunk start += max_length - overlap - if start + max_length > content_length: - start = content_length - max_length - end = content_length - chunk = content[start:end] - yield chunk - break def read_file(filename): """Read a file and return the contents""" From 6403bf112790b34fa122bdd519703e4b110f6875 Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Fri, 14 Apr 2023 10:35:30 +0200 Subject: [PATCH 12/13] Update data_ingestion.py fixed linting --- scripts/data_ingestion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py index 32811166..9addc34b 100644 --- a/scripts/data_ingestion.py +++ b/scripts/data_ingestion.py @@ -6,6 +6,7 @@ from file_operations import ingest_file, search_files cfg = Config() + def configure_logging(): logging.basicConfig(filename='log-ingestion.txt', filemode='a', @@ -43,7 +44,6 @@ def main(): args = parser.parse_args() - # Initialize memory memory = get_memory(cfg, init=args.init) print('Using memory of type: ' + memory.__class__.__name__) From c0462dbe7768d41ac205644987ad0fa9f14a5afc Mon Sep 17 00:00:00 2001 From: Maiko Bossuyt Date: Fri, 14 Apr 2023 10:35:52 +0200 Subject: [PATCH 13/13] Update file_operations.py fixed linting --- scripts/file_operations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/file_operations.py b/scripts/file_operations.py index ed5aa4ec..1a072561 100644 --- a/scripts/file_operations.py +++ b/scripts/file_operations.py @@ -42,6 +42,7 @@ def split_file(content, max_length=4000, overlap=0): yield chunk start += max_length - overlap + def read_file(filename): """Read a file and return the contents""" try: