Vector memory revamp (part 1: refactoring) (#4208)

Additional changes:

* Improve typing

* Modularize message history memory & fix/refactor lots of things

* Fix summarization

* Move memory relevance calculation to MemoryItem & improve test

* Fix import warnings in web_selenium.py

* Remove `memory_add` ghost command

* Implement overlap in `split_text`

* Move memory tests into subdirectory

* Remove deprecated `get_ada_embedding()` and helpers

* Fix used token calculation in `chat_with_ai`

* Replace Message TypedDict by dataclass

* Fix AgentManager singleton issues in tests

---------

Co-authored-by: Auto-GPT-Bot <github-bot@agpt.co>

autogpt/commands/file_operations_utils.py

@@ -10,6 +10,7 @@ from bs4 import BeautifulSoup
 from pylatexenc.latex2text import LatexNodes2Text
 
 from autogpt import logs
+from autogpt.logs import logger
 
 
 class ParserStrategy:
@@ -21,6 +22,7 @@ class ParserStrategy:
 class TXTParser(ParserStrategy):
     def read(self, file_path):
         charset_match = charset_normalizer.from_path(file_path).best()
+        logger.debug(f"Reading '{file_path}' with encoding '{charset_match.encoding}'")
         return str(charset_match)
 
 
@@ -150,9 +152,7 @@ def read_textual_file(file_path: str, logger: logs.Logger):
     parser = extension_to_parser.get(file_extension)
     if not parser:
         if is_binary:
-            raise ValueError(
-                "Unsupported binary file format: {}".format(file_extension)
-            )
+            raise ValueError(f"Unsupported binary file format: {file_extension}")
         # fallback to txt file parser (to support script and code files loading)
         parser = TXTParser()
     file_context = FileContext(parser, logger)