aljaz/Auto-GPT
1
0
Fork 0
mirror of https://github.com/aljazceru/Auto-GPT.git synced 2025-12-20 15:34:23 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix split file

Browse Source
This commit is contained in:
Steve Byerly
2023-04-17 02:06:46 +00:00
committed by Pi
parent 6b64158356
commit def96ffe2f
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
autogpt/commands/file_operations.py
View File

@@ -49,14 +49,12 @@ def log_operation(operation: str, filename: str) -> None:
append_to_file(LOG_FILE, log_entry, shouldLog = False) append_to_file(LOG_FILE, log_entry, shouldLog = False)
def split_file( def split_file(
content: str, max_length: int = 4000, overlap: int = 0 content: str, max_length: int = 4000, overlap: int = 0
) -> Generator[str, None, None]: ) -> Generator[str, None, None]:
""" """
Split text into chunks of a specified maximum length with a specified overlap Split text into chunks of a specified maximum length with a specified overlap
between chunks. between chunks.
:param content: The input text to be split into chunks :param content: The input text to be split into chunks
:param max_length: The maximum length of each chunk, :param max_length: The maximum length of each chunk,
default is 4000 (about 1k token) default is 4000 (about 1k token)
@@ -70,9 +68,14 @@ def split_file(
while start < content_length: while start < content_length:
end = start + max_length end = start + max_length
if end + overlap < content_length: if end + overlap < content_length:
chunk = content[start : end + overlap] chunk = content[start : end + overlap - 1]
else: else:
chunk = content[start:content_length] chunk = content[start:content_length]
# Account for the case where the last chunk is shorter than the overlap, so it has already been consumed
if len(chunk) <= overlap:
break
yield chunk yield chunk
start += max_length - overlap start += max_length - overlap