mirror of
https://github.com/aljazceru/Auto-GPT.git
synced 2025-12-21 07:54:21 +01:00
split_file() rework
rework the split_file function to make it simple and only have one yield while providing an overlap at the start and end of each chunk
This commit is contained in:
@@ -35,15 +35,12 @@ def split_file(content, max_length=4000, overlap=0):
|
||||
|
||||
while start < content_length:
|
||||
end = start + max_length
|
||||
chunk = content[start:end]
|
||||
if end + overlap < content_length:
|
||||
chunk = content[start:end+overlap]
|
||||
else:
|
||||
chunk = content[start:content_length]
|
||||
yield chunk
|
||||
start += max_length - overlap
|
||||
if start + max_length > content_length:
|
||||
start = content_length - max_length
|
||||
end = content_length
|
||||
chunk = content[start:end]
|
||||
yield chunk
|
||||
break
|
||||
|
||||
def read_file(filename):
|
||||
"""Read a file and return the contents"""
|
||||
|
||||
Reference in New Issue
Block a user