aljaz/Auto-GPT
1
0
Fork 0
mirror of https://github.com/aljazceru/Auto-GPT.git synced 2025-12-17 22:14:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into dev

Browse Source
This commit is contained in:
Andres Caicedo
2023-04-04 11:47:37 +02:00
parent 6773e28b61 8747ff64b9
commit 239accbd82
3 changed files with 27 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
scripts/browse.py
View File

@@ -85,34 +85,27 @@ def split_text(text, max_length=8192):
yield "\n".join(current_chunk)
def summarize_text(text, is_website=True):
"""Summarize text using GPT-3"""
if text == "":
def create_message(chunk, question):
"""Create a message for the user to summarize a chunk of text"""
return {
"role": "user",
"content": f"\"\"\"{chunk}\"\"\" Using the above text, please answer the following question: \"{question}\" -- if the question cannot be answered using the text, please summarize the text."
}
def summarize_text(text, question):
"""Summarize text using the LLM model"""
if not text:
return "Error: No text to summarize"
print("Text length: " + str(len(text)) + " characters")
text_length = len(text)
print(f"Text length: {text_length} characters")
summaries = []
chunks = list(split_text(text))
for i, chunk in enumerate(chunks):
print("Summarizing chunk " + str(i + 1) + " / " + str(len(chunks)))
if is_website:
messages = [
{
"role": "user",
"content": "Please summarize the following website text, do not describe the general website, but instead concisely extract the specific information this subpage contains.: " +
chunk
}
]
else:
messages = [
{
"role": "user",
"content": "Please summarize the following text, focusing on extracting concise and specific information: " +
chunk
}
]
print(f"Summarizing chunk {i + 1} / {len(chunks)}")
messages = [create_message(chunk, question)]
summary = create_chat_completion(
model=cfg.fast_llm_model,
@@ -120,27 +113,11 @@ def summarize_text(text, is_website=True):
max_tokens=300,
)
summaries.append(summary)
print("Summarized " + str(len(chunks)) + " chunks.")
print(f"Summarized {len(chunks)} chunks.")
combined_summary = "\n".join(summaries)
# Summarize the combined summary
if is_website:
messages = [
{
"role": "user",
"content": "Please summarize the following website text, do not describe the general website, but instead concisely extract the specific information this subpage contains.: " +
combined_summary
}
]
else:
messages = [
{
"role": "user",
"content": "Please summarize the following text, focusing on extracting concise and specific infomation: " +
combined_summary
}
]
messages = [create_message(combined_summary, question)]
final_summary = create_chat_completion(
model=cfg.fast_llm_model,
@@ -148,4 +125,4 @@ def summarize_text(text, is_website=True):
max_tokens=300,
)
return final_summary
return final_summary