mirror of
https://github.com/aljazceru/Auto-GPT.git
synced 2025-12-18 14:34:23 +01:00
Update browse.py
Add separation between code blocks and standardize dictionary definition.
This commit is contained in:
@@ -5,6 +5,7 @@ from llm_utils import create_chat_completion
|
|||||||
|
|
||||||
cfg = Config()
|
cfg = Config()
|
||||||
|
|
||||||
|
|
||||||
def scrape_text(url):
|
def scrape_text(url):
|
||||||
"""Scrape text from a webpage"""
|
"""Scrape text from a webpage"""
|
||||||
response = requests.get(url)
|
response = requests.get(url)
|
||||||
@@ -29,16 +30,20 @@ def scrape_text(url):
|
|||||||
def extract_hyperlinks(soup):
|
def extract_hyperlinks(soup):
|
||||||
"""Extract hyperlinks from a BeautifulSoup object"""
|
"""Extract hyperlinks from a BeautifulSoup object"""
|
||||||
hyperlinks = []
|
hyperlinks = []
|
||||||
|
|
||||||
for link in soup.find_all('a', href=True):
|
for link in soup.find_all('a', href=True):
|
||||||
hyperlinks.append((link.text, link['href']))
|
hyperlinks.append((link.text, link['href']))
|
||||||
|
|
||||||
return hyperlinks
|
return hyperlinks
|
||||||
|
|
||||||
|
|
||||||
def format_hyperlinks(hyperlinks):
|
def format_hyperlinks(hyperlinks):
|
||||||
"""Format hyperlinks into a list of strings"""
|
"""Format hyperlinks into a list of strings"""
|
||||||
formatted_links = []
|
formatted_links = []
|
||||||
|
|
||||||
for link_text, link_url in hyperlinks:
|
for link_text, link_url in hyperlinks:
|
||||||
formatted_links.append(f"{link_text} ({link_url})")
|
formatted_links.append(f"{link_text} ({link_url})")
|
||||||
|
|
||||||
return formatted_links
|
return formatted_links
|
||||||
|
|
||||||
|
|
||||||
@@ -67,6 +72,7 @@ def split_text(text, max_length=8192):
|
|||||||
current_chunk = []
|
current_chunk = []
|
||||||
|
|
||||||
for paragraph in paragraphs:
|
for paragraph in paragraphs:
|
||||||
|
|
||||||
if current_length + len(paragraph) + 1 <= max_length:
|
if current_length + len(paragraph) + 1 <= max_length:
|
||||||
current_chunk.append(paragraph)
|
current_chunk.append(paragraph)
|
||||||
current_length += len(paragraph) + 1
|
current_length += len(paragraph) + 1
|
||||||
@@ -90,19 +96,22 @@ def summarize_text(text, is_website=True):
|
|||||||
|
|
||||||
for i, chunk in enumerate(chunks):
|
for i, chunk in enumerate(chunks):
|
||||||
print("Summarizing chunk " + str(i + 1) + " / " + str(len(chunks)))
|
print("Summarizing chunk " + str(i + 1) + " / " + str(len(chunks)))
|
||||||
|
|
||||||
if is_website:
|
if is_website:
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
"role": "user",
|
"role": "user",
|
||||||
"content": "Please summarize the following website text, do not describe the general website, but instead concisely extract the specific information this subpage contains.: " +
|
"content": "Please summarize the following website text, do not describe the general website, but instead concisely extract the specific information this subpage contains.: " +
|
||||||
chunk},
|
chunk
|
||||||
|
}
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
"role": "user",
|
"role": "user",
|
||||||
"content": "Please summarize the following text, focusing on extracting concise and specific information: " +
|
"content": "Please summarize the following text, focusing on extracting concise and specific information: " +
|
||||||
chunk},
|
chunk
|
||||||
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
summary = create_chat_completion(
|
summary = create_chat_completion(
|
||||||
@@ -121,14 +130,16 @@ def summarize_text(text, is_website=True):
|
|||||||
{
|
{
|
||||||
"role": "user",
|
"role": "user",
|
||||||
"content": "Please summarize the following website text, do not describe the general website, but instead concisely extract the specific information this subpage contains.: " +
|
"content": "Please summarize the following website text, do not describe the general website, but instead concisely extract the specific information this subpage contains.: " +
|
||||||
combined_summary},
|
combined_summary
|
||||||
|
}
|
||||||
]
|
]
|
||||||
else:
|
else:
|
||||||
messages = [
|
messages = [
|
||||||
{
|
{
|
||||||
"role": "user",
|
"role": "user",
|
||||||
"content": "Please summarize the following text, focusing on extracting concise and specific infomation: " +
|
"content": "Please summarize the following text, focusing on extracting concise and specific infomation: " +
|
||||||
combined_summary},
|
combined_summary
|
||||||
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
final_summary = create_chat_completion(
|
final_summary = create_chat_completion(
|
||||||
|
|||||||
Reference in New Issue
Block a user