mirror of
https://github.com/aljazceru/Auto-GPT.git
synced 2026-01-02 05:44:32 +01:00
113 lines
3.1 KiB
Python
113 lines
3.1 KiB
Python
"""Browse a webpage and summarize it using the LLM model"""
|
|
from __future__ import annotations
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from requests import Response
|
|
|
|
from autogpt.config import Config
|
|
from autogpt.processing.html import extract_hyperlinks, format_hyperlinks
|
|
from autogpt.url_utils.validators import validate_url
|
|
|
|
CFG = Config()
|
|
|
|
session = requests.Session()
|
|
session.headers.update({"User-Agent": CFG.user_agent})
|
|
|
|
|
|
@validate_url
|
|
def get_response(
|
|
url: str, timeout: int = 10
|
|
) -> tuple[None, str] | tuple[Response, None]:
|
|
"""Get the response from a URL
|
|
|
|
Args:
|
|
url (str): The URL to get the response from
|
|
timeout (int): The timeout for the HTTP request
|
|
|
|
Returns:
|
|
tuple[None, str] | tuple[Response, None]: The response and error message
|
|
|
|
Raises:
|
|
ValueError: If the URL is invalid
|
|
requests.exceptions.RequestException: If the HTTP request fails
|
|
"""
|
|
try:
|
|
response = session.get(url, timeout=timeout)
|
|
|
|
# Check if the response contains an HTTP error
|
|
if response.status_code >= 400:
|
|
return None, f"Error: HTTP {str(response.status_code)} error"
|
|
|
|
return response, None
|
|
except ValueError as ve:
|
|
# Handle invalid URL format
|
|
return None, f"Error: {str(ve)}"
|
|
|
|
except requests.exceptions.RequestException as re:
|
|
# Handle exceptions related to the HTTP request
|
|
# (e.g., connection errors, timeouts, etc.)
|
|
return None, f"Error: {str(re)}"
|
|
|
|
|
|
def scrape_text(url: str) -> str:
|
|
"""Scrape text from a webpage
|
|
|
|
Args:
|
|
url (str): The URL to scrape text from
|
|
|
|
Returns:
|
|
str: The scraped text
|
|
"""
|
|
response, error_message = get_response(url)
|
|
if error_message:
|
|
return error_message
|
|
if not response:
|
|
return "Error: Could not get response"
|
|
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
|
|
for script in soup(["script", "style"]):
|
|
script.extract()
|
|
|
|
text = soup.get_text()
|
|
lines = (line.strip() for line in text.splitlines())
|
|
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
|
|
text = "\n".join(chunk for chunk in chunks if chunk)
|
|
|
|
return text
|
|
|
|
|
|
def scrape_links(url: str) -> str | list[str]:
|
|
"""Scrape links from a webpage
|
|
|
|
Args:
|
|
url (str): The URL to scrape links from
|
|
|
|
Returns:
|
|
str | list[str]: The scraped links
|
|
"""
|
|
response, error_message = get_response(url)
|
|
if error_message:
|
|
return error_message
|
|
if not response:
|
|
return "Error: Could not get response"
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
|
|
|
for script in soup(["script", "style"]):
|
|
script.extract()
|
|
|
|
hyperlinks = extract_hyperlinks(soup, url)
|
|
|
|
return format_hyperlinks(hyperlinks)
|
|
|
|
|
|
def create_message(chunk, question):
|
|
"""Create a message for the user to summarize a chunk of text"""
|
|
return {
|
|
"role": "user",
|
|
"content": f'"""{chunk}""" Using the above text, answer the following'
|
|
f' question: "{question}" -- if the question cannot be answered using the'
|
|
" text, summarize the text.",
|
|
}
|