diff --git a/autogpt/commands/web_playwright.py b/autogpt/commands/web_playwright.py new file mode 100644 index 00000000..93a46ac9 --- /dev/null +++ b/autogpt/commands/web_playwright.py @@ -0,0 +1,78 @@ +"""Web scraping commands using Playwright""" +try: + from playwright.sync_api import sync_playwright +except ImportError: + print( + "Playwright not installed. Please install it with 'pip install playwright' to use." + ) +from bs4 import BeautifulSoup +from autogpt.processing.html import extract_hyperlinks, format_hyperlinks +from typing import List, Union + + +def scrape_text(url: str) -> str: + """Scrape text from a webpage + + Args: + url (str): The URL to scrape text from + + Returns: + str: The scraped text + """ + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page() + + try: + page.goto(url) + html_content = page.content() + soup = BeautifulSoup(html_content, "html.parser") + + for script in soup(["script", "style"]): + script.extract() + + text = soup.get_text() + lines = (line.strip() for line in text.splitlines()) + chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) + text = "\n".join(chunk for chunk in chunks if chunk) + + except Exception as e: + text = f"Error: {str(e)}" + + finally: + browser.close() + + return text + + +def scrape_links(url: str) -> Union[str, List[str]]: + """Scrape links from a webpage + + Args: + url (str): The URL to scrape links from + + Returns: + Union[str, List[str]]: The scraped links + """ + with sync_playwright() as p: + browser = p.chromium.launch() + page = browser.new_page() + + try: + page.goto(url) + html_content = page.content() + soup = BeautifulSoup(html_content, "html.parser") + + for script in soup(["script", "style"]): + script.extract() + + hyperlinks = extract_hyperlinks(soup, url) + formatted_links = format_hyperlinks(hyperlinks) + + except Exception as e: + formatted_links = f"Error: {str(e)}" + + finally: + browser.close() + + return formatted_links diff --git a/autogpt/commands/web_requests.py b/autogpt/commands/web_requests.py index 051cc710..a6161ec5 100644 --- a/autogpt/commands/web_requests.py +++ b/autogpt/commands/web_requests.py @@ -9,6 +9,7 @@ from bs4 import BeautifulSoup from autogpt.config import Config from autogpt.memory import get_memory +from autogpt.processing.html import extract_hyperlinks, format_hyperlinks CFG = Config() memory = get_memory(CFG) @@ -135,37 +136,6 @@ def scrape_text(url: str) -> str: return text -def extract_hyperlinks(soup: BeautifulSoup, base_url: str) -> List[Tuple[str, str]]: - """Extract hyperlinks from a BeautifulSoup object - - Args: - soup (BeautifulSoup): The BeautifulSoup object - base_url (str): The base URL - - Returns: - List[Tuple[str, str]]: The extracted hyperlinks - """ - return [ - (link.text, urljoin(base_url, link["href"])) - for link in soup.find_all("a", href=True) - ] - - -def format_hyperlinks(hyperlinks: List[Tuple[str, str]]) -> List[str]: - """Format hyperlinks into a list of strings - - Args: - hyperlinks (List[Tuple[str, str]]): The hyperlinks to format - - Returns: - List[str]: The formatted hyperlinks - """ - formatted_links = [] - for link_text, link_url in hyperlinks: - formatted_links.append(f"{link_text} ({link_url})") - return formatted_links - - def scrape_links(url: str) -> Union[str, List[str]]: """Scrape links from a webpage diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index d2ef1a7e..359803ee 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -1,5 +1,6 @@ """Selenium web scraping module.""" from selenium import webdriver +from autogpt.processing.html import extract_hyperlinks, format_hyperlinks import autogpt.processing.text as summary from bs4 import BeautifulSoup from selenium.webdriver.remote.webdriver import WebDriver @@ -33,7 +34,7 @@ def browse_website(url: str, question: str) -> Tuple[str, WebDriver]: driver, text = scrape_text_with_selenium(url) add_header(driver) summary_text = summary.summarize_text(url, text, question, driver) - links = scrape_links_with_selenium(driver) + links = scrape_links_with_selenium(driver, url) # Limit links to 5 if len(links) > 5: @@ -96,7 +97,7 @@ def scrape_text_with_selenium(url: str) -> Tuple[WebDriver, str]: return driver, text -def scrape_links_with_selenium(driver: WebDriver) -> List[str]: +def scrape_links_with_selenium(driver: WebDriver, url: str) -> List[str]: """Scrape links from a website using selenium Args: @@ -111,7 +112,7 @@ def scrape_links_with_selenium(driver: WebDriver) -> List[str]: for script in soup(["script", "style"]): script.extract() - hyperlinks = extract_hyperlinks(soup) + hyperlinks = extract_hyperlinks(soup, url) return format_hyperlinks(hyperlinks) @@ -128,30 +129,6 @@ def close_browser(driver: WebDriver) -> None: driver.quit() -def extract_hyperlinks(soup: BeautifulSoup) -> List[Tuple[str, str]]: - """Extract hyperlinks from a BeautifulSoup object - - Args: - soup (BeautifulSoup): The BeautifulSoup object to extract the hyperlinks from - - Returns: - List[Tuple[str, str]]: The hyperlinks extracted from the BeautifulSoup object - """ - return [(link.text, link["href"]) for link in soup.find_all("a", href=True)] - - -def format_hyperlinks(hyperlinks: List[Tuple[str, str]]) -> List[str]: - """Format hyperlinks to be displayed to the user - - Args: - hyperlinks (List[Tuple[str, str]]): The hyperlinks to format - - Returns: - List[str]: The formatted hyperlinks - """ - return [f"{link_text} ({link_url})" for link_text, link_url in hyperlinks] - - def add_header(driver: WebDriver) -> None: """Add a header to the website diff --git a/autogpt/processing/html.py b/autogpt/processing/html.py new file mode 100644 index 00000000..c43a0b74 --- /dev/null +++ b/autogpt/processing/html.py @@ -0,0 +1,32 @@ +"""HTML processing functions""" +from requests.compat import urljoin +from typing import List, Tuple +from bs4 import BeautifulSoup + + +def extract_hyperlinks(soup: BeautifulSoup, base_url: str) -> List[Tuple[str, str]]: + """Extract hyperlinks from a BeautifulSoup object + + Args: + soup (BeautifulSoup): The BeautifulSoup object + base_url (str): The base URL + + Returns: + List[Tuple[str, str]]: The extracted hyperlinks + """ + return [ + (link.text, urljoin(base_url, link["href"])) + for link in soup.find_all("a", href=True) + ] + + +def format_hyperlinks(hyperlinks: List[Tuple[str, str]]) -> List[str]: + """Format hyperlinks to be displayed to the user + + Args: + hyperlinks (List[Tuple[str, str]]): The hyperlinks to format + + Returns: + List[str]: The formatted hyperlinks + """ + return [f"{link_text} ({link_url})" for link_text, link_url in hyperlinks]