aljaz/Auto-GPT
1
0
Fork 0
mirror of https://github.com/aljazceru/Auto-GPT.git synced 2025-12-17 22:14:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Blacked

Browse Source
This commit is contained in:
BillSchumacher
2023-04-15 17:30:28 -05:00
parent f2035231e3
commit ef4e4eb5d4
1 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
autogpt/commands/web_playwright.py
View File

@@ -2,7 +2,9 @@
try: try:
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
except ImportError: except ImportError:
print("Playwright not installed. Please install it with 'pip install playwright' to use.") print(
"Playwright not installed. Please install it with 'pip install playwright' to use."
)
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from autogpt.processing.html import extract_hyperlinks, format_hyperlinks from autogpt.processing.html import extract_hyperlinks, format_hyperlinks
from typing import List, Union from typing import List, Union
@@ -10,10 +12,10 @@ from typing import List, Union
def scrape_text(url: str) -> str: def scrape_text(url: str) -> str:
"""Scrape text from a webpage """Scrape text from a webpage
Args: Args:
url (str): The URL to scrape text from url (str): The URL to scrape text from
Returns: Returns:
str: The scraped text str: The scraped text
""" """
@@ -32,7 +34,7 @@ def scrape_text(url: str) -> str:
text = soup.get_text() text = soup.get_text()
lines = (line.strip() for line in text.splitlines()) lines = (line.strip() for line in text.splitlines())
chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
text = '\n'.join(chunk for chunk in chunks if chunk) text = "\n".join(chunk for chunk in chunks if chunk)
except Exception as e: except Exception as e:
text = f"Error: {str(e)}" text = f"Error: {str(e)}"