mirror of
https://github.com/aljazceru/Auto-GPT.git
synced 2026-01-21 06:54:24 +01:00
Adds error check to text scraper.
This commit is contained in:
@@ -4,8 +4,14 @@ from bs4 import BeautifulSoup
|
||||
from readability import Document#
|
||||
import openai
|
||||
|
||||
|
||||
def scrape_text(url):
|
||||
response = requests.get(url)
|
||||
|
||||
# Check if the response contains an HTTP error
|
||||
if response.status_code >= 400:
|
||||
return "Error: HTTP " + str(response.status_code) + " error"
|
||||
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
|
||||
for script in soup(["script", "style"]):
|
||||
|
||||
Reference in New Issue
Block a user