Adds error check to text scraper.

This commit is contained in:
Torantulino
2023-03-29 09:43:10 +01:00
parent 70d5fbf690
commit 6d796d222d

View File

@@ -4,8 +4,14 @@ from bs4 import BeautifulSoup
from readability import Document#
import openai
def scrape_text(url):
response = requests.get(url)
# Check if the response contains an HTTP error
if response.status_code >= 400:
return "Error: HTTP " + str(response.status_code) + " error"
soup = BeautifulSoup(response.text, "html.parser")
for script in soup(["script", "style"]):