aljaz/Auto-GPT
1
0
Fork 0
mirror of https://github.com/aljazceru/Auto-GPT.git synced 2025-12-19 23:14:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

improve performance and removed code duplication

Browse Source
This commit is contained in:
pratiksinghchauhan
2023-04-10 11:39:52 +05:30
parent 816dc14d82
commit 33c8fe627a
2 changed files with 15 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
scripts/browse.py
View File

@@ -5,14 +5,17 @@ from llm_utils import create_chat_completion
cfg = Config()
def scrape_text(url):
def get_website_content(url):
response = requests.get(url, headers=cfg.user_agent_header)
# Check if the response contains an HTTP error
if response.status_code >= 400:
return "Error: HTTP " + str(response.status_code) + " error"
return response
soup = BeautifulSoup(response.text, "html.parser")
def scrape_text(website_content):
soup = BeautifulSoup(website_content.text, "html.parser")
for script in soup(["script", "style"]):
script.extract()
@@ -39,14 +42,8 @@ def format_hyperlinks(hyperlinks):
return formatted_links
def scrape_links(url):
response = requests.get(url, headers=cfg.user_agent_header)
# Check if the response contains an HTTP error
if response.status_code >= 400:
return "error"
soup = BeautifulSoup(response.text, "html.parser")
def scrape_links(website_content):
soup = BeautifulSoup(website_content.text, "html.parser")
for script in soup(["script", "style"]):
script.extract()