aljaz/Auto-GPT
1
0
Fork 0
mirror of https://github.com/aljazceru/Auto-GPT.git synced 2025-12-17 14:04:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into dev

Browse Source
This commit is contained in:
Andres Caicedo
2023-04-09 15:42:53 +02:00
parent 011699e6a1 816dc14d82
commit 26abb30674
28 changed files with 918 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
scripts/browse.py
View File

@@ -7,7 +7,7 @@ cfg = Config()
def scrape_text(url):
"""Scrape text from a webpage"""
response = requests.get(url)
response = requests.get(url, headers=cfg.user_agent_header)
# Check if the response contains an HTTP error
if response.status_code >= 400:
@@ -43,8 +43,8 @@ def format_hyperlinks(hyperlinks):
def scrape_links(url):
"""Scrape hyperlinks from a webpage"""
response = requests.get(url)
"""Scrape links from a webpage"""
response = requests.get(url, headers=cfg.user_agent_header)
# Check if the response contains an HTTP error
if response.status_code >= 400: