aljaz/Auto-GPT
1
0
Fork 0
mirror of https://github.com/aljazceru/Auto-GPT.git synced 2025-12-18 06:24:20 +01:00
Code Issues Packages Projects Releases Wiki Activity

Added fake user-agent headers to browser request.

Browse Source
This commit is contained in:
Petar Ostojic
2023-04-05 23:44:16 +02:00
parent 62dfd84599
commit 62854afe68
2 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/browse.py
View File

@@ -6,7 +6,7 @@ from llm_utils import create_chat_completion
cfg = Config()
def scrape_text(url):
response = requests.get(url)
response = requests.get(url, headers=cfg.user_agent_header)
# Check if the response contains an HTTP error
if response.status_code >= 400:
@@ -40,7 +40,7 @@ def format_hyperlinks(hyperlinks):
def scrape_links(url):
response = requests.get(url)
response = requests.get(url, headers=cfg.user_agent_header)
# Check if the response contains an HTTP error
if response.status_code >= 400:

5
scripts/config.py
View File

@@ -39,6 +39,11 @@ class Config(metaclass=Singleton):
self.google_api_key = os.getenv("GOOGLE_API_KEY")
self.custom_search_engine_id = os.getenv("CUSTOM_SEARCH_ENGINE_ID")
# User agent headers to use when browsing web
# Some websites might just completely deny request with an error code if no user agent was found.
self.user_agent_header = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"}
# Initialize the OpenAI API client
openai.api_key = self.openai_api_key