Merge branch 'master' into browse_scrape_links_test_and_validate

This commit is contained in:
Itamar Friedman
2023-04-12 23:51:53 +03:00
15 changed files with 29 additions and 19 deletions

View File

@@ -33,6 +33,7 @@ def get_response(url, headers=cfg.user_agent_header, timeout=10):
if not url.startswith('http://') and not url.startswith('https://'):
raise ValueError('Invalid URL format')
sanitized_url = sanitize_url(url)
response = requests.get(sanitized_url, headers=headers, timeout=timeout)