flake8 style

This commit is contained in:
Itamar Friedman
2023-04-13 00:00:33 +03:00
parent a40ccc1e5d
commit 9f972f4ee9
3 changed files with 15 additions and 7 deletions

View File

@@ -6,6 +6,7 @@ from urllib.parse import urlparse, urljoin
cfg = Config()
# Function to check if the URL is valid
def is_valid_url(url):
try:
@@ -14,15 +15,18 @@ def is_valid_url(url):
except ValueError:
return False
# Function to sanitize the URL
def sanitize_url(url):
return urljoin(url, urlparse(url).path)
# Define and check for local file address prefixes
def check_local_file_access(url):
local_prefixes = ['file:///', 'file://localhost', 'http://localhost', 'https://localhost']
return any(url.startswith(prefix) for prefix in local_prefixes)
def get_response(url, headers=cfg.user_agent_header, timeout=10):
try:
# Restrict access to local files
@@ -33,7 +37,6 @@ def get_response(url, headers=cfg.user_agent_header, timeout=10):
if not url.startswith('http://') and not url.startswith('https://'):
raise ValueError('Invalid URL format')
sanitized_url = sanitize_url(url)
response = requests.get(sanitized_url, headers=headers, timeout=timeout)
@@ -51,6 +54,7 @@ def get_response(url, headers=cfg.user_agent_header, timeout=10):
# Handle exceptions related to the HTTP request (e.g., connection errors, timeouts, etc.)
return None, "Error: " + str(re)
def scrape_text(url):
"""Scrape text from a webpage"""
response, error_message = get_response(url)
@@ -128,6 +132,7 @@ def create_message(chunk, question):
"content": f"\"\"\"{chunk}\"\"\" Using the above text, please answer the following question: \"{question}\" -- if the question cannot be answered using the text, please summarize the text."
}
def summarize_text(text, question):
"""Summarize text using the LLM model"""
if not text: