Merge branch 'master' into add_website_memory

scripts/browse.py

@@ -11,6 +11,7 @@ memory = get_memory(cfg)
 session = requests.Session()
 session.headers.update({'User-Agent': cfg.user_agent})
 
+
 # Function to check if the URL is valid
 def is_valid_url(url):
     try:
@@ -19,49 +20,51 @@ def is_valid_url(url):
     except ValueError:
         return False
 
+
 # Function to sanitize the URL
 def sanitize_url(url):
     return urljoin(url, urlparse(url).path)
 
-# Function to make a request with a specified timeout and handle exceptions
-def make_request(url, timeout=10):
-    try:
-        response = session.get(url, timeout=timeout)
-        response.raise_for_status()
-        return response
-    except requests.exceptions.RequestException as e:
-        return "Error: " + str(e)
 
 # Define and check for local file address prefixes
 def check_local_file_access(url):
     local_prefixes = ['file:///', 'file://localhost', 'http://localhost', 'https://localhost']
     return any(url.startswith(prefix) for prefix in local_prefixes)
 
+
+def get_response(url, timeout=10):
+    try:
+        # Restrict access to local files
+        if check_local_file_access(url):
+            raise ValueError('Access to local files is restricted')
+
+        # Most basic check if the URL is valid:
+        if not url.startswith('http://') and not url.startswith('https://'):
+            raise ValueError('Invalid URL format')
+
+        sanitized_url = sanitize_url(url)
+
+        response = session.get(sanitized_url, timeout=timeout)
+
+        # Check if the response contains an HTTP error
+        if response.status_code >= 400:
+            return None, "Error: HTTP " + str(response.status_code) + " error"
+
+        return response, None
+    except ValueError as ve:
+        # Handle invalid URL format
+        return None, "Error: " + str(ve)
+
+    except requests.exceptions.RequestException as re:
+        # Handle exceptions related to the HTTP request (e.g., connection errors, timeouts, etc.)
+        return None, "Error: " + str(re)
+
+
 def scrape_text(url):
     """Scrape text from a webpage"""
-    # Basic check if the URL is valid
-    if not url.startswith('http'):
-        return "Error: Invalid URL"
-
-    # Restrict access to local files
-    if check_local_file_access(url):
-        return "Error: Access to local files is restricted"
-
-    # Validate the input URL
-    if not is_valid_url(url):
-        # Sanitize the input URL
-        sanitized_url = sanitize_url(url)
-
-        # Make the request with a timeout and handle exceptions
-        response = make_request(sanitized_url)
-
-        if isinstance(response, str):
-            return response
-    else:
-        # Sanitize the input URL
-        sanitized_url = sanitize_url(url)
-
-        response = session.get(sanitized_url)
+    response, error_message = get_response(url)
+    if error_message:
+        return error_message
 
     soup = BeautifulSoup(response.text, "html.parser")
 
@@ -94,11 +97,9 @@ def format_hyperlinks(hyperlinks):
 
 def scrape_links(url):
     """Scrape links from a webpage"""
-    response = session.get(url)
-
-    # Check if the response contains an HTTP error
-    if response.status_code >= 400:
-        return "error"
+    response, error_message = get_response(url)
+    if error_message:
+        return error_message
 
     soup = BeautifulSoup(response.text, "html.parser")