Merge branch 'master' into add-unit-tests-workflow

scripts/agent_manager.py

@@ -13,7 +13,7 @@ def create_agent(task, prompt, model):
 
     messages = [{"role": "user", "content": prompt}, ]
 
-    # Start GTP3 instance
+    # Start GPT instance
     agent_reply = create_chat_completion(
         model=model,
         messages=messages,
@@ -41,7 +41,7 @@ def message_agent(key, message):
     # Add user message to message history before sending to agent
     messages.append({"role": "user", "content": message})
 
-    # Start GTP3 instance
+    # Start GPT instance
     agent_reply = create_chat_completion(
         model=model,
         messages=messages,

scripts/browse.py

@@ -2,9 +2,31 @@ import requests
 from bs4 import BeautifulSoup
 from config import Config
 from llm_utils import create_chat_completion
+from urllib.parse import urlparse, urljoin
 
 cfg = Config()
 
+# Function to check if the URL is valid
+def is_valid_url(url):
+    try:
+        result = urlparse(url)
+        return all([result.scheme, result.netloc])
+    except ValueError:
+        return False
+
+# Function to sanitize the URL
+def sanitize_url(url):
+    return urljoin(url, urlparse(url).path)
+
+# Function to make a request with a specified timeout and handle exceptions
+def make_request(url, timeout=10):
+    try:
+        response = requests.get(url, headers=cfg.user_agent_header, timeout=timeout)
+        response.raise_for_status()
+        return response
+    except requests.exceptions.RequestException as e:
+        return "Error: " + str(e)
+
 # Define and check for local file address prefixes
 def check_local_file_access(url):
     local_prefixes = ['file:///', 'file://localhost', 'http://localhost', 'https://localhost']
@@ -12,7 +34,7 @@ def check_local_file_access(url):
 
 def scrape_text(url):
     """Scrape text from a webpage"""
-    # Most basic check if the URL is valid:
+    # Basic check if the URL is valid
     if not url.startswith('http'):
         return "Error: Invalid URL"
 
@@ -20,14 +42,21 @@ def scrape_text(url):
     if check_local_file_access(url):
         return "Error: Access to local files is restricted"
     
-    try:
-        response = requests.get(url, headers=cfg.user_agent_header)
-    except requests.exceptions.RequestException as e:
-        return "Error: " + str(e)
+    # Validate the input URL
+    if not is_valid_url(url):
+        # Sanitize the input URL
+        sanitized_url = sanitize_url(url)
 
-    # Check if the response contains an HTTP error
-    if response.status_code >= 400:
-        return "Error: HTTP " + str(response.status_code) + " error"
+        # Make the request with a timeout and handle exceptions
+        response = make_request(sanitized_url)
+
+        if isinstance(response, str):
+            return response
+    else:
+        # Sanitize the input URL
+        sanitized_url = sanitize_url(url)
+
+        response = requests.get(sanitized_url, headers=cfg.user_agent_header)
 
     soup = BeautifulSoup(response.text, "html.parser")
 

scripts/config.py

@@ -49,6 +49,8 @@ class Config(metaclass=Singleton):
             self.openai_api_base = os.getenv("OPENAI_AZURE_API_BASE")
             self.openai_api_version = os.getenv("OPENAI_AZURE_API_VERSION")
             self.openai_deployment_id = os.getenv("OPENAI_AZURE_DEPLOYMENT_ID")
+            self.azure_chat_deployment_id = os.getenv("OPENAI_AZURE_CHAT_DEPLOYMENT_ID")
+            self.azure_embeddigs_deployment_id = os.getenv("OPENAI_AZURE_EMBEDDINGS_DEPLOYMENT_ID")
             openai.api_type = "azure"
             openai.api_base = self.openai_api_base
             openai.api_version = self.openai_api_version

scripts/json_parser.py

@@ -26,7 +26,7 @@ JSON_SCHEMA = """
 """
 
 
-def fix_and_parse_json(    
+def fix_and_parse_json(
     json_str: str,
     try_to_fix_with_gpt: bool = True
 ) -> Union[str, Dict[Any, Any]]:
@@ -35,8 +35,8 @@ def fix_and_parse_json(
         json_str = json_str.replace('\t', '')
         return json.loads(json_str)
     except json.JSONDecodeError as _:  # noqa: F841
-        json_str = correct_json(json_str)
         try:
+            json_str = correct_json(json_str)
             return json.loads(json_str)
         except json.JSONDecodeError as _:  # noqa: F841
             pass
@@ -53,6 +53,7 @@ def fix_and_parse_json(
         last_brace_index = json_str.rindex("}")
         json_str = json_str[:last_brace_index+1]
         return json.loads(json_str)
+    # Can throw a ValueError if there is no "{" or "}" in the json_str
     except (json.JSONDecodeError, ValueError) as e:  # noqa: F841
         if try_to_fix_with_gpt:
             print("Warning: Failed to parse AI output, attempting to fix."

scripts/llm_utils.py

@@ -9,7 +9,7 @@ def create_chat_completion(messages, model=None, temperature=None, max_tokens=No
     """Create a chat completion using the OpenAI API"""
     if cfg.use_azure:
         response = openai.ChatCompletion.create(
-            deployment_id=cfg.openai_deployment_id,
+            deployment_id=cfg.azure_chat_deployment_id,
             model=model,
             messages=messages,
             temperature=temperature,

scripts/main.py

@@ -266,6 +266,7 @@ def prompt_user():
 def parse_arguments():
     """Parses the arguments passed to the script"""
     global cfg
+    cfg.set_debug_mode(False)
     cfg.set_continuous_mode(False)
     cfg.set_speak_mode(False)
 
@@ -274,6 +275,7 @@ def parse_arguments():
     parser.add_argument('--speak', action='store_true', help='Enable Speak Mode')
     parser.add_argument('--debug', action='store_true', help='Enable Debug Mode')
     parser.add_argument('--gpt3only', action='store_true', help='Enable GPT3.5 Only Mode')
+    parser.add_argument('--gpt4only', action='store_true', help='Enable GPT4 Only Mode')
     args = parser.parse_args()
 
     if args.continuous:
@@ -291,7 +293,14 @@ def parse_arguments():
     if args.gpt3only:
         print_to_console("GPT3.5 Only Mode: ", Fore.GREEN, "ENABLED")
         cfg.set_smart_llm_model(cfg.fast_llm_model)
+    
+    if args.gpt4only:
+        print_to_console("GPT4 Only Mode: ", Fore.GREEN, "ENABLED")
+        cfg.set_fast_llm_model(cfg.smart_llm_model)
 
+    if args.debug:
+        print_to_console("Debug Mode: ", Fore.GREEN, "ENABLED")
+        cfg.set_debug_mode(True)
 
 
 # TODO: fill in llm values here
@@ -383,7 +392,7 @@ while True:
             f"COMMAND = {Fore.CYAN}{command_name}{Style.RESET_ALL}  ARGUMENTS = {Fore.CYAN}{arguments}{Style.RESET_ALL}")
 
     # Execute command
-    if command_name.lower().startswith( "error" ):
+    if command_name is not None and command_name.lower().startswith( "error" ):
         result = f"Command {command_name} threw the following error: " + arguments
     elif command_name == "human_feedback":
         result = f"Human feedback: {user_input}"

scripts/memory/base.py

@@ -1,12 +1,16 @@
 """Base class for memory providers."""
 import abc
-from config import AbstractSingleton
+from config import AbstractSingleton, Config
 import openai
+cfg = Config()
 
 
 def get_ada_embedding(text):
     text = text.replace("\n", " ")
-    return openai.Embedding.create(input=[text], model="text-embedding-ada-002")["data"][0]["embedding"]
+    if cfg.use_azure:
+        return openai.Embedding.create(input=[text], engine=cfg.azure_embeddigs_deployment_id, model="text-embedding-ada-002")["data"][0]["embedding"]
+    else:
+        return openai.Embedding.create(input=[text], model="text-embedding-ada-002")["data"][0]["embedding"]
 
 
 class MemoryProviderSingleton(AbstractSingleton):

scripts/memory/local.py

@@ -54,8 +54,8 @@ class LocalCache(MemoryProviderSingleton):
         vector = vector[np.newaxis, :]
         self.data.embeddings = np.concatenate(
             [
-                vector,
                 self.data.embeddings,
+                vector,
             ],
             axis=0,
         )