diff --git a/autogpt/agent/agent.py b/autogpt/agent/agent.py index dca614c7..6ec0a623 100644 --- a/autogpt/agent/agent.py +++ b/autogpt/agent/agent.py @@ -3,8 +3,8 @@ from autogpt.app import execute_command, get_command from autogpt.chat import chat_with_ai, create_chat_message from autogpt.config import Config -from autogpt.json_fixes.master_json_fix_method import fix_json_using_multiple_techniques -from autogpt.json_validation.validate_json import validate_json +from autogpt.json_utils.auto_fix import fix_json_using_multiple_techniques +from autogpt.json_utils.utilities import validate_json from autogpt.logs import logger, print_assistant_thoughts from autogpt.speech import say_text from autogpt.spinner import Spinner diff --git a/autogpt/app.py b/autogpt/app.py index 78b5bd2f..190f934b 100644 --- a/autogpt/app.py +++ b/autogpt/app.py @@ -18,7 +18,7 @@ from autogpt.commands.file_operations import ( search_files, write_to_file, ) -from autogpt.json_fixes.parsing import fix_and_parse_json +from autogpt.json_utils.auto_fix import fix_and_parse_json from autogpt.memory import get_memory from autogpt.processing.text import summarize_text from autogpt.speech import say_text diff --git a/autogpt/json_fixes/auto_fix.py b/autogpt/json_fixes/auto_fix.py deleted file mode 100644 index 9fcf909a..00000000 --- a/autogpt/json_fixes/auto_fix.py +++ /dev/null @@ -1,53 +0,0 @@ -"""This module contains the function to fix JSON strings using GPT-3.""" -import json - -from autogpt.llm_utils import call_ai_function -from autogpt.logs import logger -from autogpt.config import Config - -CFG = Config() - - -def fix_json(json_string: str, schema: str) -> str: - """Fix the given JSON string to make it parseable and fully compliant with - the provided schema. - - Args: - json_string (str): The JSON string to fix. - schema (str): The schema to use to fix the JSON. - Returns: - str: The fixed JSON string. - """ - # Try to fix the JSON using GPT: - function_string = "def fix_json(json_string: str, schema:str=None) -> str:" - args = [f"'''{json_string}'''", f"'''{schema}'''"] - description_string = ( - "This function takes a JSON string and ensures that it" - " is parseable and fully compliant with the provided schema. If an object" - " or field specified in the schema isn't contained within the correct JSON," - " it is omitted. The function also escapes any double quotes within JSON" - " string values to ensure that they are valid. If the JSON string contains" - " any None or NaN values, they are replaced with null before being parsed." - ) - - # If it doesn't already start with a "`", add one: - if not json_string.startswith("`"): - json_string = "```json\n" + json_string + "\n```" - result_string = call_ai_function( - function_string, args, description_string, model=CFG.fast_llm_model - ) - logger.debug("------------ JSON FIX ATTEMPT ---------------") - logger.debug(f"Original JSON: {json_string}") - logger.debug("-----------") - logger.debug(f"Fixed JSON: {result_string}") - logger.debug("----------- END OF FIX ATTEMPT ----------------") - - try: - json.loads(result_string) # just check the validity - return result_string - except json.JSONDecodeError: # noqa: E722 - # Get the call stack: - # import traceback - # call_stack = traceback.format_exc() - # print(f"Failed to fix JSON: '{json_string}' "+call_stack) - return "failed" diff --git a/autogpt/json_fixes/bracket_termination.py b/autogpt/json_fixes/bracket_termination.py deleted file mode 100644 index dd9a8376..00000000 --- a/autogpt/json_fixes/bracket_termination.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Fix JSON brackets.""" -from __future__ import annotations - -import contextlib -import json -from typing import Optional -from autogpt.config import Config - -CFG = Config() - - -def balance_braces(json_string: str) -> Optional[str]: - """ - Balance the braces in a JSON string. - - Args: - json_string (str): The JSON string. - - Returns: - str: The JSON string with braces balanced. - """ - - open_braces_count = json_string.count("{") - close_braces_count = json_string.count("}") - - while open_braces_count > close_braces_count: - json_string += "}" - close_braces_count += 1 - - while close_braces_count > open_braces_count: - json_string = json_string.rstrip("}") - close_braces_count -= 1 - - with contextlib.suppress(json.JSONDecodeError): - json.loads(json_string) - return json_string diff --git a/autogpt/json_fixes/escaping.py b/autogpt/json_fixes/escaping.py deleted file mode 100644 index 68eb1714..00000000 --- a/autogpt/json_fixes/escaping.py +++ /dev/null @@ -1,33 +0,0 @@ -""" Fix invalid escape sequences in JSON strings. """ -import json - -from autogpt.config import Config -from autogpt.json_fixes.utilities import extract_char_position - -CFG = Config() - - -def fix_invalid_escape(json_to_load: str, error_message: str) -> str: - """Fix invalid escape sequences in JSON strings. - - Args: - json_to_load (str): The JSON string. - error_message (str): The error message from the JSONDecodeError - exception. - - Returns: - str: The JSON string with invalid escape sequences fixed. - """ - while error_message.startswith("Invalid \\escape"): - bad_escape_location = extract_char_position(error_message) - json_to_load = ( - json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1 :] - ) - try: - json.loads(json_to_load) - return json_to_load - except json.JSONDecodeError as e: - if CFG.debug_mode: - print("json loads error - fix invalid escape", e) - error_message = str(e) - return json_to_load diff --git a/autogpt/json_fixes/master_json_fix_method.py b/autogpt/json_fixes/master_json_fix_method.py deleted file mode 100644 index 7a2cf3cc..00000000 --- a/autogpt/json_fixes/master_json_fix_method.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Any, Dict - -from autogpt.config import Config -from autogpt.logs import logger -from autogpt.speech import say_text -CFG = Config() - - -def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: - from autogpt.json_fixes.parsing import attempt_to_fix_json_by_finding_outermost_brackets - - from autogpt.json_fixes.parsing import fix_and_parse_json - - # Parse and print Assistant response - assistant_reply_json = fix_and_parse_json(assistant_reply) - if assistant_reply_json == {}: - assistant_reply_json = attempt_to_fix_json_by_finding_outermost_brackets( - assistant_reply - ) - - if assistant_reply_json != {}: - return assistant_reply_json - - logger.error("Error: The following AI output couldn't be converted to a JSON:\n", assistant_reply) - if CFG.speak_mode: - say_text("I have received an invalid JSON response from the OpenAI API.") - - return {} diff --git a/autogpt/json_fixes/missing_quotes.py b/autogpt/json_fixes/missing_quotes.py deleted file mode 100644 index 552a1517..00000000 --- a/autogpt/json_fixes/missing_quotes.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Fix quotes in a JSON string.""" -import json -import re - - -def add_quotes_to_property_names(json_string: str) -> str: - """ - Add quotes to property names in a JSON string. - - Args: - json_string (str): The JSON string. - - Returns: - str: The JSON string with quotes added to property names. - """ - - def replace_func(match: re.Match) -> str: - return f'"{match[1]}":' - - property_name_pattern = re.compile(r"(\w+):") - corrected_json_string = property_name_pattern.sub(replace_func, json_string) - - try: - json.loads(corrected_json_string) - return corrected_json_string - except json.JSONDecodeError as e: - raise e diff --git a/autogpt/json_fixes/utilities.py b/autogpt/json_fixes/utilities.py deleted file mode 100644 index 0852b18a..00000000 --- a/autogpt/json_fixes/utilities.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Utilities for the json_fixes package.""" -import re - - -def extract_char_position(error_message: str) -> int: - """Extract the character position from the JSONDecodeError message. - - Args: - error_message (str): The error message from the JSONDecodeError - exception. - - Returns: - int: The character position. - """ - - char_pattern = re.compile(r"\(char (\d+)\)") - if match := char_pattern.search(error_message): - return int(match[1]) - else: - raise ValueError("Character position not found in the error message.") diff --git a/autogpt/json_fixes/__init__.py b/autogpt/json_utils/__init__.py similarity index 100% rename from autogpt/json_fixes/__init__.py rename to autogpt/json_utils/__init__.py diff --git a/autogpt/json_fixes/parsing.py b/autogpt/json_utils/auto_fix.py similarity index 51% rename from autogpt/json_fixes/parsing.py rename to autogpt/json_utils/auto_fix.py index 1e391eed..883eba78 100644 --- a/autogpt/json_fixes/parsing.py +++ b/autogpt/json_utils/auto_fix.py @@ -1,20 +1,19 @@ -"""Fix and parse JSON strings.""" +"""This module contains the function to fix JSON strings""" from __future__ import annotations import contextlib import json -from typing import Any, Dict, Union +import re +from typing import Optional, Dict, Any + from colorama import Fore from regex import regex -from autogpt.config import Config -from autogpt.json_fixes.auto_fix import fix_json -from autogpt.json_fixes.bracket_termination import balance_braces -from autogpt.json_fixes.escaping import fix_invalid_escape -from autogpt.json_fixes.missing_quotes import add_quotes_to_property_names + +from autogpt.json_utils.utilities import extract_char_position +from autogpt.llm_utils import call_ai_function from autogpt.logs import logger from autogpt.speech import say_text - -CFG = Config() +from autogpt.config import Config JSON_SCHEMA = """ { @@ -35,6 +34,157 @@ JSON_SCHEMA = """ } """ +CFG = Config() + + +def auto_fix_json(json_string: str, schema: str) -> str: + """Fix the given JSON string to make it parseable and fully compliant with + the provided schema using GPT-3. + + Args: + json_string (str): The JSON string to fix. + schema (str): The schema to use to fix the JSON. + Returns: + str: The fixed JSON string. + """ + # Try to fix the JSON using GPT: + function_string = "def fix_json(json_string: str, schema:str=None) -> str:" + args = [f"'''{json_string}'''", f"'''{schema}'''"] + description_string = ( + "This function takes a JSON string and ensures that it" + " is parseable and fully compliant with the provided schema. If an object" + " or field specified in the schema isn't contained within the correct JSON," + " it is omitted. The function also escapes any double quotes within JSON" + " string values to ensure that they are valid. If the JSON string contains" + " any None or NaN values, they are replaced with null before being parsed." + ) + + # If it doesn't already start with a "`", add one: + if not json_string.startswith("`"): + json_string = "```json\n" + json_string + "\n```" + result_string = call_ai_function( + function_string, args, description_string, model=CFG.fast_llm_model + ) + logger.debug("------------ JSON FIX ATTEMPT ---------------") + logger.debug(f"Original JSON: {json_string}") + logger.debug("-----------") + logger.debug(f"Fixed JSON: {result_string}") + logger.debug("----------- END OF FIX ATTEMPT ----------------") + + try: + json.loads(result_string) # just check the validity + return result_string + except json.JSONDecodeError: # noqa: E722 + # Get the call stack: + # import traceback + # call_stack = traceback.format_exc() + # print(f"Failed to fix JSON: '{json_string}' "+call_stack) + return "failed" + + +def fix_invalid_escape(json_to_load: str, error_message: str) -> str: + """Fix invalid escape sequences in JSON strings. + + Args: + json_to_load (str): The JSON string. + error_message (str): The error message from the JSONDecodeError + exception. + + Returns: + str: The JSON string with invalid escape sequences fixed. + """ + while error_message.startswith("Invalid \\escape"): + bad_escape_location = extract_char_position(error_message) + json_to_load = ( + json_to_load[:bad_escape_location] + json_to_load[bad_escape_location + 1:] + ) + try: + json.loads(json_to_load) + return json_to_load + except json.JSONDecodeError as e: + if CFG.debug_mode: + print("json loads error - fix invalid escape", e) + error_message = str(e) + return json_to_load + + +def balance_braces(json_string: str) -> Optional[str]: + """ + Balance the braces in a JSON string. + + Args: + json_string (str): The JSON string. + + Returns: + str: The JSON string with braces balanced. + """ + + open_braces_count = json_string.count("{") + close_braces_count = json_string.count("}") + + while open_braces_count > close_braces_count: + json_string += "}" + close_braces_count += 1 + + while close_braces_count > open_braces_count: + json_string = json_string.rstrip("}") + close_braces_count -= 1 + + with contextlib.suppress(json.JSONDecodeError): + json.loads(json_string) + return json_string + + +def add_quotes_to_property_names(json_string: str) -> str: + """ + Add quotes to property names in a JSON string. + + Args: + json_string (str): The JSON string. + + Returns: + str: The JSON string with quotes added to property names. + """ + + def replace_func(match: re.Match) -> str: + return f'"{match[1]}":' + + property_name_pattern = re.compile(r"(\w+):") + corrected_json_string = property_name_pattern.sub(replace_func, json_string) + + try: + json.loads(corrected_json_string) + return corrected_json_string + except json.JSONDecodeError as e: + raise e + + +def fix_json_using_multiple_techniques(assistant_reply: str) -> Dict[Any, Any]: + """Fix the given JSON string to make it parseable and fully compliant with two techniques. + + Args: + json_string (str): The JSON string to fix. + + Returns: + str: The fixed JSON string. + """ + + # Parse and print Assistant response + assistant_reply_json = fix_and_parse_json(assistant_reply) + if assistant_reply_json == {}: + assistant_reply_json = attempt_to_fix_json_by_finding_outermost_brackets( + assistant_reply + ) + + if assistant_reply_json != {}: + return assistant_reply_json + + logger.error("Error: The following AI output couldn't be converted to a JSON:\n", assistant_reply) + if CFG.speak_mode: + say_text("I have received an invalid JSON response from the OpenAI API.") + + return {} + def correct_json(json_to_load: str) -> str: """ @@ -134,7 +284,7 @@ def try_ai_fix( " slightly." ) # Now try to fix this up using the ai_functions - ai_fixed_json = fix_json(json_to_load, JSON_SCHEMA) + ai_fixed_json = auto_fix_json(json_to_load, JSON_SCHEMA) if ai_fixed_json != "failed": return json.loads(ai_fixed_json) diff --git a/autogpt/json_schemas/llm_response_format_1.json b/autogpt/json_utils/llm_response_format_1.json similarity index 100% rename from autogpt/json_schemas/llm_response_format_1.json rename to autogpt/json_utils/llm_response_format_1.json diff --git a/autogpt/json_validation/validate_json.py b/autogpt/json_utils/utilities.py similarity index 63% rename from autogpt/json_validation/validate_json.py rename to autogpt/json_utils/utilities.py index 440c3b0b..af8a28c9 100644 --- a/autogpt/json_validation/validate_json.py +++ b/autogpt/json_utils/utilities.py @@ -1,10 +1,31 @@ +"""Utilities for the json_fixes package.""" import json +import re + from jsonschema import Draft7Validator -from autogpt.config import Config + from autogpt.logs import logger +from autogpt.config import Config CFG = Config() +def extract_char_position(error_message: str) -> int: + """Extract the character position from the JSONDecodeError message. + + Args: + error_message (str): The error message from the JSONDecodeError + exception. + + Returns: + int: The character position. + """ + + char_pattern = re.compile(r"\(char (\d+)\)") + if match := char_pattern.search(error_message): + return int(match[1]) + else: + raise ValueError("Character position not found in the error message.") + def validate_json(json_object: object, schema_name: object) -> object: """ diff --git a/autogpt/logs.py b/autogpt/logs.py index c1e436db..58375f14 100644 --- a/autogpt/logs.py +++ b/autogpt/logs.py @@ -204,10 +204,7 @@ logger = Logger() def print_assistant_thoughts(ai_name, assistant_reply): """Prints the assistant's thoughts to the console""" - from autogpt.json_fixes.bracket_termination import ( - attempt_to_fix_json_by_finding_outermost_brackets, - ) - from autogpt.json_fixes.parsing import fix_and_parse_json + from autogpt.json_utils.auto_fix import fix_and_parse_json, attempt_to_fix_json_by_finding_outermost_brackets try: try: diff --git a/tests/test_json_parser.py b/tests/test_json_parser.py index 2862034b..f8fa5955 100644 --- a/tests/test_json_parser.py +++ b/tests/test_json_parser.py @@ -1,7 +1,7 @@ import unittest import tests.context -from autogpt.json_fixes.parsing import fix_and_parse_json +from autogpt.json_utils.auto_fix import fix_and_parse_json class TestParseJson(unittest.TestCase): diff --git a/tests/unit/json_tests.py b/tests/unit/json_tests.py index 561b8a38..f65a6f6a 100644 --- a/tests/unit/json_tests.py +++ b/tests/unit/json_tests.py @@ -1,6 +1,6 @@ import unittest -from autogpt.json_parser import fix_and_parse_json +from autogpt.json_utils.auto_fix import fix_and_parse_json class TestParseJson(unittest.TestCase):