Auto-GPT/agbenchmark/conftest.py

import json
import os
import shutil
from pathlib import Path  # noqa
from typing import Any, Dict, Generator

import pytest

from agbenchmark.ReportManager import ReportManager
from agbenchmark.start_benchmark import (
    CONFIG_PATH,
    INFO_TESTS_PATH,
    REGRESSION_TESTS_PATH,
    get_regression_data,
)


def resolve_workspace(config: Dict[str, Any]) -> str:
    if config.get("workspace", "").startswith("${") and config.get(
        "workspace", ""
    ).endswith("}"):
        # Extract the string inside ${...}
        path_expr = config["workspace"][2:-1]

        # Check if it starts with "os.path.join"
        if path_expr.strip().startswith("os.path.join"):
            # Evaluate the path string
            path_value = eval(path_expr)

            # Replace the original string with the evaluated result
            return path_value
        else:
            raise ValueError("Invalid workspace path expression.")
    else:
        return os.path.abspath(Path(os.getcwd()) / config["workspace"])


@pytest.fixture(scope="module")
def config(request: Any) -> None:
    print(f"Config file: {CONFIG_PATH}")
    with open(CONFIG_PATH, "r") as f:
        config = json.load(f)

    if isinstance(config["workspace"], str):
        config["workspace"] = resolve_workspace(config)
    else:  # it's a input output dict
        config["workspace"]["input"] = resolve_workspace(config)
        config["workspace"]["output"] = resolve_workspace(config)

    return config


@pytest.fixture(scope="module", autouse=True)
def workspace(config: Dict[str, Any]) -> Generator[str, None, None]:
    output_path = config["workspace"]

    # checks if its an input output paradigm
    if not isinstance(config["workspace"], str):
        output_path = config["workspace"]["output"]
        if not os.path.exists(config["workspace"]["input"]):
            os.makedirs(config["workspace"]["input"], exist_ok=True)

    # create output directory if it doesn't exist
    if not os.path.exists(output_path):
        os.makedirs(output_path, exist_ok=True)

    yield config["workspace"]
    # teardown after test function completes

    for filename in os.listdir(output_path):
        file_path = os.path.join(output_path, filename)
        try:
            if os.path.isfile(file_path) or os.path.islink(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path)
        except Exception as e:
            print(f"Failed to delete {file_path}. Reason: {e}")


def pytest_addoption(parser: Any) -> None:
    parser.addoption("--mock", action="store_true", default=False)
    parser.addoption("--improve", action="store_true", default=False)
    parser.addoption("--maintain", action="store_true", default=False)


@pytest.fixture(autouse=True)
def check_regression(request: Any) -> None:
    test_name = request.node.parent.name
    data = get_regression_data()

    # Get the true location of the test
    challenge_location = getattr(request.node.parent.cls, "CHALLENGE_LOCATION", "")

    skip_string = f"Skipping {test_name} at {challenge_location}"

    # Check if the test name exists in the regression tests
    if request.config.getoption("--improve") and data.get(test_name, None):
        pytest.skip(f"{skip_string} because it's a regression test")
    elif request.config.getoption("--maintain") and not data.get(test_name, None):
        pytest.skip(f"{skip_string} because it's not a regression test")


# this is to get the challenge_data from every test
@pytest.fixture(autouse=True)
def challenge_data(request: Any) -> None:
    return request.param


regression_manager = ReportManager(REGRESSION_TESTS_PATH)
info_manager = ReportManager(INFO_TESTS_PATH)


def pytest_runtest_makereport(item: Any, call: Any) -> None:
    if call.when == "call":
        challenge_data = item.funcargs.get("challenge_data", None)
        difficulty = (
            challenge_data["info"]["difficulty"] if challenge_data else "unknown"
        )
        dependencies = dependencies = (
            challenge_data["dependencies"] if challenge_data else []
        )
        # Extract the challenge_location from the class
        challenge_location: str = getattr(item.cls, "CHALLENGE_LOCATION", "")

        test_details = {
            "difficulty": difficulty,
            "dependencies": dependencies,
            "test": challenge_location,
        }

        print("pytest_runtest_makereport", test_details)
        if call.excinfo is None:
            regression_manager.add_test(item.nodeid.split("::")[1], test_details)
            test_details["success"] = True
        else:
            regression_manager.remove_test(item.nodeid.split("::")[1])
            test_details["success"] = False
            test_details["fail_reason"] = str(call.excinfo.value)

        info_manager.add_test(item.nodeid.split("::")[1], test_details)


def pytest_sessionfinish(session: Any) -> None:
    """Called at the end of the session to save regression tests and info"""
    with open(CONFIG_PATH, "r") as f:
        config = json.load(f)

    info_manager.end_info_report(config)
    regression_manager.save()


# this is adding the dependency marker and category markers automatically from the json
def pytest_collection_modifyitems(items: Any, config: Any) -> None:
    data = get_regression_data()

    for item in items:
        # Assuming item.cls is your test class
        test_class_instance = item.cls()

        # Then you can access your properties
        name = item.parent.cls.__name__
        dependencies = test_class_instance.data.dependencies

        # Filter dependencies if they exist in regression data if its an improvement test
        if config.getoption("--improve"):
            dependencies = [dep for dep in dependencies if not data.get(dep, None)]

        categories = test_class_instance.data.category

        # Add depends marker dynamically
        item.add_marker(pytest.mark.depends(on=dependencies, name=name))

        # Add category marker dynamically
        for category in categories:
            item.add_marker(getattr(pytest.mark, category))