Files
Auto-GPT/autogpts/autogpt/tests/conftest.py
Krzysztof Czerwinski 37904a0f80 feat(agent): Fully abstracted file storage access with FileStorage (#6931)
* Rename `FileWorkspace` to `FileStorage`
   - `autogpt.file_workspace` -> `autogpt.file_storage`
   - `LocalFileWorkspace` -> `LocalFileStorage`
   - `S3FileWorkspace` -> `S3FileStorage`
   - `GCSFileWorkspace` -> `GCSFileStorage`

* Rename `WORKSPACE_BACKEND` to `FILE_STORAGE_BACKEND`
* Rename `WORKSPACE_STORAGE_BUCKET` to `STORAGE_BUCKET`

* Rewrite `AgentManager` to use `FileStorage` rather than direct local file access
* Rename `AgentManager.retrieve_state(..)` method to `load_agent_state`
* Add docstrings to `AgentManager`

* Create `AgentFileManagerMixin` to replace `AgentFileManager`, `FileWorkspaceMixin`, `BaseAgent.attach_fs(..)`
* Replace `BaseAgentSettings.save_to_json_file(..)` method by `AgentFileManagerMixin.save_state()`
* Replace `BaseAgent.set_id(..)` method by `AgentFileManagerMixin.change_agent_id(..)`
* Remove `BaseAgentSettings.load_from_json_file(..)`
* Remove `AgentSettings.agent_data_dir`

* Update `AgentProtocolServer` to work with the new `FileStorage` system and `AgentFileManagerMixin`

* Make `agent_id` and `file_storage` parameters for creating an Agent:
   - `create_agent`, `configure_agent_with_state`, `_configure_agent`, `create_agent_state` in `autogpt.agent_factory.configurators`
   - `generate_agent_for_task` in `autogpt.agent_factory.generators`
   - `Agent.__init__(..)`
   - `BaseAgent.__init__(..)`
   - Initialize and pass in `file_storage` in `autogpt.app.main.run_auto_gpt(..)` and `autogpt.app.main.run_auto_gpt_server(..)`

* Add `clone_with_subroot` to `FileStorage`
* Add `exists`, `make_dir`, `delete_dir`, `rename`, `list_files`, `list_folders` methods to `FileStorage`

* Update `autogpt.commands.file_operations` to use `FileStorage` and `AgentFileManagerMixin` features

* Update tests for `FileStorage` implementations and usages
* Rename `workspace` fixture to `storage`
   * Update conftest.py
2024-03-11 22:26:14 +01:00

154 lines
4.2 KiB
Python

from __future__ import annotations
import os
import uuid
from pathlib import Path
from tempfile import TemporaryDirectory
import pytest
import yaml
from pytest_mock import MockerFixture
from autogpt.agents.agent import Agent, AgentConfiguration, AgentSettings
from autogpt.app.main import _configure_openai_provider
from autogpt.config import AIProfile, Config, ConfigBuilder
from autogpt.core.resource.model_providers import ChatModelProvider, OpenAIProvider
from autogpt.file_storage.local import (
FileStorage,
FileStorageConfiguration,
LocalFileStorage,
)
from autogpt.llm.api_manager import ApiManager
from autogpt.logs.config import configure_logging
from autogpt.models.command_registry import CommandRegistry
pytest_plugins = [
"tests.integration.agent_factory",
"tests.integration.memory.utils",
"tests.vcr",
]
@pytest.fixture()
def tmp_project_root(tmp_path: Path) -> Path:
return tmp_path
@pytest.fixture()
def app_data_dir(tmp_project_root: Path) -> Path:
dir = tmp_project_root / "data"
dir.mkdir(parents=True, exist_ok=True)
return dir
@pytest.fixture()
def storage(app_data_dir: Path) -> FileStorage:
storage = LocalFileStorage(
FileStorageConfiguration(root=app_data_dir, restrict_to_root=False)
)
storage.initialize()
return storage
@pytest.fixture
def temp_plugins_config_file():
"""
Create a plugins_config.yaml file in a temp directory
so that it doesn't mess with existing ones.
"""
config_directory = TemporaryDirectory()
config_file = Path(config_directory.name) / "plugins_config.yaml"
with open(config_file, "w+") as f:
f.write(yaml.dump({}))
yield config_file
@pytest.fixture(scope="function")
def config(
temp_plugins_config_file: Path,
tmp_project_root: Path,
app_data_dir: Path,
mocker: MockerFixture,
):
if not os.environ.get("OPENAI_API_KEY"):
os.environ["OPENAI_API_KEY"] = "sk-dummy"
config = ConfigBuilder.build_config_from_env(project_root=tmp_project_root)
config.app_data_dir = app_data_dir
config.plugins_dir = "tests/unit/data/test_plugins"
config.plugins_config_file = temp_plugins_config_file
config.logging.log_dir = Path(__file__).parent / "logs"
config.logging.plain_console_output = True
config.noninteractive_mode = True
# avoid circular dependency
from autogpt.plugins.plugins_config import PluginsConfig
config.plugins_config = PluginsConfig.load_config(
plugins_config_file=config.plugins_config_file,
plugins_denylist=config.plugins_denylist,
plugins_allowlist=config.plugins_allowlist,
)
yield config
@pytest.fixture(scope="session")
def setup_logger(config: Config):
configure_logging(**config.logging.dict())
@pytest.fixture()
def api_manager() -> ApiManager:
if ApiManager in ApiManager._instances:
del ApiManager._instances[ApiManager]
return ApiManager()
@pytest.fixture
def llm_provider(config: Config) -> OpenAIProvider:
return _configure_openai_provider(config)
@pytest.fixture
def agent(
config: Config, llm_provider: ChatModelProvider, storage: FileStorage
) -> Agent:
ai_profile = AIProfile(
ai_name="Base",
ai_role="A base AI",
ai_goals=[],
)
command_registry = CommandRegistry()
agent_prompt_config = Agent.default_settings.prompt_config.copy(deep=True)
agent_prompt_config.use_functions_api = config.openai_functions
agent_settings = AgentSettings(
name=Agent.default_settings.name,
description=Agent.default_settings.description,
agent_id=f"AutoGPT-test-agent-{str(uuid.uuid4())[:8]}",
ai_profile=ai_profile,
config=AgentConfiguration(
fast_llm=config.fast_llm,
smart_llm=config.smart_llm,
allow_fs_access=not config.restrict_to_workspace,
use_functions_api=config.openai_functions,
plugins=config.plugins,
),
prompt_config=agent_prompt_config,
history=Agent.default_settings.history.copy(deep=True),
)
agent = Agent(
settings=agent_settings,
llm_provider=llm_provider,
command_registry=command_registry,
file_storage=storage,
legacy_config=config,
)
return agent