remove unused imports automatically (#4449)

* remove unused imports automatically

* add linters to pr template

* remove useless try statement

.github/PULL_REQUEST_TEMPLATE.md

@@ -36,7 +36,13 @@ By following these guidelines, your PRs are more likely to be merged quickly aft
 - [ ] I have considered potential risks and mitigations for my changes.
 - [ ] I have documented my changes clearly and comprehensively.
 - [ ] I have not snuck in any "extra" small tweaks changes. <!-- Submit these as separate Pull Requests, they are the easiest to merge! -->
-- [ ] I have run `black .` and `isort .` against my code to ensure it passes our linter.
+- [ ] I have run the following commands against my code to ensure it passes our linters:
+    ```shell
+    black .
+    isort .
+    mypy
+    autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests --in-place
+    ```
 
 <!-- If you haven't added tests, please explain why. If you have, check the appropriate box. If you've ensured your PR is atomic and well-documented, check the corresponding boxes. -->
 

.github/workflows/ci.yml

@@ -51,6 +51,11 @@ jobs:
         run: mypy
         if: success() || failure()
 
+      - name: Check for unused imports and pass statements
+        run: |
+          cmd="autoflake --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests"
+          $cmd --check || (echo "You have unused imports or pass statements, please run '${cmd} --in-place'" && exit 1)
+
   test:
     permissions:
       # Gives the action the necessary permissions for publishing new

.pre-commit-config.yaml

@@ -21,6 +21,7 @@ repos:
     hooks:
       - id: black
         language_version: python3.10
+
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: 'v1.3.0'
     hooks:
@@ -28,6 +29,11 @@ repos:
 
   - repo: local
     hooks:
+      - id: autoflake
+        name: autoflake
+        entry: autoflake --in-place --remove-all-unused-imports --recursive --ignore-init-module-imports autogpt tests
+        language: python
+        types: [ python ]
       - id: pytest-check
         name: pytest-check
         entry: pytest --cov=autogpt --without-integration --without-slow-integration

autogpt/app.py

@@ -1,6 +1,6 @@
 """ Command and Control """
 import json
-from typing import Dict, List, NoReturn, Union
+from typing import Dict, List, Union
 
 from autogpt.agent.agent_manager import AgentManager
 from autogpt.commands.command import CommandRegistry, command

autogpt/commands/command.py

@@ -1,7 +1,7 @@
 import functools
 import importlib
 import inspect
-from typing import TYPE_CHECKING, Any, Callable, Optional
+from typing import Any, Callable, Optional
 
 from autogpt.config import Config
 from autogpt.logs import logger

autogpt/commands/execute_code.py

@@ -2,7 +2,6 @@
 import os
 import subprocess
 from pathlib import Path
-from typing import TYPE_CHECKING
 
 import docker
 from docker.errors import ImageNotFound

autogpt/commands/file_operations.py

@@ -6,7 +6,6 @@ import os
 import os.path
 from typing import TYPE_CHECKING, Generator, Literal
 
-import charset_normalizer
 import requests
 from colorama import Back, Fore
 from requests.adapters import HTTPAdapter, Retry

autogpt/commands/web_selenium.py

@@ -29,7 +29,7 @@ from webdriver_manager.microsoft import EdgeChromiumDriverManager as EdgeDriverM
 
 from autogpt.commands.command import command
 from autogpt.logs import logger
-from autogpt.memory.vector import MemoryItem, NoMemory, get_memory
+from autogpt.memory.vector import MemoryItem, get_memory
 from autogpt.processing.html import extract_hyperlinks, format_hyperlinks
 from autogpt.url_utils.validators import validate_url
 

autogpt/llm/base.py

@@ -44,15 +44,11 @@ class ModelInfo:
 class ChatModelInfo(ModelInfo):
     """Struct for chat model information."""
 
-    pass
-
 
 @dataclass
 class TextModelInfo(ModelInfo):
     """Struct for text completion model information."""
 
-    pass
-
 
 @dataclass
 class EmbeddingModelInfo(ModelInfo):

autogpt/llm/chat.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import time
-from random import shuffle
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -13,7 +12,6 @@ from autogpt.llm.base import ChatSequence, Message
 from autogpt.llm.utils import count_message_tokens, create_chat_completion
 from autogpt.log_cycle.log_cycle import CURRENT_CONTEXT_FILE_NAME
 from autogpt.logs import logger
-from autogpt.memory.vector import MemoryItem, get_memory
 
 cfg = Config()
 

autogpt/memory/vector/utils.py

@@ -1,7 +1,6 @@
 from typing import Any, overload
 
 import numpy as np
-import numpy.typing as npt
 import openai
 
 from autogpt.config import Config

autogpt/models/base_open_ai_plugin.py

@@ -68,7 +68,6 @@ class BaseOpenAIPlugin(AutoGPTPluginTemplate):
             prompt (PromptGenerator): The prompt generator.
             messages (List[str]): The list of messages.
         """
-        pass
 
     def can_handle_post_planning(self) -> bool:
         """This method is called to check that the plugin can
@@ -116,7 +115,6 @@ class BaseOpenAIPlugin(AutoGPTPluginTemplate):
         Returns:
             Optional[str]: The resulting message.
         """
-        pass
 
     def can_handle_post_instruction(self) -> bool:
         """This method is called to check that the plugin can
@@ -196,7 +194,6 @@ class BaseOpenAIPlugin(AutoGPTPluginTemplate):
         Returns:
             str: The resulting response.
         """
-        pass
 
     def can_handle_text_embedding(self, text: str) -> bool:
         """This method is called to check that the plugin can
@@ -214,7 +211,6 @@ class BaseOpenAIPlugin(AutoGPTPluginTemplate):
         Returns:
             list: The text embedding.
         """
-        pass
 
     def can_handle_user_input(self, user_input: str) -> bool:
         """This method is called to check that the plugin can
@@ -237,8 +233,6 @@ class BaseOpenAIPlugin(AutoGPTPluginTemplate):
             str: The user input.
         """
 
-        pass
-
     def can_handle_report(self) -> bool:
         """This method is called to check that the plugin can
         handle the report method.
@@ -253,4 +247,3 @@ class BaseOpenAIPlugin(AutoGPTPluginTemplate):
         Args:
             message (str): The message to report.
         """
-        pass

autogpt/plugins.py

@@ -5,7 +5,7 @@ import json
 import os
 import zipfile
 from pathlib import Path
-from typing import List, Optional, Tuple
+from typing import List
 from urllib.parse import urlparse
 from zipimport import zipimporter
 

autogpt/singleton.py

@@ -20,5 +20,3 @@ class AbstractSingleton(abc.ABC, metaclass=Singleton):
     """
     Abstract singleton class for ensuring only one instance of a class.
     """
-
-    pass

autogpt/speech/base.py

@@ -37,7 +37,6 @@ class VoiceBase(AbstractSingleton):
         """
         Setup the voices, API key, etc.
         """
-        pass
 
     @abc.abstractmethod
     def _speech(self, text: str, voice_index: int = 0) -> bool:
@@ -47,4 +46,3 @@ class VoiceBase(AbstractSingleton):
         Args:
             text (str): The text to play.
         """
-        pass

autogpt/speech/brian.py

@@ -12,7 +12,6 @@ class BrianSpeech(VoiceBase):
 
     def _setup(self) -> None:
         """Setup the voices, API key, etc."""
-        pass
 
     def _speech(self, text: str, _: int = 0) -> bool:
         """Speak text using Brian with the streamelements API

autogpt/utils.py

@@ -6,15 +6,8 @@ import yaml
 from colorama import Fore, Style
 from git.repo import Repo
 
-from autogpt.logs import logger
-
-# Use readline if available (for clean_input)
-try:
-    import readline
-except ImportError:
-    pass
-
 from autogpt.config import Config
+from autogpt.logs import logger
 
 
 def batch(iterable, max_batch_length: int, overlap: int = 0):

tests/integration/challenges/conftest.py

@@ -2,7 +2,6 @@ import pytest
 from _pytest.config import Config
 from _pytest.config.argparsing import Parser
 from _pytest.fixtures import FixtureRequest
-from _pytest.monkeypatch import MonkeyPatch
 
 
 def pytest_addoption(parser: Parser) -> None:

tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py

@@ -1,8 +1,6 @@
-import typing
-
 import pytest
 
-from autogpt.commands.file_operations import read_file, write_to_file
+from autogpt.commands.file_operations import read_file
 from autogpt.config import Config
 from tests.integration.challenges.utils import run_interaction_loop
 from tests.utils import requires_api_key

tests/integration/challenges/utils.py

@@ -1,7 +1,7 @@
 import contextlib
 import random
 from functools import wraps
-from typing import Any, Callable, Dict, Generator, Optional, Tuple
+from typing import Any, Callable, Dict, Generator, Tuple
 
 import pytest
 

tests/integration/memory/conftest.py

@@ -1,4 +1,3 @@
-import numpy
 import pytest
 
 from autogpt.memory.vector.memory_item import MemoryItem

tests/integration/test_setup.py

@@ -3,11 +3,7 @@ from unittest.mock import patch
 import pytest
 
 from autogpt.config.ai_config import AIConfig
-from autogpt.setup import (
+from autogpt.setup import generate_aiconfig_automatic, prompt_user
-    generate_aiconfig_automatic,
-    generate_aiconfig_manual,
-    prompt_user,
-)
 from tests.utils import requires_api_key
 
 

tests/test_audio_text_read_audio.py

@@ -1,7 +1,6 @@
 # Date: 2023-5-13
 # Author: Generated by GoCodeo.
 import json
-from io import BytesIO
 from unittest.mock import MagicMock, patch
 
 import pytest

tests/test_config.py

@@ -5,7 +5,6 @@ for the AI and ensures it behaves as a singleton.
 from unittest.mock import patch
 
 import pytest
-from openai import InvalidRequestError
 
 from autogpt.configurator import create_config
 

tests/test_image_gen.py

@@ -7,7 +7,6 @@ import pytest
 from PIL import Image
 
 from autogpt.commands.image_gen import generate_image, generate_image_with_sd_webui
-from autogpt.config import Config
 from tests.utils import requires_api_key
 
 

tests/test_text_file_parsers.py

@@ -1,13 +1,11 @@
 import json
 import tempfile
-from functools import partial
 from unittest import TestCase
 from xml.etree import ElementTree
 
 import docx
 import yaml
 from bs4 import BeautifulSoup
-from PyPDF2 import PdfWriter
 
 from autogpt.commands.file_operations_utils import is_file_binary_fn, read_textual_file
 from autogpt.logs import logger

tests/test_utils.py

@@ -1,13 +1,10 @@
 import os
-from unittest.mock import Mock, patch
+from unittest.mock import patch
 
 import pytest
 import requests
-from colorama import Fore
-from git import Repo
 
 from autogpt.utils import (
-    clean_input,
     get_bulletin_from_web,
     get_current_git_branch,
     get_latest_bulletin,

tests/unit/models/test_base_open_api_plugin.py

@@ -1,19 +1,11 @@
-from typing import Any, Dict, List, Optional, Tuple
-
 import pytest
 
-from autogpt.models.base_open_ai_plugin import (
+from autogpt.models.base_open_ai_plugin import BaseOpenAIPlugin
-    BaseOpenAIPlugin,
-    Message,
-    PromptGenerator,
-)
 
 
 class DummyPlugin(BaseOpenAIPlugin):
     """A dummy plugin for testing purposes."""
 
-    pass
-
 
 @pytest.fixture
 def dummy_plugin():

tests/unit/test_browse_scrape_links.py

@@ -2,7 +2,6 @@
 
 # Dependencies:
 # pip install pytest-mock
-import pytest
 
 from autogpt.commands.web_requests import scrape_links
 

tests/unit/test_json_parser.py

@@ -1,7 +1,6 @@
 from unittest import TestCase
 
 from autogpt.json_utils.json_fix_llm import fix_and_parse_json
-from tests.utils import skip_in_ci
 
 
 class TestParseJson(TestCase):

tests/unit/test_json_utils_llm.py

@@ -1,12 +1,9 @@
 # Generated by CodiumAI
-import pytest
-from loguru import logger
 
 from autogpt.json_utils.json_fix_llm import (
     fix_and_parse_json,
     fix_json_using_multiple_techniques,
 )
-from tests.utils import requires_api_key
 
 """
 Code Analysis

tests/unit/test_plugins.py

@@ -1,6 +1,5 @@
 import pytest
 
-from autogpt.config import Config
 from autogpt.plugins import denylist_allowlist_check, inspect_zip_for_modules
 
 PLUGINS_TEST_DIR = "tests/unit/data/test_plugins"

tests/vcr/vcr_filter.py

@@ -1,5 +1,4 @@
 import json
-import os
 import re
 from typing import Any, Dict, List