Merge branch 'master' into tests/basic-spinner-tests

.github/workflows/ci.yml

@@ -14,32 +14,32 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     env:
-      min-python-version: '3.10'
+      min-python-version: "3.10"
 
     steps:
-    - name: Check out repository
-      uses: actions/checkout@v3
+      - name: Check out repository
+        uses: actions/checkout@v3
 
-    - name: Set up Python ${{ env.min-python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ env.min-python-version }}
+      - name: Set up Python ${{ env.min-python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.min-python-version }}
 
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
 
-    - name: Lint with flake8
-      run: flake8
+      - name: Lint with flake8
+        run: flake8
 
-    - name: Check black formatting
-      run: black . --check
-      if: success() || failure()
+      - name: Check black formatting
+        run: black . --check
+        if: success() || failure()
 
-    - name: Check isort formatting
-      run: isort . --check
-      if: success() || failure()
+      - name: Check isort formatting
+        run: isort . --check
+        if: success() || failure()
 
   test:
     permissions:
@@ -53,43 +53,31 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.10', '3.11']
+        python-version: ["3.10", "3.11"]
 
     steps:
-    - name: Check out repository
-      uses: actions/checkout@v3
+      - name: Check out repository
+        uses: actions/checkout@v3
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
 
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
 
-    - name: Run unittest tests with coverage
-      run: |
-        pytest --cov=autogpt --without-integration --without-slow-integration --cov-report term-missing --cov-branch --cov-report xml --cov-report term
+      - name: Run unittest tests with coverage
+        run: |
+          pytest --cov=autogpt --without-integration --without-slow-integration --cov-report term-missing --cov-branch --cov-report xml --cov-report term
 
-    - name: Generate coverage report
-      run: |
-        coverage report
-        coverage xml
-      if: success() || failure()
+      - name: Generate coverage report
+        run: |
+          coverage report
+          coverage xml
+        if: success() || failure()
 
-    - name: Coverage comment
-      id: coverage_comment
-      uses: py-cov-action/python-coverage-comment-action@v3
-      with:
-          GITHUB_TOKEN: ${{ github.token }}
-
-    - name: Store Pull Request comment to be posted
-      uses: actions/upload-artifact@v3
-      if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
-      with:
-        # If you use a different name, update COMMENT_ARTIFACT_NAME accordingly
-        name: python-coverage-comment-action
-        # If you use a different name, update COMMENT_FILENAME accordingly
-        path: python-coverage-comment-action.txt
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3

.github/workflows/coverage.yml

@@ -1,36 +0,0 @@
-# .github/workflows/coverage.yml
-name: Post coverage comment
-
-on:
-  workflow_run:
-    workflows: ["Python CI"]
-    types:
-      - completed
-
-jobs:
-  test:
-    name: Run tests & display coverage
-    runs-on: ubuntu-latest
-    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
-    permissions:
-      # Gives the action the necessary permissions for publishing new
-      # comments in pull requests.
-      pull-requests: write
-      # Gives the action the necessary permissions for editing existing
-      # comments (to avoid publishing multiple comments in the same PR)
-      contents: write
-      # Gives the action the necessary permissions for looking up the
-      # workflow that launched this workflow, and download the related
-      # artifact that contains the comment to be published
-      actions: read
-    steps:
-      # DO NOT run actions/checkout here, for security reasons
-      # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
-      - name: Post comment
-        uses: py-cov-action/python-coverage-comment-action@v3
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}
-          # Update those if you changed the default values:
-          # COMMENT_ARTIFACT_NAME: python-coverage-comment-action
-          # COMMENT_FILENAME: python-coverage-comment-action.txt

autogpt/commands/file_operations.py

@@ -49,7 +49,7 @@ def log_operation(operation: str, filename: str) -> None:
         with open(LOG_FILE_PATH, "w", encoding="utf-8") as f:
             f.write("File Operation Logger ")
 
-    append_to_file(LOG_FILE, log_entry, shouldLog=False)
+    append_to_file(str(LOG_FILE_PATH), log_entry, shouldLog=False)
 
 
 def split_file(
@@ -94,8 +94,8 @@ def read_file(filename: str) -> str:
     Returns:
         str: The contents of the file
     """
+    filepath = path_in_workspace(filename)
     try:
-        filepath = path_in_workspace(filename)
         with open(filepath, "r", encoding="utf-8") as f:
             content = f.read()
         return content

autogpt/commands/google_search.py

@@ -110,7 +110,7 @@ def safe_google_results(results: str | list) -> str:
     """
     if isinstance(results, list):
         safe_message = json.dumps(
-            [result.enocde("utf-8", "ignore") for result in results]
+            [result.encode("utf-8", "ignore") for result in results]
         )
     else:
         safe_message = results.encode("utf-8", "ignore").decode("utf-8")

tests/milvus_memory_test.py

@@ -68,5 +68,5 @@ try:
             stats = self.memory.get_stats()
             self.assertEqual(15, len(stats))
 
-except:
-    print("Milvus not installed, skipping tests")
+except ImportError as err:
+    print(f"Skipping tests for MilvusMemory: {err}")

tests/unit/test_file_operations.py

@@ -0,0 +1,150 @@
+import os
+import shutil
+import unittest
+from pathlib import Path
+
+from autogpt.commands.file_operations import (
+    LOG_FILE_PATH,
+    append_to_file,
+    check_duplicate_operation,
+    delete_file,
+    log_operation,
+    read_file,
+    search_files,
+    split_file,
+    write_to_file,
+)
+from autogpt.config import Config
+from autogpt.workspace import path_in_workspace
+
+
+class TestFileOperations(unittest.TestCase):
+    """
+    This set of unit tests is designed to test the file operations that autoGPT has access to.
+    """
+
+    def setUp(self):
+        self.test_file = "test_file.txt"
+        self.test_file2 = "test_file2.txt"
+        self.test_directory = "test_directory"
+        self.file_content = "This is a test file.\n"
+        self.file_logger_logs = "file_logger.txt"
+
+        with open(path_in_workspace(self.test_file), "w") as f:
+            f.write(self.file_content)
+
+        if os.path.exists(LOG_FILE_PATH):
+            os.remove(LOG_FILE_PATH)
+
+    def tearDown(self):
+        if os.path.exists(path_in_workspace(self.test_file)):
+            os.remove(path_in_workspace(self.test_file))
+
+        if os.path.exists(self.test_directory):
+            shutil.rmtree(self.test_directory)
+
+    def test_check_duplicate_operation(self):
+        log_operation("write", self.test_file)
+        self.assertTrue(check_duplicate_operation("write", self.test_file))
+
+    # Test logging a file operation
+    def test_log_operation(self):
+        if os.path.exists(self.file_logger_logs):
+            os.remove(self.file_logger_logs)
+
+        log_operation("log_test", self.test_file)
+        with open(LOG_FILE_PATH, "r") as f:
+            content = f.read()
+        self.assertIn("log_test: test_file.txt", content)
+
+    # Test splitting a file into chunks
+    def test_split_file(self):
+        content = "abcdefghij"
+        chunks = list(split_file(content, max_length=4, overlap=1))
+        expected = ["abcd", "defg", "ghij"]
+        self.assertEqual(chunks, expected)
+
+    def test_read_file(self):
+        content = read_file(self.test_file)
+        self.assertEqual(content, self.file_content)
+
+    def test_write_to_file(self):
+        new_content = "This is new content.\n"
+        write_to_file(self.test_file, new_content)
+        with open(path_in_workspace(self.test_file), "r") as f:
+            content = f.read()
+        self.assertEqual(content, new_content)
+
+    def test_append_to_file(self):
+        with open(path_in_workspace(self.test_file), "r") as f:
+            content_before = f.read()
+
+        append_text = "This is appended text.\n"
+        append_to_file(self.test_file, append_text)
+        with open(path_in_workspace(self.test_file), "r") as f:
+            content = f.read()
+
+        self.assertEqual(content, content_before + append_text)
+
+    def test_delete_file(self):
+        delete_file(self.test_file)
+        self.assertFalse(os.path.exists(path_in_workspace(self.test_file)))
+
+    def test_search_files(self):
+        # Case 1: Create files A and B, search for A, and ensure we don't return A and B
+        file_a = "file_a.txt"
+        file_b = "file_b.txt"
+
+        with open(path_in_workspace(file_a), "w") as f:
+            f.write("This is file A.")
+
+        with open(path_in_workspace(file_b), "w") as f:
+            f.write("This is file B.")
+
+        # Create a subdirectory and place a copy of file_a in it
+        if not os.path.exists(path_in_workspace(self.test_directory)):
+            os.makedirs(path_in_workspace(self.test_directory))
+
+        with open(
+            path_in_workspace(os.path.join(self.test_directory, file_a)), "w"
+        ) as f:
+            f.write("This is file A in the subdirectory.")
+
+        files = search_files(path_in_workspace(""))
+        self.assertIn(file_a, files)
+        self.assertIn(file_b, files)
+        self.assertIn(os.path.join(self.test_directory, file_a), files)
+
+        # Clean up
+        os.remove(path_in_workspace(file_a))
+        os.remove(path_in_workspace(file_b))
+        os.remove(path_in_workspace(os.path.join(self.test_directory, file_a)))
+        os.rmdir(path_in_workspace(self.test_directory))
+
+        # Case 2: Search for a file that does not exist and make sure we don't throw
+        non_existent_file = "non_existent_file.txt"
+        files = search_files("")
+        self.assertNotIn(non_existent_file, files)
+
+    # Test to ensure we cannot read files out of workspace
+    def test_restrict_workspace(self):
+        CFG = Config()
+        with open(self.test_file2, "w+") as f:
+            f.write("test text")
+
+        CFG.restrict_to_workspace = True
+
+        # Get the absolute path of self.test_file2
+        test_file2_abs_path = os.path.abspath(self.test_file2)
+
+        with self.assertRaises(ValueError):
+            read_file(test_file2_abs_path)
+
+        CFG.restrict_to_workspace = False
+        read_file(test_file2_abs_path)
+
+        os.remove(test_file2_abs_path)
+
+
+if __name__ == "__main__":
+    unittest.main()